CE-1328 Add job status messages

docs/actions/QueryAction.adoc

@@ -33,6 +33,9 @@ If the {link-table} has a `POST_QUERY_CUSTOMIZER` defined, then after records ar
 * `table` - *String, Required* - Name of the table being queried against.
 * `filter` - *<<QQueryFilter>> object* - Specification for what records should be returned, based on *<<QFilterCriteria>>* objects, and how they should be sorted, based on *<<QFilterOrderBy>>* objects.
 If a `filter` is not given, then all rows in the table will be returned by the query.
+* `skip` - *Integer* - Optional number of records to be skipped at the beginning of the result set.
+e.g., for implementing pagination.
+* `limit` - *Integer* - Optional maximum number of records to be returned by the query.
 * `transaction` - *QBackendTransaction object* - Optional transaction object.
 ** Behavior for this object is backend-dependant.
 In an RDBMS backend, this object is generally needed if you want your query to see data that may have been modified within the same transaction.
@@ -52,14 +55,6 @@ But if running a query to provide data as part of a process, then this can gener
 * `shouldMaskPassword` - *boolean, default: true* - Controls whether or not fields with `type` = `PASSWORD` should be masked, or if their actual values should be returned.
 * `queryJoins` - *List of <<QueryJoin>> objects* - Optional list of tables to be joined with the main table being queried.
 See QueryJoin below for further details.
-* `fieldNamesToInclude` - *Set of String* - Optional set of field names to be included in the records.
-** Fields from a queryJoin must be prefixed by the join table's name or alias, and a period.
-Field names from the table being queried should not have any sort of prefix.
-** A `null` set here (default) means to include all fields from the table and any queryJoins set as select=true.
-** An empty set will cause an error, as well any unrecognized field names.
-** `QueryAction` will validate the set of field names, and throw an exception if any unrecognized names are given.
-** _Note that this is an optional feature, which some backend modules may not implement.
-Meaning, they would always return all fields._
 
 ==== QQueryFilter
 A key component of *<<QueryInput>>*, a *QQueryFilter* defines both what records should be included in a query's results (e.g., an SQL `WHERE`), as well as how those results should be sorted (SQL `ORDER BY`).
@@ -73,9 +68,6 @@ In general, multiple *orderBys* can be given (depending on backend implementatio
 ** Each *subFilter* can include its own additional *subFilters*.
 ** Each *subFilter* can specify a different *booleanOperator*.
 ** For example, consider the following *QQueryFilter*, that uses two *subFilters*, and a mix of *booleanOperators*
-* `skip` - *Integer* - Optional number of records to be skipped at the beginning of the result set.
-e.g., for implementing pagination.
-* `limit` - *Integer* - Optional maximum number of records to be returned by the query.
 
 [source,java]
 ----

pom.xml

@@ -46,7 +46,7 @@
    </modules>
 
    <properties>
-      <revision>0.21.0</revision>
+      <revision>0.21.0-SNAPSHOT</revision>
 
       <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
       <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/RecordCustomizerUtilityInterface.java

@@ -24,12 +24,10 @@ package com.kingsrook.qqq.backend.core.actions.customizers;
 
 import java.io.Serializable;
 import java.util.HashMap;
-import java.util.List;
 import java.util.Map;
 import java.util.Objects;
 import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.logging.QLogger;
-import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
@@ -145,19 +143,4 @@ public interface RecordCustomizerUtilityInterface
       }
    }
 
-
-   /*******************************************************************************
-    **
-    *******************************************************************************/
-   default Map<Serializable, QRecord> getOldRecordMap(List<QRecord> oldRecordList, UpdateInput updateInput)
-   {
-      Map<Serializable, QRecord> oldRecordMap = new HashMap<>();
-      for(QRecord qRecord : oldRecordList)
-      {
-         oldRecordMap.put(qRecord.getValue(updateInput.getTable().getPrimaryKeyField()), qRecord);
-      }
-
-      return (oldRecordMap);
-   }
-
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/GenerateReportAction.java

@@ -42,7 +42,6 @@ import com.kingsrook.qqq.backend.core.actions.AbstractQActionFunction;
 import com.kingsrook.qqq.backend.core.actions.async.AsyncRecordPipeLoop;
 import com.kingsrook.qqq.backend.core.actions.customizers.QCodeLoader;
 import com.kingsrook.qqq.backend.core.actions.reporting.customizers.DataSourceQueryInputCustomizer;
-import com.kingsrook.qqq.backend.core.actions.reporting.customizers.ReportCustomRecordSourceInterface;
 import com.kingsrook.qqq.backend.core.actions.reporting.customizers.ReportViewCustomizer;
 import com.kingsrook.qqq.backend.core.actions.tables.CountAction;
 import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
@@ -303,19 +302,10 @@ public class GenerateReportAction extends AbstractQActionFunction<ReportInput, R
       JoinsContext joinsContext = null;
       if(dataSource != null)
       {
-         ///////////////////////////////////////////////////////////////////////////////////////
-         // count records, if applicable, from the data source - for populating into the      //
-         // countByDataSource map, as well as for checking if too many rows (e.g., for excel) //
-         ///////////////////////////////////////////////////////////////////////////////////////
-         countDataSourceRecords(reportInput, dataSource, reportFormat);
-
-         ///////////////////////////////////////////////////////////////////////////////////////////
-         // if there's a source table, set up a joins context, to use below for looking up fields //
-         ///////////////////////////////////////////////////////////////////////////////////////////
          if(StringUtils.hasContent(dataSource.getSourceTable()))
          {
-            QQueryFilter queryFilter = dataSource.getQueryFilter() == null ? new QQueryFilter() : dataSource.getQueryFilter().clone();
-            joinsContext = new JoinsContext(QContext.getQInstance(), dataSource.getSourceTable(), dataSource.getQueryJoins(), queryFilter);
+            joinsContext = new JoinsContext(QContext.getQInstance(), dataSource.getSourceTable(), cloneDataSourceQueryJoins(dataSource), dataSource.getQueryFilter() == null ? null : dataSource.getQueryFilter().clone());
+            countDataSourceRecords(reportInput, dataSource, reportFormat);
          }
       }
 
@@ -339,7 +329,6 @@ public class GenerateReportAction extends AbstractQActionFunction<ReportInput, R
             field.setName(column.getName());
             if(StringUtils.hasContent(column.getLabel()))
             {
-
                field.setLabel(column.getLabel());
             }
             fields.add(field);
@@ -357,33 +346,23 @@ public class GenerateReportAction extends AbstractQActionFunction<ReportInput, R
     *******************************************************************************/
    private void countDataSourceRecords(ReportInput reportInput, QReportDataSource dataSource, ReportFormat reportFormat) throws QException
    {
-      Integer count = null;
-      if(dataSource.getCustomRecordSource() != null)
+      QQueryFilter queryFilter = dataSource.getQueryFilter() == null ? new QQueryFilter() : dataSource.getQueryFilter().clone();
+      setInputValuesInQueryFilter(reportInput, queryFilter);
+
+      CountInput countInput = new CountInput();
+      countInput.setTableName(dataSource.getSourceTable());
+      countInput.setFilter(queryFilter);
+      countInput.setQueryJoins(cloneDataSourceQueryJoins(dataSource));
+      CountOutput countOutput = new CountAction().execute(countInput);
+
+      if(countOutput.getCount() != null)
       {
-         // todo - add `count` method to interface?
-      }
-      else if(StringUtils.hasContent(dataSource.getSourceTable()))
-      {
-         QQueryFilter queryFilter = dataSource.getQueryFilter() == null ? new QQueryFilter() : dataSource.getQueryFilter().clone();
-         setInputValuesInQueryFilter(reportInput, queryFilter);
+         countByDataSource.put(dataSource.getName(), countOutput.getCount());
 
-         CountInput countInput = new CountInput();
-         countInput.setTableName(dataSource.getSourceTable());
-         countInput.setFilter(queryFilter);
-         countInput.setQueryJoins(cloneDataSourceQueryJoins(dataSource));
-         CountOutput countOutput = new CountAction().execute(countInput);
-
-         count = countOutput.getCount();
-      }
-
-      if(count != null)
-      {
-         countByDataSource.put(dataSource.getName(), count);
-
-         if(reportFormat.getMaxRows() != null && count > reportFormat.getMaxRows())
+         if(reportFormat.getMaxRows() != null && countOutput.getCount() > reportFormat.getMaxRows())
          {
             throw (new QUserFacingException("The requested report would include more rows ("
-               + String.format("%,d", count) + ") than the maximum allowed ("
+               + String.format("%,d", countOutput.getCount()) + ") than the maximum allowed ("
                + String.format("%,d", reportFormat.getMaxRows()) + ") for the selected file format (" + reportFormat + ")."));
          }
       }
@@ -444,19 +423,13 @@ public class GenerateReportAction extends AbstractQActionFunction<ReportInput, R
       String        tableLabel    = ObjectUtils.tryElse(() -> QContext.getQInstance().getTable(dataSource.getSourceTable()).getLabel(), Objects.requireNonNullElse(dataSource.getSourceTable(), ""));
       AtomicInteger consumedCount = new AtomicInteger(0);
 
-      /////////////////////////////////////////////////////////////////////////////////////////////////
-      // run a record pipe loop, over the query (or other data-supplier/source) for this data source //
-      /////////////////////////////////////////////////////////////////////////////////////////////////
+      /////////////////////////////////////////////////////////////////
+      // run a record pipe loop, over the query for this data source //
+      /////////////////////////////////////////////////////////////////
       RecordPipe recordPipe = new BufferedRecordPipe(1000);
       new AsyncRecordPipeLoop().run("Report[" + reportInput.getReportName() + "]", null, recordPipe, (callback) ->
       {
-         if(dataSource.getCustomRecordSource() != null)
-         {
-            ReportCustomRecordSourceInterface recordSource = QCodeLoader.getAdHoc(ReportCustomRecordSourceInterface.class, dataSource.getCustomRecordSource());
-            recordSource.execute(reportInput, dataSource, recordPipe);
-            return (true);
-         }
-         else if(dataSource.getSourceTable() != null)
+         if(dataSource.getSourceTable() != null)
          {
             QQueryFilter queryFilter = dataSource.getQueryFilter() == null ? new QQueryFilter() : dataSource.getQueryFilter().clone();
             setInputValuesInQueryFilter(reportInput, queryFilter);

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/excel/poi/ExcelPoiBasedStreamingExportStreamer.java

@@ -124,11 +124,10 @@ public class ExcelPoiBasedStreamingExportStreamer implements ExportStreamerInter
    private Writer              activeSheetWriter = null;
    private StreamedSheetWriter sheetWriter       = null;
 
-   private QReportView                       currentView              = null;
-   private Map<String, List<QFieldMetaData>> fieldsPerView            = new HashMap<>();
-   private Map<String, Integer>              rowsPerView              = new HashMap<>();
-   private Map<String, String>               labelViewsByName         = new HashMap<>();
-   private Map<String, String>               sheetReferenceByViewName = new HashMap<>();
+   private QReportView                       currentView      = null;
+   private Map<String, List<QFieldMetaData>> fieldsPerView    = new HashMap<>();
+   private Map<String, Integer>              rowsPerView      = new HashMap<>();
+   private Map<String, String>               labelViewsByName = new HashMap<>();
 
 
 
@@ -181,7 +180,6 @@ public class ExcelPoiBasedStreamingExportStreamer implements ExportStreamerInter
             String    sheetReference = sheet.getPackagePart().getPartName().getName().substring(1);
             sheetMapByExcelReference.put(sheetReference, sheet);
             sheetMapByViewName.put(view.getName(), sheet);
-            sheetReferenceByViewName.put(view.getName(), sheetReference);
             sheetCounter++;
          }
 
@@ -448,7 +446,7 @@ public class ExcelPoiBasedStreamingExportStreamer implements ExportStreamerInter
          // - with a new output stream writer    //
          // - and with a SpreadsheetWriter       //
          //////////////////////////////////////////
-         zipOutputStream.putNextEntry(new ZipEntry(sheetReferenceByViewName.get(view.getName())));
+         zipOutputStream.putNextEntry(new ZipEntry("xl/worksheets/sheet" + this.sheetIndex++ + ".xml"));
          activeSheetWriter = new OutputStreamWriter(zipOutputStream);
          sheetWriter = new StreamedSheetWriter(activeSheetWriter);
 

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/QueryAction.java

@@ -26,7 +26,6 @@ import java.io.Serializable;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
-import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
@@ -51,7 +50,6 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperat
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
-import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryJoin;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
 import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
@@ -66,7 +64,6 @@ import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleDispatcher;
 import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleInterface;
 import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
 import com.kingsrook.qqq.backend.core.utils.ListingHash;
-import com.kingsrook.qqq.backend.core.utils.StringUtils;
 import com.kingsrook.qqq.backend.core.utils.ValueUtils;
 
 
@@ -104,8 +101,6 @@ public class QueryAction
          throw (new QException("A table named [" + queryInput.getTableName() + "] was not found in the active QInstance"));
       }
 
-      validateFieldNamesToInclude(queryInput);
-
       QBackendMetaData backend = queryInput.getBackend();
       postQueryRecordCustomizer = QCodeLoader.getTableCustomizer(table, TableCustomizers.POST_QUERY_RECORD.getRole());
       this.queryInput = queryInput;
@@ -163,109 +158,6 @@ public class QueryAction
 
 
 
-   /***************************************************************************
-    ** if QueryInput contains a set of FieldNamesToInclude, then validate that
-    ** those are known field names in the table being queried, or a selected
-    ** queryJoin.
-    ***************************************************************************/
-   static void validateFieldNamesToInclude(QueryInput queryInput) throws QException
-   {
-      Set<String> fieldNamesToInclude = queryInput.getFieldNamesToInclude();
-      if(fieldNamesToInclude == null)
-      {
-         ////////////////////////////////
-         // null set means select all. //
-         ////////////////////////////////
-         return;
-      }
-
-      if(fieldNamesToInclude.isEmpty())
-      {
-         /////////////////////////////////////
-         // empty set, however, is an error //
-         /////////////////////////////////////
-         throw (new QException("An empty set of fieldNamesToInclude was given as queryInput, which is not allowed."));
-      }
-
-      List<String>                unrecognizedFieldNames = new ArrayList<>();
-      Map<String, QTableMetaData> selectedQueryJoins     = null;
-      for(String fieldName : fieldNamesToInclude)
-      {
-         if(fieldName.contains("."))
-         {
-            ////////////////////////////////////////////////
-            // handle names with dots - fields from joins //
-            ////////////////////////////////////////////////
-            String[] parts = fieldName.split("\\.");
-            if(parts.length != 2)
-            {
-               unrecognizedFieldNames.add(fieldName);
-            }
-            else
-            {
-               String tableOrAlias  = parts[0];
-               String fieldNamePart = parts[1];
-
-               ////////////////////////////////////////////
-               // build map of queryJoins being selected //
-               ////////////////////////////////////////////
-               if(selectedQueryJoins == null)
-               {
-                  selectedQueryJoins = new HashMap<>();
-                  for(QueryJoin queryJoin : CollectionUtils.nonNullList(queryInput.getQueryJoins()))
-                  {
-                     if(queryJoin.getSelect())
-                     {
-                        String         joinTableOrAlias = queryJoin.getJoinTableOrItsAlias();
-                        QTableMetaData joinTable        = QContext.getQInstance().getTable(queryJoin.getJoinTable());
-                        if(joinTable != null)
-                        {
-                           selectedQueryJoins.put(joinTableOrAlias, joinTable);
-                        }
-                     }
-                  }
-               }
-
-               if(!selectedQueryJoins.containsKey(tableOrAlias))
-               {
-                  ///////////////////////////////////////////
-                  // unrecognized tableOrAlias is an error //
-                  ///////////////////////////////////////////
-                  unrecognizedFieldNames.add(fieldName);
-               }
-               else
-               {
-                  QTableMetaData joinTable = selectedQueryJoins.get(tableOrAlias);
-                  if(!joinTable.getFields().containsKey(fieldNamePart))
-                  {
-                     //////////////////////////////////////////////////////////
-                     // unrecognized field within the join table is an error //
-                     //////////////////////////////////////////////////////////
-                     unrecognizedFieldNames.add(fieldName);
-                  }
-               }
-            }
-         }
-         else
-         {
-            ///////////////////////////////////////////////////////////////////////
-            // non-join fields - just ensure field name is in table's fields map //
-            ///////////////////////////////////////////////////////////////////////
-            if(!queryInput.getTable().getFields().containsKey(fieldName))
-            {
-               unrecognizedFieldNames.add(fieldName);
-            }
-         }
-      }
-
-      if(!unrecognizedFieldNames.isEmpty())
-      {
-         throw (new QException("QueryInput contained " + unrecognizedFieldNames.size() + " unrecognized field name" + StringUtils.plural(unrecognizedFieldNames) + ": " + StringUtils.join(",", unrecognizedFieldNames)));
-      }
-   }
-
-
-
    /*******************************************************************************
     ** shorthand way to call for the most common use-case, when you just want the
     ** records to be returned, and you just want to pass in a table name and filter.

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidator.java

@@ -44,7 +44,6 @@ import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizers;
 import com.kingsrook.qqq.backend.core.actions.dashboard.widgets.AbstractWidgetRenderer;
 import com.kingsrook.qqq.backend.core.actions.metadata.JoinGraph;
 import com.kingsrook.qqq.backend.core.actions.processes.BackendStep;
-import com.kingsrook.qqq.backend.core.actions.reporting.customizers.ReportCustomRecordSourceInterface;
 import com.kingsrook.qqq.backend.core.actions.scripts.TestScriptActionInterface;
 import com.kingsrook.qqq.backend.core.actions.values.QCustomPossibleValueProvider;
 import com.kingsrook.qqq.backend.core.exceptions.QInstanceValidationException;
@@ -1661,12 +1660,9 @@ public class QInstanceValidator
 
                   String dataSourceErrorPrefix = "Report " + reportName + " data source " + dataSource.getName() + " ";
 
-                  boolean hasASource = false;
-
                   if(StringUtils.hasContent(dataSource.getSourceTable()))
                   {
-                     hasASource = true;
-                     assertCondition(dataSource.getStaticDataSupplier() == null, dataSourceErrorPrefix + "has both a sourceTable and a staticDataSupplier (not compatible together).");
+                     assertCondition(dataSource.getStaticDataSupplier() == null, dataSourceErrorPrefix + "has both a sourceTable and a staticDataSupplier (exactly 1 is required).");
                      if(assertCondition(qInstance.getTable(dataSource.getSourceTable()) != null, dataSourceErrorPrefix + "source table " + dataSource.getSourceTable() + " is not a table in this instance."))
                      {
                         if(dataSource.getQueryFilter() != null)
@@ -1675,21 +1671,14 @@ public class QInstanceValidator
                         }
                      }
                   }
-
-                  if(dataSource.getStaticDataSupplier() != null)
+                  else if(dataSource.getStaticDataSupplier() != null)
                   {
-                     assertCondition(dataSource.getCustomRecordSource() == null, dataSourceErrorPrefix + "has both a staticDataSupplier and a customRecordSource (not compatible together).");
-                     hasASource = true;
                      validateSimpleCodeReference(dataSourceErrorPrefix, dataSource.getStaticDataSupplier(), Supplier.class);
                   }
-
-                  if(dataSource.getCustomRecordSource() != null)
+                  else
                   {
-                     hasASource = true;
-                     validateSimpleCodeReference(dataSourceErrorPrefix, dataSource.getCustomRecordSource(), ReportCustomRecordSourceInterface.class);
+                     errors.add(dataSourceErrorPrefix + "does not have a sourceTable or a staticDataSupplier (exactly 1 is required).");
                   }
-
-                  assertCondition(hasASource, dataSourceErrorPrefix + "does not have a sourceTable, customRecordSource, or a staticDataSupplier.");
                }
             }
 

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QueryInput.java

@@ -66,14 +66,6 @@ public class QueryInput extends AbstractTableActionInput implements QueryOrGetIn
    private List<QueryJoin> queryJoins     = null;
    private boolean         selectDistinct = false;
 
-   /////////////////////////////////////////////////////////////////////////////
-   // if this set is null, then the default (all fields) should be included   //
-   // if it's an empty set, that should throw an error                        //
-   // or if there are any fields in it that aren't valid fields on the table, //
-   // or in a selected queryJoin.                                             //
-   /////////////////////////////////////////////////////////////////////////////
-   private Set<String> fieldNamesToInclude;
-
    ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
    // if you say you want to includeAssociations, you can limit which ones by passing them in associationNamesToInclude. //
    // if you leave it null, you get all associations defined on the table.  if you pass it as empty, you get none.       //
@@ -694,35 +686,4 @@ public class QueryInput extends AbstractTableActionInput implements QueryOrGetIn
       return (queryHints.contains(queryHint));
    }
 
-
-
-   /*******************************************************************************
-    ** Getter for fieldNamesToInclude
-    *******************************************************************************/
-   public Set<String> getFieldNamesToInclude()
-   {
-      return (this.fieldNamesToInclude);
-   }
-
-
-
-   /*******************************************************************************
-    ** Setter for fieldNamesToInclude
-    *******************************************************************************/
-   public void setFieldNamesToInclude(Set<String> fieldNamesToInclude)
-   {
-      this.fieldNamesToInclude = fieldNamesToInclude;
-   }
-
-
-
-   /*******************************************************************************
-    ** Fluent setter for fieldNamesToInclude
-    *******************************************************************************/
-   public QueryInput withFieldNamesToInclude(Set<String> fieldNamesToInclude)
-   {
-      this.fieldNamesToInclude = fieldNamesToInclude;
-      return (this);
-   }
-
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/dashboard/widgets/CompositeWidgetData.java

@@ -40,10 +40,9 @@ public class CompositeWidgetData extends AbstractBlockWidgetData<CompositeWidget
 {
    private List<AbstractBlockWidgetData<?, ?, ?, ?>> blocks = new ArrayList<>();
 
-   private Layout                    layout;
-   private Map<String, Serializable> styleOverrides        = new HashMap<>();
-   private String                    overlayHtml;
-   private Map<String, Serializable> overlayStyleOverrides = new HashMap<>();
+   private Map<String, Serializable> styleOverrides = new HashMap<>();
+
+   private Layout layout;
 
 
 
@@ -219,91 +218,4 @@ public class CompositeWidgetData extends AbstractBlockWidgetData<CompositeWidget
       return (this);
    }
 
-
-
-   /*******************************************************************************
-    ** Getter for overlayHtml
-    *******************************************************************************/
-   public String getOverlayHtml()
-   {
-      return (this.overlayHtml);
-   }
-
-
-
-   /*******************************************************************************
-    ** Setter for overlayHtml
-    *******************************************************************************/
-   public void setOverlayHtml(String overlayHtml)
-   {
-      this.overlayHtml = overlayHtml;
-   }
-
-
-
-   /*******************************************************************************
-    ** Fluent setter for overlayHtml
-    *******************************************************************************/
-   public CompositeWidgetData withOverlayHtml(String overlayHtml)
-   {
-      this.overlayHtml = overlayHtml;
-      return (this);
-   }
-
-
-
-   /*******************************************************************************
-    ** Getter for overlayStyleOverrides
-    *******************************************************************************/
-   public Map<String, Serializable> getOverlayStyleOverrides()
-   {
-      return (this.overlayStyleOverrides);
-   }
-
-
-
-   /*******************************************************************************
-    ** Setter for overlayStyleOverrides
-    *******************************************************************************/
-   public void setOverlayStyleOverrides(Map<String, Serializable> overlayStyleOverrides)
-   {
-      this.overlayStyleOverrides = overlayStyleOverrides;
-   }
-
-
-
-   /*******************************************************************************
-    ** Fluent setter for overlayStyleOverrides
-    *******************************************************************************/
-   public CompositeWidgetData withOverlayStyleOverrides(Map<String, Serializable> overlayStyleOverrides)
-   {
-      this.overlayStyleOverrides = overlayStyleOverrides;
-      return (this);
-   }
-
-
-
-   /*******************************************************************************
-    **
-    *******************************************************************************/
-   public CompositeWidgetData withOverlayStyleOverride(String key, Serializable value)
-   {
-      addOverlayStyleOverride(key, value);
-      return (this);
-   }
-
-
-
-   /*******************************************************************************
-    **
-    *******************************************************************************/
-   public void addOverlayStyleOverride(String key, Serializable value)
-   {
-      if(this.overlayStyleOverrides == null)
-      {
-         this.overlayStyleOverrides = new HashMap<>();
-      }
-      this.overlayStyleOverrides.put(key, value);
-   }
-
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/dashboard/widgets/blocks/AbstractBlockWidgetData.java

@@ -24,7 +24,6 @@ package com.kingsrook.qqq.backend.core.model.dashboard.widgets.blocks;
 
 import java.util.HashMap;
 import java.util.Map;
-import com.kingsrook.qqq.backend.core.model.dashboard.widgets.CompositeWidgetData;
 import com.kingsrook.qqq.backend.core.model.dashboard.widgets.QWidgetData;
 
 
@@ -204,19 +203,6 @@ public abstract class AbstractBlockWidgetData<
 
 
 
-   /*******************************************************************************
-    ** Fluent setter for tooltip
-    **
-    *******************************************************************************/
-   @SuppressWarnings("unchecked")
-   public T withTooltip(CompositeWidgetData data)
-   {
-      this.tooltip = new BlockTooltip(data);
-      return (T) (this);
-   }
-
-
-
    /*******************************************************************************
     **
     *******************************************************************************/
@@ -412,7 +398,6 @@ public abstract class AbstractBlockWidgetData<
    }
 
 
-
    /*******************************************************************************
     ** Getter for blockId
     *******************************************************************************/
@@ -443,4 +428,5 @@ public abstract class AbstractBlockWidgetData<
       return (T) this;
    }
 
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/dashboard/widgets/blocks/BlockTooltip.java

@@ -22,18 +22,14 @@
 package com.kingsrook.qqq.backend.core.model.dashboard.widgets.blocks;
 
 
-import com.kingsrook.qqq.backend.core.model.dashboard.widgets.CompositeWidgetData;
-
-
 /*******************************************************************************
  ** A tooltip used within a (widget) block.
  **
  *******************************************************************************/
 public class BlockTooltip
 {
-   private CompositeWidgetData blockData;
-   private String              title;
-   private Placement           placement = Placement.BOTTOM;
+   private String    title;
+   private Placement placement = Placement.BOTTOM;
 
 
 
@@ -66,17 +62,6 @@ public class BlockTooltip
 
 
 
-   /*******************************************************************************
-    ** Constructor
-    **
-    *******************************************************************************/
-   public BlockTooltip(CompositeWidgetData blockData)
-   {
-      this.blockData = blockData;
-   }
-
-
-
    /*******************************************************************************
     ** Getter for title
     *******************************************************************************/
@@ -137,35 +122,4 @@ public class BlockTooltip
       return (this);
    }
 
-
-
-   /*******************************************************************************
-    ** Getter for blockData
-    *******************************************************************************/
-   public CompositeWidgetData getBlockData()
-   {
-      return (this.blockData);
-   }
-
-
-
-   /*******************************************************************************
-    ** Setter for blockData
-    *******************************************************************************/
-   public void setBlockData(CompositeWidgetData blockData)
-   {
-      this.blockData = blockData;
-   }
-
-
-
-   /*******************************************************************************
-    ** Fluent setter for blockData
-    *******************************************************************************/
-   public BlockTooltip withBlockData(CompositeWidgetData blockData)
-   {
-      this.blockData = blockData;
-      return (this);
-   }
-
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/QBackendMetaData.java

@@ -53,8 +53,6 @@ public class QBackendMetaData implements TopLevelMetaDataInterface
    private String  variantOptionsTableUsernameField;
    private String  variantOptionsTablePasswordField;
    private String  variantOptionsTableApiKeyField;
-   private String  variantOptionsTableClientIdField;
-   private String  variantOptionsTableClientSecretField;
    private String  variantOptionsTableName;
 
    // todo - at some point, we may want to apply this to secret properties on subclasses?
@@ -650,66 +648,4 @@ public class QBackendMetaData implements TopLevelMetaDataInterface
    {
       qInstance.addBackend(this);
    }
-
-   /*******************************************************************************
-    ** Getter for variantOptionsTableClientIdField
-    *******************************************************************************/
-   public String getVariantOptionsTableClientIdField()
-   {
-      return (this.variantOptionsTableClientIdField);
-   }
-
-
-
-   /*******************************************************************************
-    ** Setter for variantOptionsTableClientIdField
-    *******************************************************************************/
-   public void setVariantOptionsTableClientIdField(String variantOptionsTableClientIdField)
-   {
-      this.variantOptionsTableClientIdField = variantOptionsTableClientIdField;
-   }
-
-
-
-   /*******************************************************************************
-    ** Fluent setter for variantOptionsTableClientIdField
-    *******************************************************************************/
-   public QBackendMetaData withVariantOptionsTableClientIdField(String variantOptionsTableClientIdField)
-   {
-      this.variantOptionsTableClientIdField = variantOptionsTableClientIdField;
-      return (this);
-   }
-
-
-
-   /*******************************************************************************
-    ** Getter for variantOptionsTableClientSecretField
-    *******************************************************************************/
-   public String getVariantOptionsTableClientSecretField()
-   {
-      return (this.variantOptionsTableClientSecretField);
-   }
-
-
-
-   /*******************************************************************************
-    ** Setter for variantOptionsTableClientSecretField
-    *******************************************************************************/
-   public void setVariantOptionsTableClientSecretField(String variantOptionsTableClientSecretField)
-   {
-      this.variantOptionsTableClientSecretField = variantOptionsTableClientSecretField;
-   }
-
-
-
-   /*******************************************************************************
-    ** Fluent setter for variantOptionsTableClientSecretField
-    *******************************************************************************/
-   public QBackendMetaData withVariantOptionsTableClientSecretField(String variantOptionsTableClientSecretField)
-   {
-      this.variantOptionsTableClientSecretField = variantOptionsTableClientSecretField;
-      return (this);
-   }
-
-
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/reporting/QReportDataSource.java

@@ -32,13 +32,6 @@ import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
 /*******************************************************************************
  ** Meta-data definition of a source of data for a report (e.g., a table and query
  ** filter or custom-code reference).
- **
- ** Runs in 3 modes:
- **
- ** - If a customRecordSource is specified, then that code is executed to get the records.
- ** - else, if a sourceTable is specified, then the corresponding queryFilter
- ** (optionally along with queryJoins and queryInputCustomizer) is used.
- ** - else a staticDataSupplier is used.
  *******************************************************************************/
 public class QReportDataSource
 {
@@ -51,7 +44,6 @@ public class QReportDataSource
 
    private QCodeReference queryInputCustomizer;
    private QCodeReference staticDataSupplier;
-   private QCodeReference customRecordSource;
 
 
 
@@ -273,35 +265,4 @@ public class QReportDataSource
       return (this);
    }
 
-
-   /*******************************************************************************
-    ** Getter for customRecordSource
-    *******************************************************************************/
-   public QCodeReference getCustomRecordSource()
-   {
-      return (this.customRecordSource);
-   }
-
-
-
-   /*******************************************************************************
-    ** Setter for customRecordSource
-    *******************************************************************************/
-   public void setCustomRecordSource(QCodeReference customRecordSource)
-   {
-      this.customRecordSource = customRecordSource;
-   }
-
-
-
-   /*******************************************************************************
-    ** Fluent setter for customRecordSource
-    *******************************************************************************/
-   public QReportDataSource withCustomRecordSource(QCodeReference customRecordSource)
-   {
-      this.customRecordSource = customRecordSource;
-      return (this);
-   }
-
-
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/garbagecollector/GenericGarbageCollectorExecuteStep.java

@@ -0,0 +1,287 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.processes.implementations.garbagecollector;
+
+
+import java.time.Instant;
+import java.util.ArrayList;
+import java.util.List;
+import com.kingsrook.qqq.backend.core.actions.async.AsyncJobCallback;
+import com.kingsrook.qqq.backend.core.actions.processes.BackendStep;
+import com.kingsrook.qqq.backend.core.actions.tables.AggregateAction;
+import com.kingsrook.qqq.backend.core.actions.tables.DeleteAction;
+import com.kingsrook.qqq.backend.core.context.QContext;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.exceptions.QUserFacingException;
+import com.kingsrook.qqq.backend.core.logging.QLogger;
+import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLine;
+import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLineInterface;
+import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
+import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
+import com.kingsrook.qqq.backend.core.model.actions.processes.Status;
+import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.Aggregate;
+import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateOperator;
+import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateOutput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateResult;
+import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteOutput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess;
+import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
+import com.kingsrook.qqq.backend.core.utils.ValueUtils;
+import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
+
+
+/*******************************************************************************
+ **
+ *******************************************************************************/
+public class GenericGarbageCollectorExecuteStep implements BackendStep
+{
+   private static final QLogger LOG = QLogger.getLogger(GenericGarbageCollectorExecuteStep.class);
+
+   private ProcessSummaryLine partitionsLine = new ProcessSummaryLine(Status.INFO)
+      .withSingularPastMessage("partition was processed")
+      .withPluralPastMessage("partitions were processed");
+
+   private ProcessSummaryLine deletedLine = new ProcessSummaryLine(Status.OK)
+      .withSingularPastMessage("record was deleted")
+      .withPluralPastMessage("records were deleted");
+
+   private ProcessSummaryLine warningLine = new ProcessSummaryLine(Status.WARNING, "had an warning");
+   private ProcessSummaryLine errorLine   = new ProcessSummaryLine(Status.ERROR, "had an error");
+
+   private AsyncJobCallback asyncJobCallback;
+
+   private Integer total        = null;
+   private Integer deletedSoFar = 0;
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   @Override
+   public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException
+   {
+      String         tableName = runBackendStepInput.getValueString("table");
+      QTableMetaData table     = QContext.getQInstance().getTable(tableName);
+      if(table == null)
+      {
+         throw new QUserFacingException("Unrecognized table: " + tableName);
+      }
+
+      String         fieldName = runBackendStepInput.getValueString("field");
+      QFieldMetaData field     = table.getFields().get(fieldName);
+      if(field == null)
+      {
+         throw new QUserFacingException("Unrecognized field: " + fieldName);
+      }
+
+      if(!QFieldType.DATE_TIME.equals(field.getType()) && !QFieldType.DATE.equals(field.getType()))
+      {
+         throw new QUserFacingException("Field " + field + " is not a date-time or date type field.");
+      }
+
+      Integer daysBack = runBackendStepInput.getValueInteger("daysBack");
+      if(daysBack == null || daysBack < 0)
+      {
+         throw new QUserFacingException("Illegal value for daysBack: " + daysBack + "; Must be positive.");
+      }
+
+      Integer maxPageSize = runBackendStepInput.getValueInteger("maxPageSize");
+      if(maxPageSize == null || maxPageSize < 0)
+      {
+         throw new QUserFacingException("Illegal value for maxPageSize: " + maxPageSize + "; Must be positive.");
+      }
+
+      asyncJobCallback = runBackendStepInput.getAsyncJobCallback();
+      long start = System.currentTimeMillis();
+      execute(table, field, daysBack, maxPageSize);
+      long end = System.currentTimeMillis();
+
+      deletedLine.prepareForFrontend(true);
+      partitionsLine.prepareForFrontend(true);
+
+      ArrayList<ProcessSummaryLineInterface> processSummary = new ArrayList<>();
+      processSummary.add(partitionsLine);
+      processSummary.add(deletedLine);
+      warningLine.addSelfToListIfAnyCount(processSummary);
+      errorLine.addSelfToListIfAnyCount(processSummary);
+      processSummary.add(new ProcessSummaryLine(Status.INFO, "Total time:  " + String.format("%,d", ((end - start) / 1000)) + " seconds"));
+      runBackendStepOutput.addValue(StreamedETLWithFrontendProcess.FIELD_PROCESS_SUMMARY, processSummary);
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   private void execute(QTableMetaData table, QFieldMetaData field, Integer daysBack, Integer maxPageSize) throws QException
+   {
+      asyncJobCallback.updateStatus("Counting records");
+      Instant maxDate = Instant.now().minusSeconds(daysBack * 60 * 60 * 24);
+      Instant minDate = findMinDateInTable(table, field);
+
+      processDateRange(table, field, maxPageSize, minDate, maxDate);
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   private void processDateRange(QTableMetaData table, QFieldMetaData field, Integer maxPageSize, Instant minDate, Instant maxDate) throws QException
+   {
+      partitionsLine.incrementCount();
+
+      LOG.info("Counting", logPair("table", table.getName()), logPair("field", field.getName()), logPair("minDate", minDate), logPair("maxDate", maxDate));
+      Integer count = count(table, field, minDate, maxDate);
+      LOG.info("Count", logPair("count", count), logPair("table", table.getName()), logPair("field", field.getName()), logPair("minDate", minDate), logPair("maxDate", maxDate));
+
+      if(this.total == null)
+      {
+         this.total = count;
+      }
+
+      if(count == 0)
+      {
+         LOG.info("0 rows in this partition - nothing to delete", logPair("count", count), logPair("table", table.getName()), logPair("field", field.getName()), logPair("minDate", minDate), logPair("maxDate", maxDate));
+      }
+      else if(count <= maxPageSize)
+      {
+         asyncJobCallback.updateStatus("Deleting records", deletedSoFar, total);
+         LOG.info("Deleting", logPair("count", count), logPair("table", table.getName()), logPair("field", field.getName()), logPair("minDate", minDate), logPair("maxDate", maxDate));
+         delete(table, field, minDate, maxDate);
+         this.deletedSoFar += count;
+      }
+      else
+      {
+         if(deletedSoFar == 0)
+         {
+            asyncJobCallback.updateStatus("Partitioning table", deletedSoFar, total);
+         }
+
+         LOG.info("Too many rows", logPair("count", count), logPair("maxPageSize", maxPageSize), logPair("table", table.getName()), logPair("field", field.getName()), logPair("minDate", minDate), logPair("maxDate", maxDate));
+         partition(table, field, minDate, maxDate, count, maxPageSize);
+      }
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   private void partition(QTableMetaData table, QFieldMetaData field, Instant minDate, Instant maxDate, Integer count, Integer maxPageSize) throws QException
+   {
+      int  noOfPartitions    = (int) Math.ceil((float) count / (float) maxPageSize);
+      long milliDiff         = maxDate.toEpochMilli() - minDate.toEpochMilli();
+      long milliPerPartition = milliDiff / noOfPartitions;
+
+      if(milliPerPartition < 1000)
+      {
+         throw (new QUserFacingException("To find a maxPageSize under " + String.format("%,d", maxPageSize) + ", the partition size would become smaller than 1 second (between " + minDate + " and " + maxDate + " there are " + String.format("%,d", count) + " rows) - you must use a larger maxPageSize to continue."));
+      }
+
+      LOG.info("Partitioning", logPair("count", count), logPair("noOfPartitions", noOfPartitions), logPair("milliDiff", milliDiff), logPair("milliPerPartition", milliPerPartition), logPair("table", table.getName()), logPair("field", field.getName()), logPair("minDate", minDate), logPair("maxDate", maxDate));
+      for(int i = 0; i < noOfPartitions; i++)
+      {
+         maxDate = minDate.plusMillis(milliPerPartition);
+         processDateRange(table, field, maxPageSize, minDate, maxDate);
+         minDate = maxDate;
+      }
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   private void delete(QTableMetaData table, QFieldMetaData field, Instant minDate, Instant maxDate) throws QException
+   {
+      DeleteOutput deleteOutput = new DeleteAction().execute(new DeleteInput(table.getName())
+         .withQueryFilter(new QQueryFilter(new QFilterCriteria(field.getName(), QCriteriaOperator.BETWEEN, minDate, maxDate))));
+
+      deletedLine.incrementCount(deleteOutput.getDeletedRecordCount());
+
+      if(CollectionUtils.nullSafeHasContents(deleteOutput.getRecordsWithErrors()))
+      {
+         warningLine.incrementCount(deleteOutput.getRecordsWithWarnings().size());
+      }
+
+      if(CollectionUtils.nullSafeHasContents(deleteOutput.getRecordsWithErrors()))
+      {
+         errorLine.incrementCount(deleteOutput.getRecordsWithErrors().size());
+      }
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   private Integer count(QTableMetaData table, QFieldMetaData field, Instant minDate, Instant maxDate) throws QException
+   {
+      Aggregate      countDateField = new Aggregate(field.getName(), AggregateOperator.COUNT).withFieldType(QFieldType.INTEGER);
+      AggregateInput aggregateInput = new AggregateInput();
+      aggregateInput.setTableName(table.getName());
+      aggregateInput.withFilter(new QQueryFilter(new QFilterCriteria(field.getName(), QCriteriaOperator.BETWEEN, minDate, maxDate)));
+      aggregateInput.withAggregate(countDateField);
+
+      AggregateOutput       aggregateOutput = new AggregateAction().execute(aggregateInput);
+      List<AggregateResult> results         = aggregateOutput.getResults();
+      if(CollectionUtils.nullSafeIsEmpty(results))
+      {
+         throw (new QUserFacingException("Could not count rows table (null or empty aggregate result)."));
+      }
+
+      return (ValueUtils.getValueAsInteger(results.get(0).getAggregateValue(countDateField)));
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   private Instant findMinDateInTable(QTableMetaData table, QFieldMetaData field) throws QException
+   {
+      Aggregate      minDate        = new Aggregate(field.getName(), AggregateOperator.MIN);
+      AggregateInput aggregateInput = new AggregateInput();
+      aggregateInput.setTableName(table.getName());
+      aggregateInput.withAggregate(minDate);
+
+      AggregateOutput       aggregateOutput = new AggregateAction().execute(aggregateInput);
+      List<AggregateResult> results         = aggregateOutput.getResults();
+      if(CollectionUtils.nullSafeIsEmpty(results))
+      {
+         throw (new QUserFacingException("Could not find min date value in table (null or empty aggregate result)."));
+      }
+
+      return (ValueUtils.getValueAsInstant(results.get(0).getAggregateValue(minDate)));
+   }
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/garbagecollector/GenericGarbageCollectorProcessMetaDataProducer.java

@@ -0,0 +1,88 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2023.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.processes.implementations.garbagecollector;
+
+
+import java.util.List;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.metadata.MetaDataProducer;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
+import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon;
+import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.processes.QComponentType;
+import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendComponentMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendStepMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
+
+
+/*******************************************************************************
+ ** Generic process that can perform garbage collection on any table (at least,
+ ** any table with a date or date-time field).
+ **
+ ** When running, this process prompts for:
+ ** - table name
+ ** - field name (e.g., the date/date-time field on that table)
+ ** - daysBack - any records older than that many days ago will be deleted.
+ ** - maxPageSize - to avoid running "1 huge query", if there are more than
+ ** this number of records between the min-date in the table and the max-date
+ ** (based on daysBack), then the time range is partitioned recursively until
+ ** pages smaller than this parameter are found.  The partitioning attempts to
+ ** be smart (e.g., not just ÷ 2), by doing count / maxPageSize.
+ *******************************************************************************/
+public class GenericGarbageCollectorProcessMetaDataProducer extends MetaDataProducer<QProcessMetaData>
+{
+   public static final String NAME = "GenericGarbageCollector";
+
+
+
+   /*******************************************************************************
+    ** See class header for param descriptions.
+    *******************************************************************************/
+   @Override
+   public QProcessMetaData produce(QInstance qInstance) throws QException
+   {
+      QProcessMetaData processMetaData = new QProcessMetaData()
+         .withName(NAME)
+         .withIcon(new QIcon().withName("auto_delete"))
+         .withStepList(List.of(
+            new QFrontendStepMetaData()
+               .withName("input")
+               .withComponent(new QFrontendComponentMetaData().withType(QComponentType.EDIT_FORM))
+               .withFormField(new QFieldMetaData("table", QFieldType.STRING))
+               .withFormField(new QFieldMetaData("field", QFieldType.STRING))
+               .withFormField(new QFieldMetaData("daysBack", QFieldType.INTEGER).withDefaultValue(90))
+               .withFormField(new QFieldMetaData("maxPageSize", QFieldType.INTEGER).withDefaultValue(100000)),
+            new QBackendStepMetaData()
+               .withName("execute")
+               .withCode(new QCodeReference(GenericGarbageCollectorExecuteStep.class)),
+            new QFrontendStepMetaData()
+               .withName("result")
+               .withComponent(new QFrontendComponentMetaData().withType(QComponentType.PROCESS_SUMMARY_RESULTS))
+         ));
+
+      return (processMetaData);
+   }
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/reports/BasicRunReportProcess.java

@@ -46,8 +46,7 @@ public class BasicRunReportProcess
    public static final String STEP_NAME_EXECUTE = "execute";
    public static final String STEP_NAME_ACCESS  = "accessReport";
 
-   public static final String FIELD_REPORT_NAME   = "reportName";
-   public static final String FIELD_REPORT_FORMAT = "reportFormat";
+   public static final String FIELD_REPORT_NAME = "reportName";
 
 
 

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/reports/ExecuteReportStep.java

@@ -33,7 +33,6 @@ import com.kingsrook.qqq.backend.core.actions.processes.BackendStep;
 import com.kingsrook.qqq.backend.core.actions.reporting.GenerateReportAction;
 import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
-import com.kingsrook.qqq.backend.core.exceptions.QUserFacingException;
 import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
 import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
 import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportDestination;
@@ -51,7 +50,6 @@ import com.kingsrook.qqq.backend.core.utils.StringUtils;
 public class ExecuteReportStep implements BackendStep
 {
 
-
    /*******************************************************************************
     **
     *******************************************************************************/
@@ -60,10 +58,9 @@ public class ExecuteReportStep implements BackendStep
    {
       try
       {
-         ReportFormat    reportFormat = getReportFormat(runBackendStepInput);
-         String          reportName   = runBackendStepInput.getValueString("reportName");
-         QReportMetaData report       = QContext.getQInstance().getReport(reportName);
-         File            tmpFile      = File.createTempFile(reportName, "." + reportFormat.getExtension());
+         String          reportName = runBackendStepInput.getValueString("reportName");
+         QReportMetaData report     = QContext.getQInstance().getReport(reportName);
+         File            tmpFile    = File.createTempFile(reportName, ".xlsx", new File("/tmp/"));
 
          runBackendStepInput.getAsyncJobCallback().updateStatus("Generating Report");
 
@@ -72,7 +69,7 @@ public class ExecuteReportStep implements BackendStep
             ReportInput reportInput = new ReportInput();
             reportInput.setReportName(reportName);
             reportInput.setReportDestination(new ReportDestination()
-               .withReportFormat(reportFormat)
+               .withReportFormat(ReportFormat.XLSX) // todo - variable
                .withReportOutputStream(reportOutputStream));
 
             Map<String, Serializable> values = runBackendStepInput.getValues();
@@ -82,7 +79,7 @@ public class ExecuteReportStep implements BackendStep
 
             String downloadFileBaseName = getDownloadFileBaseName(runBackendStepInput, report);
 
-            runBackendStepOutput.addValue("downloadFileName", downloadFileBaseName + "." + reportFormat.getExtension());
+            runBackendStepOutput.addValue("downloadFileName", downloadFileBaseName + ".xlsx");
             runBackendStepOutput.addValue("serverFilePath", tmpFile.getCanonicalPath());
          }
       }
@@ -94,22 +91,6 @@ public class ExecuteReportStep implements BackendStep
 
 
 
-   /***************************************************************************
-    **
-    ***************************************************************************/
-   private ReportFormat getReportFormat(RunBackendStepInput runBackendStepInput) throws QUserFacingException
-   {
-      String reportFormatInput = runBackendStepInput.getValueString(BasicRunReportProcess.FIELD_REPORT_FORMAT);
-      if(StringUtils.hasContent(reportFormatInput))
-      {
-         return (ReportFormat.fromString(reportFormatInput));
-      }
-
-      return (ReportFormat.XLSX);
-   }
-
-
-
    /*******************************************************************************
     **
     *******************************************************************************/

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/tables/QueryActionTest.java

@@ -24,8 +24,6 @@ package com.kingsrook.qqq.backend.core.actions.tables;
 
 import java.io.Serializable;
 import java.util.ArrayList;
-import java.util.HashSet;
-import java.util.LinkedHashSet;
 import java.util.List;
 import java.util.Set;
 import com.kingsrook.qqq.backend.core.BaseTest;
@@ -40,7 +38,6 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
-import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryJoin;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
 import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
@@ -57,7 +54,6 @@ import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
 import com.kingsrook.qqq.backend.core.utils.TestUtils;
 import org.junit.jupiter.api.Test;
 import static org.assertj.core.api.Assertions.assertThat;
-import static org.assertj.core.api.Assertions.assertThatThrownBy;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertFalse;
 import static org.junit.jupiter.api.Assertions.assertNotNull;
@@ -545,89 +541,4 @@ class QueryActionTest extends BaseTest
       assertEquals(1, QueryAction.execute(TestUtils.TABLE_NAME_SHAPE, new QQueryFilter(new QFilterCriteria("name", QCriteriaOperator.EQUALS, "SqUaRe"))).size());
    }
 
-
-
-   /*******************************************************************************
-    **
-    *******************************************************************************/
-   @Test
-   void testValidateFieldNamesToInclude() throws QException
-   {
-      /////////////////////////////
-      // cases that do not throw //
-      /////////////////////////////
-      QueryAction.validateFieldNamesToInclude(new QueryInput(TestUtils.TABLE_NAME_PERSON)
-         .withFieldNamesToInclude(null));
-
-      QueryAction.validateFieldNamesToInclude(new QueryInput(TestUtils.TABLE_NAME_PERSON)
-         .withFieldNamesToInclude(Set.of("id")));
-
-      QueryAction.validateFieldNamesToInclude(new QueryInput(TestUtils.TABLE_NAME_PERSON)
-         .withFieldNamesToInclude(Set.of("id", "firstName", "lastName", "birthDate", "modifyDate", "createDate")));
-
-      QueryAction.validateFieldNamesToInclude(new QueryInput(TestUtils.TABLE_NAME_PERSON)
-         .withQueryJoin(new QueryJoin(TestUtils.TABLE_NAME_SHAPE).withSelect(true))
-         .withFieldNamesToInclude(Set.of("id", "firstName", "lastName", "birthDate", "modifyDate", "createDate")));
-
-      QueryAction.validateFieldNamesToInclude(new QueryInput(TestUtils.TABLE_NAME_PERSON)
-         .withQueryJoin(new QueryJoin(TestUtils.TABLE_NAME_SHAPE).withSelect(true))
-         .withFieldNamesToInclude(Set.of("id", "firstName", TestUtils.TABLE_NAME_SHAPE + ".id", TestUtils.TABLE_NAME_SHAPE + ".noOfSides")));
-
-      QueryAction.validateFieldNamesToInclude(new QueryInput(TestUtils.TABLE_NAME_PERSON)
-         .withQueryJoin(new QueryJoin(TestUtils.TABLE_NAME_SHAPE).withSelect(true).withAlias("s"))
-         .withFieldNamesToInclude(Set.of("id", "s.id", "s.noOfSides")));
-
-      //////////////////////////
-      // cases that do throw! //
-      //////////////////////////
-      assertThatThrownBy(() -> QueryAction.validateFieldNamesToInclude(new QueryInput(TestUtils.TABLE_NAME_PERSON)
-         .withFieldNamesToInclude(new HashSet<>())))
-         .hasMessageContaining("empty set of fieldNamesToInclude");
-
-      assertThatThrownBy(() -> QueryAction.validateFieldNamesToInclude(new QueryInput(TestUtils.TABLE_NAME_PERSON)
-         .withFieldNamesToInclude(Set.of("notAField"))))
-         .hasMessageContaining("1 unrecognized field name: notAField");
-
-      assertThatThrownBy(() -> QueryAction.validateFieldNamesToInclude(new QueryInput(TestUtils.TABLE_NAME_PERSON)
-         .withFieldNamesToInclude(new LinkedHashSet<>(List.of("notAField", "alsoNot")))))
-         .hasMessageContaining("2 unrecognized field names: notAField,alsoNot");
-
-      assertThatThrownBy(() -> QueryAction.validateFieldNamesToInclude(new QueryInput(TestUtils.TABLE_NAME_PERSON)
-         .withFieldNamesToInclude(new LinkedHashSet<>(List.of("notAField", "alsoNot", "join.not")))))
-         .hasMessageContaining("3 unrecognized field names: notAField,alsoNot,join.not");
-
-      assertThatThrownBy(() -> QueryAction.validateFieldNamesToInclude(new QueryInput(TestUtils.TABLE_NAME_PERSON)
-         .withQueryJoin(new QueryJoin(TestUtils.TABLE_NAME_SHAPE)) // oops, didn't select it!
-         .withFieldNamesToInclude(new LinkedHashSet<>(List.of("id", "firstName", TestUtils.TABLE_NAME_SHAPE + ".id", TestUtils.TABLE_NAME_SHAPE + ".noOfSides")))))
-         .hasMessageContaining("2 unrecognized field names: shape.id,shape.noOfSides");
-
-      assertThatThrownBy(() -> QueryAction.validateFieldNamesToInclude(new QueryInput(TestUtils.TABLE_NAME_PERSON)
-         .withQueryJoin(new QueryJoin(TestUtils.TABLE_NAME_SHAPE).withSelect(true))
-         .withFieldNamesToInclude(Set.of(TestUtils.TABLE_NAME_SHAPE + ".noField"))))
-         .hasMessageContaining("1 unrecognized field name: shape.noField");
-
-      assertThatThrownBy(() -> QueryAction.validateFieldNamesToInclude(new QueryInput(TestUtils.TABLE_NAME_PERSON)
-         .withQueryJoin(new QueryJoin(TestUtils.TABLE_NAME_SHAPE).withSelect(true).withAlias("s")) // oops, not using alias
-         .withFieldNamesToInclude(new LinkedHashSet<>(List.of("id", "firstName", TestUtils.TABLE_NAME_SHAPE + ".id", "s.noOfSides")))))
-         .hasMessageContaining("1 unrecognized field name: shape.id");
-
-      assertThatThrownBy(() -> QueryAction.validateFieldNamesToInclude(new QueryInput(TestUtils.TABLE_NAME_PERSON)
-         .withQueryJoin(new QueryJoin(TestUtils.TABLE_NAME_SHAPE).withSelect(true))
-         .withFieldNamesToInclude(new LinkedHashSet<>(List.of("id", "firstName", TestUtils.TABLE_NAME_SHAPE + ".id", "noOfSides")))))
-         .hasMessageContaining("1 unrecognized field name: noOfSides");
-
-      assertThatThrownBy(() -> QueryAction.validateFieldNamesToInclude(new QueryInput(TestUtils.TABLE_NAME_PERSON)
-         .withQueryJoin(new QueryJoin("noJoinTable").withSelect(true))
-         .withFieldNamesToInclude(Set.of("noJoinTable.id"))))
-         .hasMessageContaining("1 unrecognized field name: noJoinTable.id");
-
-      assertThatThrownBy(() -> QueryAction.validateFieldNamesToInclude(new QueryInput(TestUtils.TABLE_NAME_PERSON)
-         .withFieldNamesToInclude(Set.of("noJoin.id"))))
-         .hasMessageContaining("1 unrecognized field name: noJoin.id");
-
-      assertThatThrownBy(() -> QueryAction.validateFieldNamesToInclude(new QueryInput(TestUtils.TABLE_NAME_PERSON)
-         .withFieldNamesToInclude(Set.of("noDb.noJoin.id"))))
-         .hasMessageContaining("1 unrecognized field name: noDb.noJoin.id");
-   }
-
 }

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidatorTest.java

@@ -22,7 +22,6 @@
 package com.kingsrook.qqq.backend.core.instances;
 
 
-import java.io.Serializable;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashMap;
@@ -39,8 +38,6 @@ import com.kingsrook.qqq.backend.core.actions.dashboard.PersonsByCreateDateBarCh
 import com.kingsrook.qqq.backend.core.actions.dashboard.widgets.AbstractWidgetRenderer;
 import com.kingsrook.qqq.backend.core.actions.dashboard.widgets.ParentWidgetRenderer;
 import com.kingsrook.qqq.backend.core.actions.processes.CancelProcessActionTest;
-import com.kingsrook.qqq.backend.core.actions.reporting.RecordPipe;
-import com.kingsrook.qqq.backend.core.actions.reporting.customizers.ReportCustomRecordSourceInterface;
 import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.exceptions.QInstanceValidationException;
@@ -48,7 +45,6 @@ import com.kingsrook.qqq.backend.core.instances.validation.plugins.AlwaysFailsPr
 import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLineInterface;
 import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
 import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
-import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy;
@@ -1716,7 +1712,7 @@ public class QInstanceValidatorTest extends BaseTest
    @Test
    void testReportDataSourceStaticDataSupplier()
    {
-      assertValidationFailureReasons((qInstance) -> qInstance.getReport(TestUtils.REPORT_NAME_SHAPES_PERSON).getDataSources().get(0).withStaticDataSupplier(new QCodeReference(TestReportStaticDataSupplier.class)),
+      assertValidationFailureReasons((qInstance) -> qInstance.getReport(TestUtils.REPORT_NAME_SHAPES_PERSON).getDataSources().get(0).withStaticDataSupplier(new QCodeReference()),
          "has both a sourceTable and a staticDataSupplier");
 
       assertValidationFailureReasons((qInstance) ->
@@ -1724,43 +1720,16 @@ public class QInstanceValidatorTest extends BaseTest
          QReportDataSource dataSource = qInstance.getReport(TestUtils.REPORT_NAME_SHAPES_PERSON).getDataSources().get(0);
          dataSource.setSourceTable(null);
          dataSource.setStaticDataSupplier(new QCodeReference(null, QCodeType.JAVA));
-      }, "missing a code reference name");
+      },
+         "missing a code reference name");
 
       assertValidationFailureReasons((qInstance) ->
       {
          QReportDataSource dataSource = qInstance.getReport(TestUtils.REPORT_NAME_SHAPES_PERSON).getDataSources().get(0);
          dataSource.setSourceTable(null);
          dataSource.setStaticDataSupplier(new QCodeReference(ArrayList.class));
-      }, "is not of the expected type");
-   }
-
-
-
-   /*******************************************************************************
-    **
-    *******************************************************************************/
-   @Test
-   void testReportDataSourceCustomRecordSource()
-   {
-      assertValidationFailureReasons((qInstance) -> qInstance.getReport(TestUtils.REPORT_NAME_SHAPES_PERSON).getDataSources().get(0)
-            .withSourceTable(null)
-            .withStaticDataSupplier(new QCodeReference(TestReportStaticDataSupplier.class))
-            .withCustomRecordSource(new QCodeReference(TestReportCustomRecordSource.class)),
-         "has both a staticDataSupplier and a customRecordSource");
-
-      assertValidationFailureReasons((qInstance) ->
-      {
-         QReportDataSource dataSource = qInstance.getReport(TestUtils.REPORT_NAME_SHAPES_PERSON).getDataSources().get(0);
-         dataSource.setSourceTable(null);
-         dataSource.setCustomRecordSource(new QCodeReference(null, QCodeType.JAVA));
-      }, "missing a code reference name");
-
-      assertValidationFailureReasons((qInstance) ->
-      {
-         QReportDataSource dataSource = qInstance.getReport(TestUtils.REPORT_NAME_SHAPES_PERSON).getDataSources().get(0);
-         dataSource.setSourceTable(null);
-         dataSource.setCustomRecordSource(new QCodeReference(ArrayList.class));
-      }, "is not of the expected type");
+      },
+         "is not of the expected type");
    }
 
 
@@ -2402,38 +2371,5 @@ public class QInstanceValidatorTest extends BaseTest
     *******************************************************************************/
    public static class ValidAuthCustomizer implements QAuthenticationModuleCustomizerInterface {}
 
-
-   /***************************************************************************
-    **
-    ***************************************************************************/
-   public static class TestReportStaticDataSupplier implements Supplier<List<List<Serializable>>>
-   {
-
-      /***************************************************************************
-       **
-       ***************************************************************************/
-      @Override
-      public List<List<Serializable>> get()
-      {
-         return List.of();
-      }
-   }
-
-
-   /***************************************************************************
-    **
-    ***************************************************************************/
-   public static class TestReportCustomRecordSource implements ReportCustomRecordSourceInterface
-   {
-
-      /***************************************************************************
-       **
-       ***************************************************************************/
-      @Override
-      public void execute(ReportInput reportInput, QReportDataSource reportDataSource, RecordPipe recordPipe) throws QException
-      {
-
-      }
-   }
 }
 

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/garbagecollector/GenericGarbageCollectorExecuteStepTest.java

@@ -0,0 +1,243 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.processes.implementations.garbagecollector;
+
+
+import java.time.Instant;
+import java.time.temporal.ChronoUnit;
+import java.util.List;
+import java.util.Set;
+import java.util.stream.Collectors;
+import com.kingsrook.qqq.backend.core.BaseTest;
+import com.kingsrook.qqq.backend.core.actions.processes.RunProcessAction;
+import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
+import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
+import com.kingsrook.qqq.backend.core.context.QContext;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.exceptions.QUserFacingException;
+import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput;
+import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessOutput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryRecordStore;
+import com.kingsrook.qqq.backend.core.utils.TestUtils;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Disabled;
+import org.junit.jupiter.api.Test;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+
+/*******************************************************************************
+ ** Unit test for GenericGarbageCollectorExecuteStep 
+ *******************************************************************************/
+class GenericGarbageCollectorExecuteStepTest extends BaseTest
+{
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @BeforeEach
+   @AfterEach
+   void beforeAndAfterEach()
+   {
+      MemoryRecordStore.getInstance().reset();
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testErrors() throws Exception
+   {
+      QContext.getQInstance().addProcess(new GenericGarbageCollectorProcessMetaDataProducer().produce(QContext.getQInstance()));
+
+      RunProcessInput input = new RunProcessInput();
+      input.setProcessName(GenericGarbageCollectorProcessMetaDataProducer.NAME);
+      input.setFrontendStepBehavior(RunProcessInput.FrontendStepBehavior.SKIP);
+
+      assertThatThrownBy(() -> runAndThrow(input)).isInstanceOf(QUserFacingException.class).hasMessageContaining("Unrecognized table: null");
+
+      input.addValue("table", "notATable");
+      assertThatThrownBy(() -> runAndThrow(input)).isInstanceOf(QUserFacingException.class).hasMessageContaining("Unrecognized table: notATable");
+
+      input.addValue("table", TestUtils.TABLE_NAME_PERSON_MEMORY);
+      assertThatThrownBy(() -> runAndThrow(input)).isInstanceOf(QUserFacingException.class).hasMessageContaining("Unrecognized field: null");
+
+      input.addValue("field", "notAField");
+      assertThatThrownBy(() -> runAndThrow(input)).isInstanceOf(QUserFacingException.class).hasMessageContaining("Unrecognized field: notAField");
+
+      input.addValue("field", "firstName");
+      assertThatThrownBy(() -> runAndThrow(input)).isInstanceOf(QUserFacingException.class).hasMessageContaining("not a date");
+
+      input.addValue("field", "timestamp");
+      assertThatThrownBy(() -> runAndThrow(input)).isInstanceOf(QUserFacingException.class).hasMessageContaining("daysBack: null");
+
+      input.addValue("daysBack", "-1");
+      assertThatThrownBy(() -> runAndThrow(input)).isInstanceOf(QUserFacingException.class).hasMessageContaining("daysBack: -1");
+
+      input.addValue("daysBack", "1");
+      assertThatThrownBy(() -> runAndThrow(input)).isInstanceOf(QUserFacingException.class).hasMessageContaining("maxPageSize: null");
+
+      input.addValue("maxPageSize", "-1");
+      assertThatThrownBy(() -> runAndThrow(input)).isInstanceOf(QUserFacingException.class).hasMessageContaining("maxPageSize: -1");
+
+      input.addValue("maxPageSize", "1");
+      assertThatThrownBy(() -> runAndThrow(input)).isInstanceOf(QUserFacingException.class).hasMessageContaining("Could not find min date value in table");
+   }
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   private void runAndThrow(RunProcessInput input) throws Exception
+   {
+      input.setStartAfterStep(null);
+      input.setProcessUUID(null);
+      RunProcessOutput runProcessOutput = new RunProcessAction().execute(input);
+      if(runProcessOutput.getException().isPresent())
+      {
+         throw (runProcessOutput.getException().get());
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void test30days() throws QException
+   {
+      insertAndRunGC(30);
+      QueryOutput queryOutput = new QueryAction().execute(new QueryInput(TestUtils.TABLE_NAME_PERSON_MEMORY).withFilter(new QQueryFilter()));
+      assertEquals(2, queryOutput.getRecords().size());
+      assertEquals(Set.of(4, 5), queryOutput.getRecords().stream().map(r -> r.getValueInteger("id")).collect(Collectors.toSet()));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   @Disabled("memory aggregator is failing to return an aggregate when no rows found, which is throwing an error...")
+   void test100days() throws QException
+   {
+      insertAndRunGC(100);
+      QueryOutput queryOutput = new QueryAction().execute(new QueryInput(TestUtils.TABLE_NAME_PERSON_MEMORY).withFilter(new QQueryFilter()));
+      assertEquals(5, queryOutput.getRecords().size());
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void test10days() throws QException
+   {
+      insertAndRunGC(10);
+      QueryOutput queryOutput = new QueryAction().execute(new QueryInput(TestUtils.TABLE_NAME_PERSON_MEMORY).withFilter(new QQueryFilter()));
+      assertEquals(1, queryOutput.getRecords().size());
+      assertEquals(Set.of(5), queryOutput.getRecords().stream().map(r -> r.getValueInteger("id")).collect(Collectors.toSet()));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void test1day() throws QException
+   {
+      insertAndRunGC(1);
+      QueryOutput queryOutput = new QueryAction().execute(new QueryInput(TestUtils.TABLE_NAME_PERSON_MEMORY).withFilter(new QQueryFilter()));
+      assertEquals(0, queryOutput.getRecords().size());
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void test1dayPartitioned() throws QException
+   {
+      insertAndRunGC(1, 2);
+      QueryOutput queryOutput = new QueryAction().execute(new QueryInput(TestUtils.TABLE_NAME_PERSON_MEMORY).withFilter(new QQueryFilter()));
+      assertEquals(0, queryOutput.getRecords().size());
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   private void insertAndRunGC(Integer daysBack) throws QException
+   {
+      insertAndRunGC(daysBack, 1000);
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   private void insertAndRunGC(Integer daysBack, Integer maxPageSize) throws QException
+   {
+      new InsertAction().execute(new InsertInput(TestUtils.TABLE_NAME_PERSON_MEMORY).withRecords(getPersonRecords()));
+      QContext.getQInstance().addProcess(new GenericGarbageCollectorProcessMetaDataProducer().produce(QContext.getQInstance()));
+
+      RunProcessInput input = new RunProcessInput();
+      input.setProcessName(GenericGarbageCollectorProcessMetaDataProducer.NAME);
+      input.addValue("table", TestUtils.TABLE_NAME_PERSON_MEMORY);
+      input.addValue("field", "timestamp");
+      input.addValue("daysBack", daysBack);
+      input.addValue("maxPageSize", maxPageSize);
+      input.setFrontendStepBehavior(RunProcessInput.FrontendStepBehavior.SKIP);
+      new RunProcessAction().execute(input);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private static List<QRecord> getPersonRecords()
+   {
+      List<QRecord> records = List.of(
+         new QRecord().withValue("id", 1).withValue("timestamp", Instant.now().minus(90, ChronoUnit.DAYS)),
+         new QRecord().withValue("id", 2).withValue("timestamp", Instant.now().minus(31, ChronoUnit.DAYS)),
+         new QRecord().withValue("id", 3).withValue("timestamp", Instant.now().minus(30, ChronoUnit.DAYS).minus(5, ChronoUnit.MINUTES)),
+         new QRecord().withValue("id", 4).withValue("timestamp", Instant.now().minus(29, ChronoUnit.DAYS).minus(23, ChronoUnit.HOURS)),
+         new QRecord().withValue("id", 5).withValue("timestamp", Instant.now().minus(5, ChronoUnit.DAYS)));
+      return records;
+   }
+
+}

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/reports/BasicRunReportProcessTest.java

@@ -30,7 +30,6 @@ import com.kingsrook.qqq.backend.core.actions.reporting.GenerateReportActionTest
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput;
 import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessOutput;
-import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportFormat;
 import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
 import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportMetaData;
@@ -44,7 +43,6 @@ import static org.assertj.core.api.Assertions.assertThat;
  *******************************************************************************/
 class BasicRunReportProcessTest extends BaseTest
 {
-
    /*******************************************************************************
     **
     *******************************************************************************/
@@ -76,18 +74,6 @@ class BasicRunReportProcessTest extends BaseTest
       runProcessOutput = new RunProcessAction().execute(runProcessInput);
       assertThat(runProcessOutput.getProcessState().getNextStepName()).isPresent().get().isEqualTo(BasicRunReportProcess.STEP_NAME_ACCESS);
       assertThat(runProcessOutput.getValues()).containsKeys("downloadFileName", "serverFilePath");
-
-      ///////////////////////////////////
-      // assert we get xlsx by default //
-      ///////////////////////////////////
-      assertThat(runProcessOutput.getValueString("downloadFileName")).endsWith(".xlsx");
-
-      /////////////////////////////////////////////////////
-      // re-run, requesting CSV, then assert we get that //
-      /////////////////////////////////////////////////////
-      runProcessInput.addValue(BasicRunReportProcess.FIELD_REPORT_FORMAT, ReportFormat.CSV.name());
-      runProcessOutput = new RunProcessAction().execute(runProcessInput);
-      assertThat(runProcessOutput.getValueString("downloadFileName")).endsWith(".csv");
    }
 
 }

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/savedviews/SavedViewProcessTests.java

@@ -192,6 +192,7 @@ class SavedViewProcessTests extends BaseTest
     **
     *******************************************************************************/
    @Test
+   @SuppressWarnings("unchecked")
    void testNotFoundThrowsProperly() throws QException
    {
       QInstance qInstance = QContext.getQInstance();
@@ -244,4 +245,4 @@ class SavedViewProcessTests extends BaseTest
       }
    }
 
-}
+}

qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/actions/BaseAPIActionUtil.java

@@ -35,7 +35,6 @@ import java.util.Base64;
 import java.util.List;
 import java.util.Map;
 import java.util.concurrent.TimeUnit;
-import java.util.function.Supplier;
 import com.kingsrook.qqq.backend.core.actions.tables.GetAction;
 import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
 import com.kingsrook.qqq.backend.core.context.QContext;
@@ -65,7 +64,6 @@ import com.kingsrook.qqq.backend.core.model.session.QSession;
 import com.kingsrook.qqq.backend.core.model.statusmessages.SystemErrorStatusMessage;
 import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
 import com.kingsrook.qqq.backend.core.utils.JsonUtils;
-import com.kingsrook.qqq.backend.core.utils.Pair;
 import com.kingsrook.qqq.backend.core.utils.SleepUtils;
 import com.kingsrook.qqq.backend.core.utils.StringUtils;
 import com.kingsrook.qqq.backend.core.utils.ValueUtils;
@@ -89,7 +87,6 @@ import org.apache.http.client.methods.CloseableHttpResponse;
 import org.apache.http.client.methods.HttpDelete;
 import org.apache.http.client.methods.HttpEntityEnclosingRequestBase;
 import org.apache.http.client.methods.HttpGet;
-import org.apache.http.client.methods.HttpPatch;
 import org.apache.http.client.methods.HttpPost;
 import org.apache.http.client.methods.HttpPut;
 import org.apache.http.client.methods.HttpRequestBase;
@@ -121,36 +118,10 @@ public class BaseAPIActionUtil
 
 
    /***************************************************************************
-    ** enum of which HTTP Method the backend uses for Updates.
+    **
     ***************************************************************************/
    public enum UpdateHttpMethod
-   {
-      PUT(HttpPut::new),
-      POST(HttpPost::new),
-      PATCH(HttpPatch::new);
-
-      private Supplier<HttpEntityEnclosingRequestBase> httpEntitySupplier;
-
-
-
-      /***************************************************************************
-       **
-       ***************************************************************************/
-      UpdateHttpMethod(Supplier<HttpEntityEnclosingRequestBase> httpEnttySupplier)
-      {
-         this.httpEntitySupplier = httpEnttySupplier;
-      }
-
-
-
-      /***************************************************************************
-       **
-       ***************************************************************************/
-      public HttpEntityEnclosingRequestBase newRequest()
-      {
-         return (this.httpEntitySupplier.get());
-      }
-   }
+   {PUT, POST}
 
 
 
@@ -379,9 +350,7 @@ public class BaseAPIActionUtil
             {
                String                         paramString = buildQueryStringForUpdate(table, recordList);
                String                         url         = buildTableUrl(table) + paramString;
-               HttpEntityEnclosingRequestBase request     = getUpdateMethod().newRequest();
-
-               request.setURI(new URI(url));
+               HttpEntityEnclosingRequestBase request     = getUpdateMethod().equals(UpdateHttpMethod.PUT) ? new HttpPut(url) : new HttpPost(url);
                request.setEntity(recordsToEntity(table, recordList));
 
                QHttpResponse response = makeRequest(table, request);
@@ -719,22 +688,60 @@ public class BaseAPIActionUtil
     *******************************************************************************/
    public void setupAuthorizationInRequest(HttpRequestBase request) throws QException
    {
-      ///////////////////////////////////////////////////////////////////
-      // update the request based on the authorization type being used //
-      ///////////////////////////////////////////////////////////////////
+      ///////////////////////////////////////////////////////////////////////////////////
+      // if backend specifies that it uses variants, look for that data in the session //
+      ///////////////////////////////////////////////////////////////////////////////////
+      if(backendMetaData.getUsesVariants())
+      {
+         QSession session = QContext.getQSession();
+         if(session.getBackendVariants() == null || !session.getBackendVariants().containsKey(backendMetaData.getVariantOptionsTableTypeValue()))
+         {
+            throw (new QException("Could not find Backend Variant information for Backend '" + backendMetaData.getName() + "'"));
+         }
+
+         Serializable variantId = session.getBackendVariants().get(backendMetaData.getVariantOptionsTableTypeValue());
+         GetInput     getInput  = new GetInput();
+         getInput.setShouldMaskPasswords(false);
+         getInput.setTableName(backendMetaData.getVariantOptionsTableName());
+         getInput.setPrimaryKey(variantId);
+         GetOutput getOutput = new GetAction().execute(getInput);
+
+         QRecord record = getOutput.getRecord();
+         if(record == null)
+         {
+            throw (new QException("Could not find Backend Variant in table " + backendMetaData.getVariantOptionsTableName() + " with id '" + variantId + "'"));
+         }
+
+         if(backendMetaData.getAuthorizationType().equals(AuthorizationType.BASIC_AUTH_USERNAME_PASSWORD))
+         {
+            request.setHeader("Authorization", getBasicAuthenticationHeader(record.getValueString(backendMetaData.getVariantOptionsTableUsernameField()), record.getValueString(backendMetaData.getVariantOptionsTablePasswordField())));
+         }
+         else if(backendMetaData.getAuthorizationType().equals(AuthorizationType.API_KEY_HEADER))
+         {
+            request.setHeader("API-Key", record.getValueString(backendMetaData.getVariantOptionsTableApiKeyField()));
+         }
+         else
+         {
+            throw (new IllegalArgumentException("Unexpected variant authorization type specified: " + backendMetaData.getAuthorizationType()));
+         }
+         return;
+      }
+
+      ///////////////////////////////////////////////////////////////////////////////////////////
+      // if not using variants, the authorization data will be in the backend meta data object //
+      ///////////////////////////////////////////////////////////////////////////////////////////
       switch(backendMetaData.getAuthorizationType())
       {
-         case BASIC_AUTH_API_KEY -> request.setHeader("Authorization", getBasicAuthenticationHeader(getApiKey()));
-         case BASIC_AUTH_USERNAME_PASSWORD ->
-         {
-            Pair<String, String> usernameAndPassword = getUsernameAndPassword();
-            request.setHeader("Authorization", getBasicAuthenticationHeader(usernameAndPassword.getA(), usernameAndPassword.getB()));
-         }
-         case API_KEY_HEADER -> request.setHeader("API-Key", getApiKey());
-         case API_TOKEN -> request.setHeader("Authorization", "Token " + getApiKey());
+         case BASIC_AUTH_API_KEY -> request.setHeader("Authorization", getBasicAuthenticationHeader(backendMetaData.getApiKey()));
+         case BASIC_AUTH_USERNAME_PASSWORD -> request.setHeader("Authorization", getBasicAuthenticationHeader(backendMetaData.getUsername(), backendMetaData.getPassword()));
+         case API_KEY_HEADER -> request.setHeader("API-Key", backendMetaData.getApiKey());
+         case API_TOKEN -> request.setHeader("Authorization", "Token " + backendMetaData.getApiKey());
          case OAUTH2 -> request.setHeader("Authorization", "Bearer " + getOAuth2Token());
          case API_KEY_QUERY_PARAM -> addApiKeyQueryParamToRequest(request);
-         case CUSTOM -> handleCustomAuthorization(request);
+         case CUSTOM ->
+         {
+            handleCustomAuthorization(request);
+         }
          default -> throw new IllegalArgumentException("Unexpected authorization type: " + backendMetaData.getAuthorizationType());
       }
    }
@@ -749,7 +756,7 @@ public class BaseAPIActionUtil
       try
       {
          String uri = request.getURI().toString();
-         String pair = backendMetaData.getApiKeyQueryParamName() + "=" + getApiKey();
+         String pair = backendMetaData.getApiKeyQueryParamName() + "=" + backendMetaData.getApiKey();
 
          ///////////////////////////////////////////////////////////////////////////////////
          // avoid re-adding the name=value pair if it's already there (e.g., for a retry) //
@@ -770,106 +777,39 @@ public class BaseAPIActionUtil
 
 
 
-   /***************************************************************************
-    **
-    ***************************************************************************/
-   protected String getApiKey() throws QException
-   {
-      if(backendMetaData.getUsesVariants())
-      {
-         QRecord record = getVariantRecord();
-         return (record.getValueString(backendMetaData.getVariantOptionsTableApiKeyField()));
-      }
-
-      return (backendMetaData.getApiKey());
-   }
-
-
-
-   /***************************************************************************
-    **
-    ***************************************************************************/
-   protected Pair<String, String> getUsernameAndPassword() throws QException
-   {
-      if(backendMetaData.getUsesVariants())
-      {
-         QRecord record = getVariantRecord();
-         return (Pair.of(record.getValueString(backendMetaData.getVariantOptionsTableUsernameField()), record.getValueString(backendMetaData.getVariantOptionsTablePasswordField())));
-      }
-
-      return (Pair.of(backendMetaData.getUsername(), backendMetaData.getPassword()));
-   }
-
-
-
-   /*******************************************************************************
-    ** For backends that use variants, look up the variant record (in theory, based
-    ** on an id in the session's backend variants map, then fetched from the backend's
-    ** variant options table.
-    *******************************************************************************/
-   protected QRecord getVariantRecord() throws QException
-   {
-      Serializable variantId = getVariantId();
-      GetInput     getInput  = new GetInput();
-      getInput.setShouldMaskPasswords(false);
-      getInput.setTableName(backendMetaData.getVariantOptionsTableName());
-      getInput.setPrimaryKey(variantId);
-      GetOutput getOutput = new GetAction().execute(getInput);
-
-      QRecord record = getOutput.getRecord();
-      if(record == null)
-      {
-         throw (new QException("Could not find Backend Variant in table " + backendMetaData.getVariantOptionsTableName() + " with id '" + variantId + "'"));
-      }
-      return record;
-   }
-
-
-
-   /*******************************************************************************
-    ** Get the variant id from the session for the backend.
-    *******************************************************************************/
-   protected Serializable getVariantId() throws QException
-   {
-      QSession session = QContext.getQSession();
-      if(session.getBackendVariants() == null || !session.getBackendVariants().containsKey(backendMetaData.getVariantOptionsTableTypeValue()))
-      {
-         throw (new QException("Could not find Backend Variant information for Backend '" + backendMetaData.getName() + "'"));
-      }
-      Serializable variantId = session.getBackendVariants().get(backendMetaData.getVariantOptionsTableTypeValue());
-      return variantId;
-   }
-
-
-
    /*******************************************************************************
     **
     *******************************************************************************/
-   public String getOAuth2Token() throws OAuthCredentialsException, QException
+   public String getOAuth2Token() throws OAuthCredentialsException
    {
-      //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-      // define the key that will be used in the backend's customValues map, to stash the access token.                   //
-      // for non-variant backends, this is just a constant string.  But for variant-backends, append the variantId to it. //
-      //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-      String accessTokenKey = "accessToken";
-      if(backendMetaData.getUsesVariants())
-      {
-         Serializable variantId = getVariantId();
-         accessTokenKey = accessTokenKey + ":" + variantId;
-      }
-
       ////////////////////////////////////////////////////////////////////////////////////////////////////////////////
       // check for the access token in the backend meta data.  if it's not there, then issue a request for a token. //
       // this is not generally meant to be put in the meta data by the app programmer - rather, we're just using    //
       // it as a "cheap & easy" way to "cache" the token within our process's memory...                             //
       ////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-      String accessToken = ValueUtils.getValueAsString(backendMetaData.getCustomValue(accessTokenKey));
+      String  accessToken            = ValueUtils.getValueAsString(backendMetaData.getCustomValue("accessToken"));
+      Boolean setCredentialsInHeader = BooleanUtils.isTrue(ValueUtils.getValueAsBoolean(backendMetaData.getCustomValue("setCredentialsInHeader")));
 
       if(!StringUtils.hasContent(accessToken))
       {
+         String fullURL  = backendMetaData.getBaseUrl() + "oauth/token";
+         String postBody = "grant_type=client_credentials";
+
+         if(!setCredentialsInHeader)
+         {
+            postBody += "&client_id=" + backendMetaData.getClientId() + "&client_secret=" + backendMetaData.getClientSecret();
+         }
+
          try(CloseableHttpClient client = HttpClients.custom().setConnectionManager(new PoolingHttpClientConnectionManager()).build())
          {
-            HttpRequestBase request = createOAuth2TokenRequest();
+            HttpPost request = new HttpPost(fullURL);
+            request.setEntity(new StringEntity(postBody, getCharsetForEntity()));
+
+            if(setCredentialsInHeader)
+            {
+               request.setHeader("Authorization", getBasicAuthenticationHeader(backendMetaData.getClientId(), backendMetaData.getClientSecret()));
+            }
+            request.setHeader("Content-Type", "application/x-www-form-urlencoded; charset=utf-8");
 
             HttpResponse response     = executeOAuthTokenRequest(client, request);
             int          statusCode   = response.getStatusLine().getStatusCode();
@@ -887,7 +827,7 @@ public class BaseAPIActionUtil
             ///////////////////////////////////////////////////////////////////////////////////////////////////
             // stash the access token in the backendMetaData, from which it will be used for future requests //
             ///////////////////////////////////////////////////////////////////////////////////////////////////
-            backendMetaData.withCustomValue(accessTokenKey, accessToken);
+            backendMetaData.withCustomValue("accessToken", accessToken);
          }
          catch(OAuthCredentialsException oce)
          {
@@ -906,53 +846,6 @@ public class BaseAPIActionUtil
 
 
 
-   /***************************************************************************
-    ** For doing OAuth2 authentication, create a request for a token.
-    ***************************************************************************/
-   protected HttpRequestBase createOAuth2TokenRequest() throws QException
-   {
-      String fullURL  = backendMetaData.getBaseUrl() + "oauth/token";
-      String postBody = "grant_type=client_credentials";
-
-      Pair<String, String> clientIdAndSecret = getClientIdAndSecret();
-      String               clientId          = clientIdAndSecret.getA();
-      String               clientSecret      = clientIdAndSecret.getB();
-
-      Boolean setCredentialsInHeader = BooleanUtils.isTrue(ValueUtils.getValueAsBoolean(backendMetaData.getCustomValue("setCredentialsInHeader")));
-      if(!setCredentialsInHeader)
-      {
-         postBody += "&client_id=" + clientId + "&client_secret=" + clientSecret;
-      }
-
-      HttpPost request = new HttpPost(fullURL);
-      request.setEntity(new StringEntity(postBody, getCharsetForEntity()));
-
-      if(setCredentialsInHeader)
-      {
-         request.setHeader("Authorization", getBasicAuthenticationHeader(clientId, clientSecret));
-      }
-      request.setHeader("Content-Type", "application/x-www-form-urlencoded; charset=utf-8");
-      return request;
-   }
-
-
-
-   /***************************************************************************
-    **
-    ***************************************************************************/
-   protected Pair<String, String> getClientIdAndSecret() throws QException
-   {
-      if(backendMetaData.getUsesVariants())
-      {
-         QRecord record = getVariantRecord();
-         return (Pair.of(record.getValueString(backendMetaData.getVariantOptionsTableClientIdField()), record.getValueString(backendMetaData.getVariantOptionsTableClientSecretField())));
-      }
-
-      return (Pair.of(backendMetaData.getClientId(), backendMetaData.getClientSecret()));
-   }
-
-
-
    /*******************************************************************************
     ** Let a subclass change what charset to use for entities (bodies) being posted/put/etc.
     *******************************************************************************/
@@ -966,18 +859,6 @@ public class BaseAPIActionUtil
    /*******************************************************************************
     ** one-line method, factored out so mock/tests can override
     *******************************************************************************/
-   protected CloseableHttpResponse executeOAuthTokenRequest(CloseableHttpClient client, HttpRequestBase request) throws IOException
-   {
-      return client.execute(request);
-   }
-
-
-
-   /*******************************************************************************
-    ** one-line method, factored out so mock/tests can override
-    ** Deprecated, in favor of more generic overload that takes HttpRequestBase
-    *******************************************************************************/
-   @Deprecated
    protected CloseableHttpResponse executeOAuthTokenRequest(CloseableHttpClient client, HttpPost request) throws IOException
    {
       return client.execute(request);

qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/model/OutboundAPILogHeader.java

@@ -0,0 +1,316 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2023.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.module.api.model;
+
+
+import java.time.Instant;
+import java.util.List;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.data.QAssociation;
+import com.kingsrook.qqq.backend.core.model.data.QField;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.data.QRecordEntity;
+
+
+/*******************************************************************************
+ ** Entity bean for OutboundApiLog table
+ *******************************************************************************/
+public class OutboundAPILogHeader extends QRecordEntity
+{
+   public static final String TABLE_NAME = "outboundApiLogHeader";
+
+   @QField(isEditable = false)
+   private Integer id;
+
+   @QField()
+   private Instant timestamp;
+
+   @QField(possibleValueSourceName = "outboundApiMethod")
+   private String method;
+
+   @QField(possibleValueSourceName = "outboundApiStatusCode")
+   private Integer statusCode;
+
+   @QField(label = "URL")
+   private String url;
+
+   @QAssociation(name = OutboundAPILogRequest.TABLE_NAME)
+   private List<OutboundAPILogRequest> outboundAPILogRequestList;
+
+   @QAssociation(name = OutboundAPILogResponse.TABLE_NAME)
+   private List<OutboundAPILogResponse> outboundAPILogResponseList;
+
+
+
+   /*******************************************************************************
+    ** Constructor
+    **
+    *******************************************************************************/
+   public OutboundAPILogHeader()
+   {
+   }
+
+
+
+   /*******************************************************************************
+    ** Constructor
+    **
+    *******************************************************************************/
+   public OutboundAPILogHeader(QRecord qRecord) throws QException
+   {
+      populateFromQRecord(qRecord);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for id
+    **
+    *******************************************************************************/
+   public Integer getId()
+   {
+      return id;
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for id
+    **
+    *******************************************************************************/
+   public void setId(Integer id)
+   {
+      this.id = id;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for id
+    **
+    *******************************************************************************/
+   public OutboundAPILogHeader withId(Integer id)
+   {
+      this.id = id;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for timestamp
+    **
+    *******************************************************************************/
+   public Instant getTimestamp()
+   {
+      return timestamp;
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for timestamp
+    **
+    *******************************************************************************/
+   public void setTimestamp(Instant timestamp)
+   {
+      this.timestamp = timestamp;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for timestamp
+    **
+    *******************************************************************************/
+   public OutboundAPILogHeader withTimestamp(Instant timestamp)
+   {
+      this.timestamp = timestamp;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for method
+    **
+    *******************************************************************************/
+   public String getMethod()
+   {
+      return method;
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for method
+    **
+    *******************************************************************************/
+   public void setMethod(String method)
+   {
+      this.method = method;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for method
+    **
+    *******************************************************************************/
+   public OutboundAPILogHeader withMethod(String method)
+   {
+      this.method = method;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for statusCode
+    **
+    *******************************************************************************/
+   public Integer getStatusCode()
+   {
+      return statusCode;
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for statusCode
+    **
+    *******************************************************************************/
+   public void setStatusCode(Integer statusCode)
+   {
+      this.statusCode = statusCode;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for statusCode
+    **
+    *******************************************************************************/
+   public OutboundAPILogHeader withStatusCode(Integer statusCode)
+   {
+      this.statusCode = statusCode;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for url
+    **
+    *******************************************************************************/
+   public String getUrl()
+   {
+      return url;
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for url
+    **
+    *******************************************************************************/
+   public void setUrl(String url)
+   {
+      this.url = url;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for url
+    **
+    *******************************************************************************/
+   public OutboundAPILogHeader withUrl(String url)
+   {
+      this.url = url;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for outboundAPILogRequestList
+    *******************************************************************************/
+   public List<OutboundAPILogRequest> getOutboundAPILogRequestList()
+   {
+      return (this.outboundAPILogRequestList);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for outboundAPILogRequestList
+    *******************************************************************************/
+   public void setOutboundAPILogRequestList(List<OutboundAPILogRequest> outboundAPILogRequestList)
+   {
+      this.outboundAPILogRequestList = outboundAPILogRequestList;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for outboundAPILogRequestList
+    *******************************************************************************/
+   public OutboundAPILogHeader withOutboundAPILogRequestList(List<OutboundAPILogRequest> outboundAPILogRequestList)
+   {
+      this.outboundAPILogRequestList = outboundAPILogRequestList;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for outboundAPILogResponseList
+    *******************************************************************************/
+   public List<OutboundAPILogResponse> getOutboundAPILogResponseList()
+   {
+      return (this.outboundAPILogResponseList);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for outboundAPILogResponseList
+    *******************************************************************************/
+   public void setOutboundAPILogResponseList(List<OutboundAPILogResponse> outboundAPILogResponseList)
+   {
+      this.outboundAPILogResponseList = outboundAPILogResponseList;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for outboundAPILogResponseList
+    *******************************************************************************/
+   public OutboundAPILogHeader withOutboundAPILogResponseList(List<OutboundAPILogResponse> outboundAPILogResponseList)
+   {
+      this.outboundAPILogResponseList = outboundAPILogResponseList;
+      return (this);
+   }
+
+}
+

qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/model/OutboundAPILogMetaDataProvider.java

@@ -22,21 +22,33 @@
 package com.kingsrook.qqq.backend.module.api.model;
 
 
+import java.util.ArrayList;
 import java.util.List;
 import java.util.function.Consumer;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.AdornmentType;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.FieldAdornment;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.ValueTooLongBehavior;
+import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinOn;
+import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinType;
+import com.kingsrook.qqq.backend.core.model.metadata.joins.QJoinMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon;
 import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValue;
 import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSource;
 import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSourceType;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.Association;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.Capability;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.ExposedJoin;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QFieldSection;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.Tier;
+import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess;
+import com.kingsrook.qqq.backend.module.api.processes.MigrateOutboundAPILogExtractStep;
+import com.kingsrook.qqq.backend.module.api.processes.MigrateOutboundAPILogLoadStep;
+import com.kingsrook.qqq.backend.module.api.processes.MigrateOutboundAPILogTransformStep;
 
 
 /*******************************************************************************
@@ -50,8 +62,15 @@ public class OutboundAPILogMetaDataProvider
     *******************************************************************************/
    public static void defineAll(QInstance qInstance, String backendName, Consumer<QTableMetaData> backendDetailEnricher) throws QException
    {
-      definePossibleValueSources(qInstance);
-      defineOutboundAPILogTable(qInstance, backendName, backendDetailEnricher);
+      definePossibleValueSources().forEach(pvs ->
+      {
+         if(qInstance.getPossibleValueSource(pvs.getName()) == null)
+         {
+            qInstance.addPossibleValueSource(pvs);
+         }
+      });
+
+      qInstance.addTable(defineOutboundAPILogTable(backendName, backendDetailEnricher));
    }
 
 
@@ -59,9 +78,71 @@ public class OutboundAPILogMetaDataProvider
    /*******************************************************************************
     **
     *******************************************************************************/
-   private static void definePossibleValueSources(QInstance instance)
+   public static void defineNewVersion(QInstance qInstance, String backendName, Consumer<QTableMetaData> backendDetailEnricher) throws QException
    {
-      instance.addPossibleValueSource(new QPossibleValueSource()
+      definePossibleValueSources().forEach(pvs ->
+      {
+         if(qInstance.getPossibleValueSource(pvs.getName()) == null)
+         {
+            qInstance.addPossibleValueSource(pvs);
+         }
+      });
+
+      qInstance.addTable(defineOutboundAPILogHeaderTable(backendName, backendDetailEnricher));
+      qInstance.addPossibleValueSource(defineOutboundAPILogHeaderPossibleValueSource());
+      qInstance.addTable(defineOutboundAPILogRequestTable(backendName, backendDetailEnricher));
+      qInstance.addTable(defineOutboundAPILogResponseTable(backendName, backendDetailEnricher));
+      defineJoins().forEach(join -> qInstance.add(join));
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   public static void defineMigrationProcesses(QInstance qInstance, String sourceTableName)
+   {
+      qInstance.addProcess(StreamedETLWithFrontendProcess.processMetaDataBuilder()
+         .withName("migrateOutboundApiLogToHeaderChildProcess")
+         .withLabel("Migrate Outbound API Log Test to Header/Child")
+         .withIcon(new QIcon().withName("drive_file_move"))
+         .withTableName(sourceTableName)
+         .withSourceTable(sourceTableName)
+         .withDestinationTable(OutboundAPILogHeader.TABLE_NAME)
+         .withExtractStepClass(MigrateOutboundAPILogExtractStep.class)
+         .withTransformStepClass(MigrateOutboundAPILogTransformStep.class)
+         .withLoadStepClass(MigrateOutboundAPILogLoadStep.class)
+         .withReviewStepRecordFields(List.of(
+            new QFieldMetaData("url", QFieldType.INTEGER)
+         ))
+         .getProcessMetaData());
+
+      qInstance.addProcess(StreamedETLWithFrontendProcess.processMetaDataBuilder()
+         .withName("migrateOutboundApiLogToMongoDBProcess")
+         .withLabel("Migrate Outbound API Log Test to MongoDB")
+         .withIcon(new QIcon().withName("drive_file_move"))
+         .withTableName(sourceTableName)
+         .withSourceTable(sourceTableName)
+         .withDestinationTable(OutboundAPILog.TABLE_NAME + "MongoDB")
+         .withExtractStepClass(MigrateOutboundAPILogExtractStep.class)
+         .withTransformStepClass(MigrateOutboundAPILogTransformStep.class)
+         .withLoadStepClass(MigrateOutboundAPILogLoadStep.class)
+         .withReviewStepRecordFields(List.of(
+            new QFieldMetaData("url", QFieldType.INTEGER)
+         ))
+         .getProcessMetaData());
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private static List<QPossibleValueSource> definePossibleValueSources()
+   {
+      List<QPossibleValueSource> rs = new ArrayList<>();
+
+      rs.add(new QPossibleValueSource()
          .withName("outboundApiMethod")
          .withType(QPossibleValueSourceType.ENUM)
          .withEnumValues(List.of(
@@ -72,7 +153,7 @@ public class OutboundAPILogMetaDataProvider
             new QPossibleValue<>("DELETE")
          )));
 
-      instance.addPossibleValueSource(new QPossibleValueSource()
+      rs.add(new QPossibleValueSource()
          .withName("outboundApiStatusCode")
          .withType(QPossibleValueSourceType.ENUM)
          .withEnumValues(List.of(
@@ -91,6 +172,8 @@ public class OutboundAPILogMetaDataProvider
             new QPossibleValue<>(503, "503 (Service Unavailable)"),
             new QPossibleValue<>(504, "500 (Gateway Timeout)")
          )));
+
+      return (rs);
    }
 
 
@@ -98,9 +181,8 @@ public class OutboundAPILogMetaDataProvider
    /*******************************************************************************
     **
     *******************************************************************************/
-   private static void defineOutboundAPILogTable(QInstance qInstance, String backendName, Consumer<QTableMetaData> backendDetailEnricher) throws QException
+   public static QTableMetaData defineOutboundAPILogTable(String backendName, Consumer<QTableMetaData> backendDetailEnricher) throws QException
    {
-
       QTableMetaData tableMetaData = new QTableMetaData()
          .withName(OutboundAPILog.TABLE_NAME)
          .withLabel("Outbound API Log")
@@ -119,29 +201,8 @@ public class OutboundAPILogMetaDataProvider
       tableMetaData.getField("requestBody").withFieldAdornment(new FieldAdornment(AdornmentType.CODE_EDITOR).withValue(AdornmentType.CodeEditorValues.languageMode("json")));
       tableMetaData.getField("responseBody").withFieldAdornment(new FieldAdornment(AdornmentType.CODE_EDITOR).withValue(AdornmentType.CodeEditorValues.languageMode("json")));
 
-      tableMetaData.getField("method").withFieldAdornment(new FieldAdornment(AdornmentType.CHIP)
-         .withValue(AdornmentType.ChipValues.colorValue("GET", AdornmentType.ChipValues.COLOR_INFO))
-         .withValue(AdornmentType.ChipValues.colorValue("POST", AdornmentType.ChipValues.COLOR_SUCCESS))
-         .withValue(AdornmentType.ChipValues.colorValue("DELETE", AdornmentType.ChipValues.COLOR_ERROR))
-         .withValue(AdornmentType.ChipValues.colorValue("PATCH", AdornmentType.ChipValues.COLOR_WARNING))
-         .withValue(AdornmentType.ChipValues.colorValue("PUT", AdornmentType.ChipValues.COLOR_WARNING)));
-
-      tableMetaData.getField("statusCode").withFieldAdornment(new FieldAdornment(AdornmentType.CHIP)
-         .withValue(AdornmentType.ChipValues.colorValue(200, AdornmentType.ChipValues.COLOR_SUCCESS))
-         .withValue(AdornmentType.ChipValues.colorValue(201, AdornmentType.ChipValues.COLOR_SUCCESS))
-         .withValue(AdornmentType.ChipValues.colorValue(204, AdornmentType.ChipValues.COLOR_SUCCESS))
-         .withValue(AdornmentType.ChipValues.colorValue(207, AdornmentType.ChipValues.COLOR_INFO))
-         .withValue(AdornmentType.ChipValues.colorValue(400, AdornmentType.ChipValues.COLOR_ERROR))
-         .withValue(AdornmentType.ChipValues.colorValue(401, AdornmentType.ChipValues.COLOR_ERROR))
-         .withValue(AdornmentType.ChipValues.colorValue(403, AdornmentType.ChipValues.COLOR_ERROR))
-         .withValue(AdornmentType.ChipValues.colorValue(404, AdornmentType.ChipValues.COLOR_ERROR))
-         .withValue(AdornmentType.ChipValues.colorValue(422, AdornmentType.ChipValues.COLOR_ERROR))
-         .withValue(AdornmentType.ChipValues.colorValue(429, AdornmentType.ChipValues.COLOR_ERROR))
-         .withValue(AdornmentType.ChipValues.colorValue(500, AdornmentType.ChipValues.COLOR_ERROR))
-         .withValue(AdornmentType.ChipValues.colorValue(502, AdornmentType.ChipValues.COLOR_ERROR))
-         .withValue(AdornmentType.ChipValues.colorValue(503, AdornmentType.ChipValues.COLOR_ERROR))
-         .withValue(AdornmentType.ChipValues.colorValue(504, AdornmentType.ChipValues.COLOR_ERROR))
-      );
+      addChipAdornmentToMethodField(tableMetaData);
+      addChipAdornmentToStatusCodeField(tableMetaData);
 
       ///////////////////////////////////////////
       // these are the lengths of a MySQL TEXT //
@@ -160,6 +221,210 @@ public class OutboundAPILogMetaDataProvider
          backendDetailEnricher.accept(tableMetaData);
       }
 
-      qInstance.addTable(tableMetaData);
+      return (tableMetaData);
    }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   private static void addChipAdornmentToStatusCodeField(QTableMetaData tableMetaData)
+   {
+      tableMetaData.getField("statusCode").withFieldAdornment(new FieldAdornment(AdornmentType.CHIP)
+         .withValue(AdornmentType.ChipValues.colorValue(200, AdornmentType.ChipValues.COLOR_SUCCESS))
+         .withValue(AdornmentType.ChipValues.colorValue(201, AdornmentType.ChipValues.COLOR_SUCCESS))
+         .withValue(AdornmentType.ChipValues.colorValue(204, AdornmentType.ChipValues.COLOR_SUCCESS))
+         .withValue(AdornmentType.ChipValues.colorValue(207, AdornmentType.ChipValues.COLOR_INFO))
+         .withValue(AdornmentType.ChipValues.colorValue(400, AdornmentType.ChipValues.COLOR_ERROR))
+         .withValue(AdornmentType.ChipValues.colorValue(401, AdornmentType.ChipValues.COLOR_ERROR))
+         .withValue(AdornmentType.ChipValues.colorValue(403, AdornmentType.ChipValues.COLOR_ERROR))
+         .withValue(AdornmentType.ChipValues.colorValue(404, AdornmentType.ChipValues.COLOR_ERROR))
+         .withValue(AdornmentType.ChipValues.colorValue(422, AdornmentType.ChipValues.COLOR_ERROR))
+         .withValue(AdornmentType.ChipValues.colorValue(429, AdornmentType.ChipValues.COLOR_ERROR))
+         .withValue(AdornmentType.ChipValues.colorValue(500, AdornmentType.ChipValues.COLOR_ERROR))
+         .withValue(AdornmentType.ChipValues.colorValue(502, AdornmentType.ChipValues.COLOR_ERROR))
+         .withValue(AdornmentType.ChipValues.colorValue(503, AdornmentType.ChipValues.COLOR_ERROR))
+         .withValue(AdornmentType.ChipValues.colorValue(504, AdornmentType.ChipValues.COLOR_ERROR))
+      );
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   private static void addChipAdornmentToMethodField(QTableMetaData tableMetaData)
+   {
+      tableMetaData.getField("method").withFieldAdornment(new FieldAdornment(AdornmentType.CHIP)
+         .withValue(AdornmentType.ChipValues.colorValue("GET", AdornmentType.ChipValues.COLOR_INFO))
+         .withValue(AdornmentType.ChipValues.colorValue("POST", AdornmentType.ChipValues.COLOR_SUCCESS))
+         .withValue(AdornmentType.ChipValues.colorValue("DELETE", AdornmentType.ChipValues.COLOR_ERROR))
+         .withValue(AdornmentType.ChipValues.colorValue("PATCH", AdornmentType.ChipValues.COLOR_WARNING))
+         .withValue(AdornmentType.ChipValues.colorValue("PUT", AdornmentType.ChipValues.COLOR_WARNING)));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private static QTableMetaData defineOutboundAPILogHeaderTable(String backendName, Consumer<QTableMetaData> backendDetailEnricher) throws QException
+   {
+      QTableMetaData tableMetaData = new QTableMetaData()
+         .withName(OutboundAPILogHeader.TABLE_NAME)
+         .withLabel("Outbound API Log Header/Child")
+         .withIcon(new QIcon().withName("data_object"))
+         .withBackendName(backendName)
+         .withRecordLabelFormat("%s")
+         .withRecordLabelFields("id")
+         .withPrimaryKeyField("id")
+         .withFieldsFromEntity(OutboundAPILogHeader.class)
+         .withSection(new QFieldSection("identity", new QIcon().withName("badge"), Tier.T1, List.of("id")))
+         .withSection(new QFieldSection("request", new QIcon().withName("arrow_upward"), Tier.T2, List.of("method", "url", OutboundAPILogRequest.TABLE_NAME + ".requestBody")))
+         .withSection(new QFieldSection("response", new QIcon().withName("arrow_downward"), Tier.T2, List.of("statusCode", OutboundAPILogResponse.TABLE_NAME + ".responseBody")))
+         .withSection(new QFieldSection("dates", new QIcon().withName("calendar_month"), Tier.T3, List.of("timestamp")))
+         .withoutCapabilities(Capability.TABLE_INSERT, Capability.TABLE_UPDATE, Capability.TABLE_DELETE);
+
+      // tableMetaData.getField(OutboundAPILogRequest.TABLE_NAME + ".requestBody").withFieldAdornment(new FieldAdornment(AdornmentType.CODE_EDITOR).withValue(AdornmentType.CodeEditorValues.languageMode("json")));
+      // tableMetaData.getField(OutboundAPILogResponse.TABLE_NAME + ".responseBody").withFieldAdornment(new FieldAdornment(AdornmentType.CODE_EDITOR).withValue(AdornmentType.CodeEditorValues.languageMode("json")));
+
+      addChipAdornmentToMethodField(tableMetaData);
+      addChipAdornmentToStatusCodeField(tableMetaData);
+
+      tableMetaData.withAssociation(new Association()
+         .withName(OutboundAPILogRequest.TABLE_NAME)
+         .withAssociatedTableName(OutboundAPILogRequest.TABLE_NAME)
+         .withJoinName(QJoinMetaData.makeInferredJoinName(OutboundAPILogHeader.TABLE_NAME, OutboundAPILogRequest.TABLE_NAME)));
+
+      tableMetaData.withAssociation(new Association()
+         .withName(OutboundAPILogResponse.TABLE_NAME)
+         .withAssociatedTableName(OutboundAPILogResponse.TABLE_NAME)
+         .withJoinName(QJoinMetaData.makeInferredJoinName(OutboundAPILogHeader.TABLE_NAME, OutboundAPILogResponse.TABLE_NAME)));
+
+      tableMetaData.withExposedJoin(new ExposedJoin()
+         .withJoinTable(OutboundAPILogRequest.TABLE_NAME)
+         .withJoinPath(List.of(QJoinMetaData.makeInferredJoinName(OutboundAPILogHeader.TABLE_NAME, OutboundAPILogRequest.TABLE_NAME))));
+
+      tableMetaData.withExposedJoin(new ExposedJoin()
+         .withJoinTable(OutboundAPILogResponse.TABLE_NAME)
+         .withJoinPath(List.of(QJoinMetaData.makeInferredJoinName(OutboundAPILogHeader.TABLE_NAME, OutboundAPILogResponse.TABLE_NAME))));
+
+      tableMetaData.getField("url").withMaxLength(4096).withBehavior(ValueTooLongBehavior.TRUNCATE_ELLIPSIS);
+      tableMetaData.getField("url").withFieldAdornment(AdornmentType.Size.XLARGE.toAdornment());
+
+      if(backendDetailEnricher != null)
+      {
+         backendDetailEnricher.accept(tableMetaData);
+      }
+
+      return (tableMetaData);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private static QTableMetaData defineOutboundAPILogRequestTable(String backendName, Consumer<QTableMetaData> backendDetailEnricher) throws QException
+   {
+      QTableMetaData tableMetaData = new QTableMetaData()
+         .withName(OutboundAPILogRequest.TABLE_NAME)
+         .withLabel("Outbound API Log Request")
+         .withIcon(new QIcon().withName("arrow_upward"))
+         .withBackendName(backendName)
+         .withRecordLabelFormat("%s")
+         .withRecordLabelFields("id")
+         .withPrimaryKeyField("id")
+         .withFieldsFromEntity(OutboundAPILogRequest.class)
+         .withSection(new QFieldSection("identity", new QIcon().withName("badge"), Tier.T1, List.of("id", "outboundApiLogHeaderId")))
+         .withSection(new QFieldSection("request", new QIcon().withName("arrow_upward"), Tier.T2, List.of("requestBody")))
+         .withoutCapabilities(Capability.TABLE_INSERT, Capability.TABLE_UPDATE, Capability.TABLE_DELETE);
+
+      tableMetaData.getField("requestBody").withFieldAdornment(new FieldAdornment(AdornmentType.CODE_EDITOR).withValue(AdornmentType.CodeEditorValues.languageMode("json")));
+
+      //////////////////////////////////////////////
+      // this is the length of a MySQL MEDIUMTEXT //
+      //////////////////////////////////////////////
+      tableMetaData.getField("requestBody").withMaxLength(16_777_215).withBehavior(ValueTooLongBehavior.TRUNCATE_ELLIPSIS);
+
+      if(backendDetailEnricher != null)
+      {
+         backendDetailEnricher.accept(tableMetaData);
+      }
+
+      return (tableMetaData);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private static QTableMetaData defineOutboundAPILogResponseTable(String backendName, Consumer<QTableMetaData> backendDetailEnricher) throws QException
+   {
+      QTableMetaData tableMetaData = new QTableMetaData()
+         .withName(OutboundAPILogResponse.TABLE_NAME)
+         .withLabel("Outbound API Log Response")
+         .withIcon(new QIcon().withName("arrow_upward"))
+         .withBackendName(backendName)
+         .withRecordLabelFormat("%s")
+         .withRecordLabelFields("id")
+         .withPrimaryKeyField("id")
+         .withFieldsFromEntity(OutboundAPILogResponse.class)
+         .withSection(new QFieldSection("identity", new QIcon().withName("badge"), Tier.T1, List.of("id", "outboundApiLogHeaderId")))
+         .withSection(new QFieldSection("response", new QIcon().withName("arrow_upward"), Tier.T2, List.of("responseBody")))
+         .withoutCapabilities(Capability.TABLE_INSERT, Capability.TABLE_UPDATE, Capability.TABLE_DELETE);
+
+      tableMetaData.getField("responseBody").withFieldAdornment(new FieldAdornment(AdornmentType.CODE_EDITOR).withValue(AdornmentType.CodeEditorValues.languageMode("json")));
+
+      //////////////////////////////////////////////
+      // this is the length of a MySQL MEDIUMTEXT //
+      //////////////////////////////////////////////
+      tableMetaData.getField("responseBody").withMaxLength(16_777_215).withBehavior(ValueTooLongBehavior.TRUNCATE_ELLIPSIS);
+
+      if(backendDetailEnricher != null)
+      {
+         backendDetailEnricher.accept(tableMetaData);
+      }
+
+      return (tableMetaData);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private static List<QJoinMetaData> defineJoins()
+   {
+      List<QJoinMetaData> rs = new ArrayList<>();
+
+      rs.add(new QJoinMetaData()
+         .withLeftTable(OutboundAPILogHeader.TABLE_NAME)
+         .withRightTable(OutboundAPILogRequest.TABLE_NAME)
+         .withInferredName()
+         .withType(JoinType.ONE_TO_ONE)
+         .withJoinOn(new JoinOn("id", "outboundApiLogHeaderId")));
+
+      rs.add(new QJoinMetaData()
+         .withLeftTable(OutboundAPILogHeader.TABLE_NAME)
+         .withRightTable(OutboundAPILogResponse.TABLE_NAME)
+         .withInferredName()
+         .withType(JoinType.ONE_TO_ONE)
+         .withJoinOn(new JoinOn("id", "outboundApiLogHeaderId")));
+
+      return (rs);
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   private static QPossibleValueSource defineOutboundAPILogHeaderPossibleValueSource()
+   {
+      return QPossibleValueSource.newForTable(OutboundAPILogHeader.TABLE_NAME);
+   }
+
 }

qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/model/OutboundAPILogRequest.java

@@ -0,0 +1,165 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2023.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.module.api.model;
+
+
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.data.QField;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.data.QRecordEntity;
+
+
+/*******************************************************************************
+ ** Entity bean for OutboundApiLogRequest table
+ *******************************************************************************/
+public class OutboundAPILogRequest extends QRecordEntity
+{
+   public static final String TABLE_NAME = "outboundApiLogRequest";
+
+   @QField(isEditable = false)
+   private Integer id;
+
+   @QField(possibleValueSourceName = OutboundAPILogHeader.TABLE_NAME)
+   private Integer outboundApiLogHeaderId;
+
+   @QField()
+   private String requestBody;
+
+
+
+   /*******************************************************************************
+    ** Constructor
+    **
+    *******************************************************************************/
+   public OutboundAPILogRequest()
+   {
+   }
+
+
+
+   /*******************************************************************************
+    ** Constructor
+    **
+    *******************************************************************************/
+   public OutboundAPILogRequest(QRecord qRecord) throws QException
+   {
+      populateFromQRecord(qRecord);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for id
+    **
+    *******************************************************************************/
+   public Integer getId()
+   {
+      return id;
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for id
+    **
+    *******************************************************************************/
+   public void setId(Integer id)
+   {
+      this.id = id;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for id
+    **
+    *******************************************************************************/
+   public OutboundAPILogRequest withId(Integer id)
+   {
+      this.id = id;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for requestBody
+    *******************************************************************************/
+   public String getRequestBody()
+   {
+      return (this.requestBody);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for requestBody
+    *******************************************************************************/
+   public void setRequestBody(String requestBody)
+   {
+      this.requestBody = requestBody;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for requestBody
+    *******************************************************************************/
+   public OutboundAPILogRequest withRequestBody(String requestBody)
+   {
+      this.requestBody = requestBody;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for outboundApiLogHeaderId
+    *******************************************************************************/
+   public Integer getOutboundApiLogHeaderId()
+   {
+      return (this.outboundApiLogHeaderId);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for outboundApiLogHeaderId
+    *******************************************************************************/
+   public void setOutboundApiLogHeaderId(Integer outboundApiLogHeaderId)
+   {
+      this.outboundApiLogHeaderId = outboundApiLogHeaderId;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for outboundApiLogHeaderId
+    *******************************************************************************/
+   public OutboundAPILogRequest withOutboundApiLogHeaderId(Integer outboundApiLogHeaderId)
+   {
+      this.outboundApiLogHeaderId = outboundApiLogHeaderId;
+      return (this);
+   }
+
+}
+

qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/model/OutboundAPILogResponse.java

@@ -0,0 +1,165 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2023.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.module.api.model;
+
+
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.data.QField;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.data.QRecordEntity;
+
+
+/*******************************************************************************
+ ** Entity bean for OutboundApiLogResponse table
+ *******************************************************************************/
+public class OutboundAPILogResponse extends QRecordEntity
+{
+   public static final String TABLE_NAME = "outboundApiLogResponse";
+
+   @QField(isEditable = false)
+   private Integer id;
+
+   @QField(possibleValueSourceName = OutboundAPILogHeader.TABLE_NAME)
+   private Integer outboundApiLogHeaderId;
+
+   @QField()
+   private String responseBody;
+
+
+
+   /*******************************************************************************
+    ** Constructor
+    **
+    *******************************************************************************/
+   public OutboundAPILogResponse()
+   {
+   }
+
+
+
+   /*******************************************************************************
+    ** Constructor
+    **
+    *******************************************************************************/
+   public OutboundAPILogResponse(QRecord qRecord) throws QException
+   {
+      populateFromQRecord(qRecord);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for id
+    **
+    *******************************************************************************/
+   public Integer getId()
+   {
+      return id;
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for id
+    **
+    *******************************************************************************/
+   public void setId(Integer id)
+   {
+      this.id = id;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for id
+    **
+    *******************************************************************************/
+   public OutboundAPILogResponse withId(Integer id)
+   {
+      this.id = id;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for responseBody
+    *******************************************************************************/
+   public String getResponseBody()
+   {
+      return (this.responseBody);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for responseBody
+    *******************************************************************************/
+   public void setResponseBody(String responseBody)
+   {
+      this.responseBody = responseBody;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for responseBody
+    *******************************************************************************/
+   public OutboundAPILogResponse withResponseBody(String responseBody)
+   {
+      this.responseBody = responseBody;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for outboundApiLogHeaderId
+    *******************************************************************************/
+   public Integer getOutboundApiLogHeaderId()
+   {
+      return (this.outboundApiLogHeaderId);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for outboundApiLogHeaderId
+    *******************************************************************************/
+   public void setOutboundApiLogHeaderId(Integer outboundApiLogHeaderId)
+   {
+      this.outboundApiLogHeaderId = outboundApiLogHeaderId;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for outboundApiLogHeaderId
+    *******************************************************************************/
+   public OutboundAPILogResponse withOutboundApiLogHeaderId(Integer outboundApiLogHeaderId)
+   {
+      this.outboundApiLogHeaderId = outboundApiLogHeaderId;
+      return (this);
+   }
+
+}
+

qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/processes/MigrateOutboundAPILogExtractStep.java

@@ -19,25 +19,27 @@
  * along with this program.  If not, see <https://www.gnu.org/licenses/>.
  */
 
-package com.kingsrook.qqq.backend.core.actions.reporting.customizers;
+package com.kingsrook.qqq.backend.module.api.processes;
 
 
-import com.kingsrook.qqq.backend.core.actions.reporting.RecordPipe;
-import com.kingsrook.qqq.backend.core.exceptions.QException;
-import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportInput;
-import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportDataSource;
+import com.kingsrook.qqq.backend.core.model.actions.tables.QueryHint;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
+import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.ExtractViaQueryStep;
 
 
 /*******************************************************************************
- ** Interface to be implemented to do a custom source of data for a report
- ** (instead of just a query against a table).
+ **
  *******************************************************************************/
-public interface ReportCustomRecordSourceInterface
+public class MigrateOutboundAPILogExtractStep extends ExtractViaQueryStep
 {
 
-   /***************************************************************************
-    ** Given the report input, put records into the pipe, for the report.
-    ***************************************************************************/
-   void execute(ReportInput reportInput, QReportDataSource reportDataSource, RecordPipe recordPipe) throws QException;
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   protected void customizeInputPreQuery(QueryInput queryInput)
+   {
+      queryInput.withQueryHint(QueryHint.POTENTIALLY_LARGE_NUMBER_OF_RESULTS);
+   }
 
 }

qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/processes/MigrateOutboundAPILogLoadStep.java

@@ -0,0 +1,54 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.module.api.processes;
+
+
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.logging.QLogger;
+import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
+import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
+import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.LoadViaInsertStep;
+
+
+/*******************************************************************************
+ ** store outboundApiLogHeaders and maybe more...
+ *******************************************************************************/
+public class MigrateOutboundAPILogLoadStep extends LoadViaInsertStep
+{
+   private static final QLogger LOG = QLogger.getLogger(MigrateOutboundAPILogLoadStep.class);
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public void runOnePage(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException
+   {
+      super.runOnePage(runBackendStepInput, runBackendStepOutput);
+
+      ///////////////////////////////////////
+      // todo - track what we've migrated? //
+      ///////////////////////////////////////
+   }
+
+}

qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/processes/MigrateOutboundAPILogTransformStep.java

@@ -0,0 +1,127 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.module.api.processes;
+
+
+import java.util.ArrayList;
+import java.util.List;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.logging.QLogger;
+import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLine;
+import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLineInterface;
+import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
+import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.AbstractTransformStep;
+import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess;
+import com.kingsrook.qqq.backend.core.processes.implementations.general.StandardProcessSummaryLineProducer;
+import com.kingsrook.qqq.backend.module.api.model.OutboundAPILogHeader;
+import com.kingsrook.qqq.backend.module.api.model.OutboundAPILogRequest;
+import com.kingsrook.qqq.backend.module.api.model.OutboundAPILogResponse;
+
+
+/*******************************************************************************
+ ** migrate records from original (singular) outboundApiLog table to new split-up
+ ** version (outboundApiLogHeader)
+ *******************************************************************************/
+public class MigrateOutboundAPILogTransformStep extends AbstractTransformStep
+{
+   private static final QLogger LOG = QLogger.getLogger(MigrateOutboundAPILogTransformStep.class);
+
+   private ProcessSummaryLine okToInsertLine = StandardProcessSummaryLineProducer.getOkToInsertLine();
+   private ProcessSummaryLine errorLine      = StandardProcessSummaryLineProducer.getErrorLine();
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   /*
+   @Override
+   public Integer getOverrideRecordPipeCapacity(RunBackendStepInput runBackendStepInput)
+   {
+      return (100);
+   }
+   */
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public ArrayList<ProcessSummaryLineInterface> getProcessSummary(RunBackendStepOutput runBackendStepOutput, boolean isForResultScreen)
+   {
+      ArrayList<ProcessSummaryLineInterface> rs = new ArrayList<>();
+      okToInsertLine.addSelfToListIfAnyCount(rs);
+      errorLine.addSelfToListIfAnyCount(rs);
+      return (rs);
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   @Override
+   public void preRun(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException
+   {
+      runBackendStepOutput.addValue("counter", 0);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public void runOnePage(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException
+   {
+      int counter = runBackendStepOutput.getValueInteger("counter") + runBackendStepInput.getRecords().size();
+      runBackendStepOutput.addValue("counter", counter);
+      runBackendStepInput.getAsyncJobCallback().updateStatus("Migrating records (at #" + String.format("%,d", counter) + ")");
+
+      String destinationTable = runBackendStepInput.getValueString(StreamedETLWithFrontendProcess.FIELD_DESTINATION_TABLE);
+
+      for(QRecord record : runBackendStepInput.getRecords())
+      {
+         okToInsertLine.incrementCountAndAddPrimaryKey(record.getValue("id"));
+
+         if(destinationTable.equals(OutboundAPILogHeader.TABLE_NAME))
+         {
+            OutboundAPILogHeader outboundAPILogHeader = new OutboundAPILogHeader(record);
+            outboundAPILogHeader.withOutboundAPILogRequestList(List.of(new OutboundAPILogRequest().withRequestBody(record.getValueString("requestBody"))));
+            outboundAPILogHeader.withOutboundAPILogResponseList(List.of(new OutboundAPILogResponse().withResponseBody(record.getValueString("responseBody"))));
+            runBackendStepOutput.addRecord(outboundAPILogHeader.toQRecord());
+         }
+         else
+         {
+            ///////////////////////////////////////////////////////////////////
+            // for the mongodb migration, just pass records straight through //
+            ///////////////////////////////////////////////////////////////////
+            record.setValue("id", null);
+            runBackendStepOutput.addRecord(record);
+         }
+      }
+   }
+
+}

qqq-backend-module-api/src/test/java/com/kingsrook/qqq/backend/module/api/BaseTest.java

@@ -26,6 +26,7 @@ import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.logging.QLogger;
 import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
 import com.kingsrook.qqq.backend.core.model.session.QSession;
+import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryRecordStore;
 import org.junit.jupiter.api.AfterEach;
 import org.junit.jupiter.api.BeforeEach;
 
@@ -46,6 +47,7 @@ public class BaseTest
    void baseBeforeEach()
    {
       QContext.init(TestUtils.defineInstance(), new QSession());
+      MemoryRecordStore.getInstance().reset();
    }
 
 
@@ -57,6 +59,7 @@ public class BaseTest
    void baseAfterEach()
    {
       QContext.clear();
+      MemoryRecordStore.getInstance().reset();
    }
 
 

qqq-backend-module-api/src/test/java/com/kingsrook/qqq/backend/module/api/mocks/MockApiActionUtils.java

@@ -28,6 +28,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 import com.kingsrook.qqq.backend.module.api.actions.BaseAPIActionUtil;
 import com.kingsrook.qqq.backend.module.api.actions.QHttpResponse;
 import org.apache.http.client.methods.CloseableHttpResponse;
+import org.apache.http.client.methods.HttpPost;
 import org.apache.http.client.methods.HttpRequestBase;
 import org.apache.http.impl.client.CloseableHttpClient;
 
@@ -88,7 +89,7 @@ public class MockApiActionUtils extends BaseAPIActionUtil
     **
     *******************************************************************************/
    @Override
-   protected CloseableHttpResponse executeOAuthTokenRequest(CloseableHttpClient client, HttpRequestBase request) throws IOException
+   protected CloseableHttpResponse executeOAuthTokenRequest(CloseableHttpClient client, HttpPost request) throws IOException
    {
       runMockAsserter(request);
       return new MockHttpResponse(mockApiUtilsHelper);

qqq-backend-module-api/src/test/java/com/kingsrook/qqq/backend/module/api/processes/MigrateOutboundAPILogProcessTest.java

@@ -0,0 +1,98 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.module.api.processes;
+
+
+import java.util.List;
+import com.kingsrook.qqq.backend.core.actions.processes.QProcessCallbackFactory;
+import com.kingsrook.qqq.backend.core.actions.processes.RunProcessAction;
+import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
+import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
+import com.kingsrook.qqq.backend.core.context.QContext;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput;
+import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessOutput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
+import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryRecordStore;
+import com.kingsrook.qqq.backend.module.api.BaseTest;
+import com.kingsrook.qqq.backend.module.api.TestUtils;
+import com.kingsrook.qqq.backend.module.api.model.OutboundAPILog;
+import com.kingsrook.qqq.backend.module.api.model.OutboundAPILogHeader;
+import com.kingsrook.qqq.backend.module.api.model.OutboundAPILogMetaDataProvider;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+
+/*******************************************************************************
+ ** Unit test for MigrateOutboundAPILog process
+ *******************************************************************************/
+class MigrateOutboundAPILogProcessTest extends BaseTest
+{
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @BeforeEach
+   void beforeEach() throws QException
+   {
+      MemoryRecordStore.getInstance().reset();
+      OutboundAPILogMetaDataProvider.defineAll(QContext.getQInstance(), TestUtils.MEMORY_BACKEND_NAME, null);
+      OutboundAPILogMetaDataProvider.defineNewVersion(QContext.getQInstance(), TestUtils.MEMORY_BACKEND_NAME, null);
+      OutboundAPILogMetaDataProvider.defineMigrationProcesses(QContext.getQInstance(), OutboundAPILog.TABLE_NAME);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void test() throws QException
+   {
+      new InsertAction().execute(new InsertInput(OutboundAPILog.TABLE_NAME).withRecordEntity(new OutboundAPILog()
+         .withMethod("POST")
+         .withUrl("www.google.com")
+         .withRequestBody("please")
+         .withResponseBody("you're welcome")
+         .withStatusCode(201)
+      ));
+
+      RunProcessInput input = new RunProcessInput();
+      input.setProcessName("migrateOutboundApiLogToHeaderChildProcess");
+      input.setCallback(QProcessCallbackFactory.forFilter(new QQueryFilter()));
+      input.setFrontendStepBehavior(RunProcessInput.FrontendStepBehavior.SKIP);
+      RunProcessOutput runProcessOutput = new RunProcessAction().execute(input);
+
+      List<OutboundAPILogHeader> outboundApiLogHeaderList = new QueryAction().execute(new QueryInput(OutboundAPILogHeader.TABLE_NAME).withIncludeAssociations(true)).getRecordEntities(OutboundAPILogHeader.class);
+      assertEquals(1, outboundApiLogHeaderList.size());
+      assertEquals("POST", outboundApiLogHeaderList.get(0).getMethod());
+      assertEquals(201, outboundApiLogHeaderList.get(0).getStatusCode());
+      assertEquals(1, outboundApiLogHeaderList.get(0).getOutboundAPILogRequestList().size());
+      assertEquals("please", outboundApiLogHeaderList.get(0).getOutboundAPILogRequestList().get(0).getRequestBody());
+      assertEquals(1, outboundApiLogHeaderList.get(0).getOutboundAPILogResponseList().size());
+      assertEquals("you're welcome", outboundApiLogHeaderList.get(0).getOutboundAPILogResponseList().get(0).getResponseBody());
+   }
+
+}

qqq-backend-module-mongodb/src/main/java/com/kingsrook/qqq/backend/module/mongodb/actions/AbstractMongoDBAction.java

@@ -41,7 +41,6 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.query.JoinsContext;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
-import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.expressions.AbstractFilterExpression;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
 import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
@@ -177,28 +176,18 @@ public class AbstractMongoDBAction
    /*******************************************************************************
     ** Convert a mongodb document to a QRecord.
     *******************************************************************************/
-   protected QRecord documentToRecord(QueryInput queryInput, Document document)
+   protected QRecord documentToRecord(QTableMetaData table, Document document)
    {
-      QTableMetaData table  = queryInput.getTable();
-      QRecord        record = new QRecord();
-
+      QRecord record = new QRecord();
       record.setTableName(table.getName());
 
-      /////////////////////////////////////////////
-      // build the set of field names to include //
-      /////////////////////////////////////////////
-      Set<String> fieldNamesToInclude = queryInput.getFieldNamesToInclude();
-      List<QFieldMetaData> selectedFields = table.getFields().values()
-         .stream().filter(field -> fieldNamesToInclude == null || fieldNamesToInclude.contains(field.getName()))
-         .toList();
-
       //////////////////////////////////////////////////////////////////////////////////////////////
       // first iterate over the table's fields, looking for them (at their backend name (path,    //
       // if it has dots) inside the document note that we'll remove values from the document      //
       // as we go - then after this loop, will handle all remaining values as unstructured fields //
       //////////////////////////////////////////////////////////////////////////////////////////////
       Map<String, Serializable> values = record.getValues();
-      for(QFieldMetaData field : selectedFields)
+      for(QFieldMetaData field : table.getFields().values())
       {
          String fieldName        = field.getName();
          String fieldBackendName = getFieldBackendName(field);

qqq-backend-module-mongodb/src/main/java/com/kingsrook/qqq/backend/module/mongodb/actions/MongoDBQueryAction.java

@@ -41,7 +41,6 @@ import com.kingsrook.qqq.backend.module.mongodb.model.metadata.MongoDBBackendMet
 import com.mongodb.client.FindIterable;
 import com.mongodb.client.MongoCollection;
 import com.mongodb.client.MongoDatabase;
-import com.mongodb.client.model.Projections;
 import org.bson.Document;
 import org.bson.conversions.Bson;
 
@@ -97,15 +96,6 @@ public class MongoDBQueryAction extends AbstractMongoDBAction implements QueryIn
          ////////////////////////////////////////////////////////////
          FindIterable<Document> cursor = collection.find(mongoClientContainer.getMongoSession(), searchQuery);
 
-         ///////////////////////////////////////////////////////////////////////////////////////////////
-         // if input specifies a set of field names to include, then add a 'projection' to the cursor //
-         ///////////////////////////////////////////////////////////////////////////////////////////////
-         if(queryInput.getFieldNamesToInclude() != null)
-         {
-            List<String> backendFieldNames = queryInput.getFieldNamesToInclude().stream().map(f -> getFieldBackendName(table.getField(f))).toList();
-            cursor.projection(Projections.include(backendFieldNames));
-         }
-
          ///////////////////////////////////
          // add a sort operator if needed //
          ///////////////////////////////////
@@ -148,7 +138,7 @@ public class MongoDBQueryAction extends AbstractMongoDBAction implements QueryIn
             actionTimeoutHelper.cancel();
             setQueryStatFirstResultTime();
 
-            QRecord record = documentToRecord(queryInput, document);
+            QRecord record = documentToRecord(table, document);
             queryOutput.addRecord(record);
 
             if(queryInput.getAsyncJobCallback().wasCancelRequested())

qqq-backend-module-mongodb/src/test/java/com/kingsrook/qqq/backend/module/mongodb/actions/MongoDBQueryActionTest.java

@@ -29,7 +29,6 @@ import java.util.Collections;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
-import java.util.Set;
 import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
 import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
 import com.kingsrook.qqq.backend.core.context.QContext;
@@ -55,8 +54,6 @@ import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
 import static org.assertj.core.api.Assertions.assertThat;
 import static org.junit.jupiter.api.Assertions.assertEquals;
-import static org.junit.jupiter.api.Assertions.assertFalse;
-import static org.junit.jupiter.api.Assertions.assertTrue;
 
 
 /*******************************************************************************
@@ -997,46 +994,4 @@ class MongoDBQueryActionTest extends BaseTest
          .allMatch(r -> r.getValueInteger("storeKey").equals(1));
    }
 
-
-
-   /*******************************************************************************
-    **
-    *******************************************************************************/
-   @Test
-   void testFieldNamesToInclude() throws QException
-   {
-      QQueryFilter  filter     = new QQueryFilter().withCriteria("firstName", QCriteriaOperator.EQUALS, "Darin");
-      QueryInput    queryInput = new QueryInput(TestUtils.TABLE_NAME_PERSON).withFilter(filter);
-
-      QRecord record = new QueryAction().execute(queryInput.withFieldNamesToInclude(null)).getRecords().get(0);
-      assertTrue(record.getValues().containsKey("id"));
-      assertTrue(record.getValues().containsKey("firstName"));
-      assertTrue(record.getValues().containsKey("createDate"));
-      //////////////////////////////////////////////////////////////////////////////
-      // note, we get an extra "metaData" field (??), which, i guess is expected? //
-      //////////////////////////////////////////////////////////////////////////////
-      assertEquals(QContext.getQInstance().getTable(TestUtils.TABLE_NAME_PERSON).getFields().size() + 1, record.getValues().size());
-
-      record = new QueryAction().execute(queryInput.withFieldNamesToInclude(Set.of("id", "firstName"))).getRecords().get(0);
-      assertTrue(record.getValues().containsKey("id"));
-      assertTrue(record.getValues().containsKey("firstName"));
-      assertFalse(record.getValues().containsKey("createDate"));
-      assertEquals(2, record.getValues().size());
-      //////////////////////////////////////////////////////////////////////////////////////////////
-      // here, we'd have put _id (which mongo always returns) as "id", since caller requested it. //
-      //////////////////////////////////////////////////////////////////////////////////////////////
-      assertFalse(record.getValues().containsKey("_id"));
-
-      record = new QueryAction().execute(queryInput.withFieldNamesToInclude(Set.of("homeTown"))).getRecords().get(0);
-      assertFalse(record.getValues().containsKey("id"));
-      assertFalse(record.getValues().containsKey("firstName"));
-      assertFalse(record.getValues().containsKey("createDate"));
-      assertTrue(record.getValues().containsKey("homeTown"));
-      //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-      // here, mongo always gives back _id (but, we won't have re-mapped it to "id", since caller didn't request it), so, do expect 2 fields here //
-      //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-      assertEquals(2, record.getValues().size());
-      assertTrue(record.getValues().containsKey("_id"));
-   }
-
 }

qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSQueryAction.java

@@ -34,7 +34,6 @@ import java.util.HashMap;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
-import java.util.Set;
 import java.util.concurrent.TimeUnit;
 import java.util.stream.Collectors;
 import com.kingsrook.qqq.backend.core.actions.interfaces.QueryInterface;
@@ -95,8 +94,7 @@ public class RDBMSQueryAction extends AbstractRDBMSAction implements QueryInterf
          QTableMetaData table     = queryInput.getTable();
          String         tableName = queryInput.getTableName();
 
-         Selection     selection = makeSelection(queryInput);
-         StringBuilder sql = new StringBuilder(selection.selectClause());
+         StringBuilder sql = new StringBuilder(makeSelectClause(queryInput));
 
          QQueryFilter filter       = clonedOrNewFilter(queryInput.getFilter());
          JoinsContext joinsContext = new JoinsContext(QContext.getQInstance(), tableName, queryInput.getQueryJoins(), filter);
@@ -137,6 +135,23 @@ public class RDBMSQueryAction extends AbstractRDBMSAction implements QueryInterf
             needToCloseConnection = true;
          }
 
+         ////////////////////////////////////////////////////////////////////////////
+         // build the list of fields that will be processed in the result-set loop //
+         ////////////////////////////////////////////////////////////////////////////
+         List<QFieldMetaData> fieldList = new ArrayList<>(table.getFields().values().stream().toList());
+         for(QueryJoin queryJoin : CollectionUtils.nonNullList(queryInput.getQueryJoins()))
+         {
+            if(queryJoin.getSelect())
+            {
+               QTableMetaData joinTable        = QContext.getQInstance().getTable(queryJoin.getJoinTable());
+               String         tableNameOrAlias = queryJoin.getJoinTableOrItsAlias();
+               for(QFieldMetaData joinField : joinTable.getFields().values())
+               {
+                  fieldList.add(joinField.clone().withName(tableNameOrAlias + "." + joinField.getName()));
+               }
+            }
+         }
+
          Long mark = System.currentTimeMillis();
 
          try
@@ -184,7 +199,7 @@ public class RDBMSQueryAction extends AbstractRDBMSAction implements QueryInterf
 
                   for(int i = 1; i <= metaData.getColumnCount(); i++)
                   {
-                     QFieldMetaData field = selection.fields().get(i - 1);
+                     QFieldMetaData field = fieldList.get(i - 1);
 
                      if(!queryInput.getShouldFetchHeavyFields() && field.getIsHeavy())
                      {
@@ -279,62 +294,26 @@ public class RDBMSQueryAction extends AbstractRDBMSAction implements QueryInterf
 
 
 
-   /***************************************************************************
-    ** output wrapper for makeSelection method.
-    ** - selectClause is everything from SELECT up to (but not including) FROM
-    ** - fields are those being selected, in the same order, and with mutated
-    ** names for join fields.
-    ***************************************************************************/
-   private record Selection(String selectClause, List<QFieldMetaData> fields)
-   {
-
-   }
-
-
-
    /*******************************************************************************
-    ** For a given queryInput, determine what fields are being selected - returning
-    ** a record containing the SELECT clause, as well as a List of QFieldMetaData
-    ** representing those fields - where - note - the names for fields from join
-    ** tables will be prefixed by the join table nameOrAlias.
+    **
     *******************************************************************************/
-   private Selection makeSelection(QueryInput queryInput) throws QException
+   private String makeSelectClause(QueryInput queryInput) throws QException
    {
       QInstance       instance   = QContext.getQInstance();
       String          tableName  = queryInput.getTableName();
       List<QueryJoin> queryJoins = queryInput.getQueryJoins();
       QTableMetaData  table      = instance.getTable(tableName);
 
-      Set<String> fieldNamesToInclude = queryInput.getFieldNamesToInclude();
+      boolean requiresDistinct = queryInput.getSelectDistinct() || doesSelectClauseRequireDistinct(table);
+      String  clausePrefix     = (requiresDistinct) ? "SELECT DISTINCT " : "SELECT ";
 
-      ///////////////////////////////////////////////////////////////////////////////////////////////
-      // start with the main table's fields, optionally filtered by the set of fieldNamesToInclude //
-      ///////////////////////////////////////////////////////////////////////////////////////////////
-      List<QFieldMetaData> fieldList = table.getFields().values()
-         .stream().filter(field -> fieldNamesToInclude == null || fieldNamesToInclude.contains(field.getName()))
-         .toList();
-
-      //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-      // map those field names to columns, joined with ", ".                                                              //
-      // if a field is heavy, and heavy fields aren't being selected, then replace that field name with a LENGTH function //
-      //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      List<QFieldMetaData> fieldList = new ArrayList<>(table.getFields().values());
       String columns = fieldList.stream()
          .map(field -> Pair.of(field, escapeIdentifier(tableName) + "." + escapeIdentifier(getColumnName(field))))
          .map(pair -> wrapHeavyFieldsWithLengthFunctionIfNeeded(pair, queryInput.getShouldFetchHeavyFields()))
          .collect(Collectors.joining(", "));
+      StringBuilder rs = new StringBuilder(clausePrefix).append(columns);
 
-      ///////////////////////////////////////////////////////////////////////////////////////////////////////////
-      // figure out if distinct is being used.  then start building the select clause with the table's columns //
-      ///////////////////////////////////////////////////////////////////////////////////////////////////////////
-      boolean              requiresDistinct   = queryInput.getSelectDistinct() || doesSelectClauseRequireDistinct(table);
-      StringBuilder        selectClause       = new StringBuilder((requiresDistinct) ? "SELECT DISTINCT " : "SELECT ").append(columns);
-      List<QFieldMetaData> selectionFieldList = new ArrayList<>(fieldList);
-
-      boolean needCommaBeforeJoinFields = !columns.isEmpty();
-
-      ///////////////////////////////////
-      // add any 'selected' queryJoins //
-      ///////////////////////////////////
       for(QueryJoin queryJoin : CollectionUtils.nonNullList(queryJoins))
       {
          if(queryJoin.getSelect())
@@ -346,41 +325,16 @@ public class RDBMSQueryAction extends AbstractRDBMSAction implements QueryInterf
                throw new QException("Requested join table [" + queryJoin.getJoinTable() + "] is not a defined table.");
             }
 
-            ///////////////////////////////////
-            // filter by fieldNamesToInclude //
-            ///////////////////////////////////
-            List<QFieldMetaData> joinFieldList = joinTable.getFields().values()
-               .stream().filter(field -> fieldNamesToInclude == null || fieldNamesToInclude.contains(tableNameOrAlias + "." + field.getName()))
-               .toList();
-            if(joinFieldList.isEmpty())
-            {
-               continue;
-            }
-
-            /////////////////////////////////////////////////////
-            // map to columns, wrapping heavy fields as needed //
-            /////////////////////////////////////////////////////
+            List<QFieldMetaData> joinFieldList = new ArrayList<>(joinTable.getFields().values());
             String joinColumns = joinFieldList.stream()
                .map(field -> Pair.of(field, escapeIdentifier(tableNameOrAlias) + "." + escapeIdentifier(getColumnName(field))))
                .map(pair -> wrapHeavyFieldsWithLengthFunctionIfNeeded(pair, queryInput.getShouldFetchHeavyFields()))
                .collect(Collectors.joining(", "));
-
-            ////////////////////////////////////////////////////////////////////////////////////////////////
-            // append to output objects.                                                                  //
-            // note that fields are cloned, since we are changing their names to have table/alias prefix. //
-            ////////////////////////////////////////////////////////////////////////////////////////////////
-            if(needCommaBeforeJoinFields)
-            {
-               selectClause.append(", ");
-            }
-            selectClause.append(joinColumns);
-            needCommaBeforeJoinFields = true;
-
-            selectionFieldList.addAll(joinFieldList.stream().map(field -> field.clone().withName(tableNameOrAlias + "." + field.getName())).toList());
+            rs.append(", ").append(joinColumns);
          }
       }
 
-      return (new Selection(selectClause.toString(), selectionFieldList));
+      return (rs.toString());
    }
 
 

qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSQueryActionJoinsTest.java

@@ -25,7 +25,6 @@ package com.kingsrook.qqq.backend.module.rdbms.actions;
 import java.util.Collections;
 import java.util.List;
 import java.util.Map;
-import java.util.Set;
 import java.util.concurrent.atomic.AtomicInteger;
 import com.kingsrook.qqq.backend.core.actions.tables.CountAction;
 import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
@@ -1053,48 +1052,4 @@ public class RDBMSQueryActionJoinsTest extends RDBMSActionTest
       assertEquals(5, new QueryAction().execute(new QueryInput(TestUtils.TABLE_NAME_PERSON)).getRecords().size());
    }
 
-
-
-   /*******************************************************************************
-    **
-    *******************************************************************************/
-   @Test
-   void testFieldNamesToInclude() throws QException
-   {
-      QueryInput queryInput = initQueryRequest();
-      queryInput.withQueryJoin(new QueryJoin(TestUtils.TABLE_NAME_PERSONAL_ID_CARD).withSelect(true));
-      queryInput.withFieldNamesToInclude(Set.of("firstName", TestUtils.TABLE_NAME_PERSONAL_ID_CARD + ".idNumber"));
-      QueryOutput queryOutput = new QueryAction().execute(queryInput);
-      assertThat(queryOutput.getRecords()).allMatch(r -> r.getValues().containsKey("firstName"));
-      assertThat(queryOutput.getRecords()).allMatch(r -> r.getValues().containsKey(TestUtils.TABLE_NAME_PERSONAL_ID_CARD + ".idNumber"));
-      assertThat(queryOutput.getRecords()).allMatch(r -> r.getValues().size() == 2);
-
-      ////////////////////////////////////////////////////////////////////////////////////////////////////
-      // re-run w/ null fieldNamesToInclude -- and should still see those 2, and more (values size > 2) //
-      ////////////////////////////////////////////////////////////////////////////////////////////////////
-      queryInput.withFieldNamesToInclude(null);
-      queryOutput = new QueryAction().execute(queryInput);
-      assertThat(queryOutput.getRecords()).allMatch(r -> r.getValues().containsKey("firstName"));
-      assertThat(queryOutput.getRecords()).allMatch(r -> r.getValues().containsKey(TestUtils.TABLE_NAME_PERSONAL_ID_CARD + ".idNumber"));
-      assertThat(queryOutput.getRecords()).allMatch(r -> r.getValues().size() > 2);
-
-      ////////////////////////////////////////////////////////////////////////////
-      // regression from original build - where only join fields made sql error //
-      ////////////////////////////////////////////////////////////////////////////
-      queryInput.withFieldNamesToInclude(Set.of(TestUtils.TABLE_NAME_PERSONAL_ID_CARD + ".idNumber"));
-      queryOutput = new QueryAction().execute(queryInput);
-      assertThat(queryOutput.getRecords()).allMatch(r -> r.getValues().containsKey(TestUtils.TABLE_NAME_PERSONAL_ID_CARD + ".idNumber"));
-      assertThat(queryOutput.getRecords()).allMatch(r -> r.getValues().size() == 1);
-
-      //////////////////////////////////////////////////////////////////////////////////////////
-      // similar regression to above, if one of the join tables didn't have anything selected //
-      //////////////////////////////////////////////////////////////////////////////////////////
-      queryInput.withQueryJoin(new QueryJoin(TestUtils.TABLE_NAME_PERSONAL_ID_CARD).withAlias("id2").withSelect(true));
-      queryInput.withFieldNamesToInclude(Set.of("firstName", "id2.idNumber"));
-      queryOutput = new QueryAction().execute(queryInput);
-      assertThat(queryOutput.getRecords()).allMatch(r -> r.getValues().containsKey("firstName"));
-      assertThat(queryOutput.getRecords()).allMatch(r -> r.getValues().containsKey("id2.idNumber"));
-      assertThat(queryOutput.getRecords()).allMatch(r -> r.getValues().size() == 2);
-   }
-
 }

qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSQueryActionTest.java

@@ -28,7 +28,6 @@ import java.time.temporal.ChronoUnit;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
-import java.util.Set;
 import java.util.function.Predicate;
 import com.kingsrook.qqq.backend.core.actions.QBackendTransaction;
 import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
@@ -49,12 +48,11 @@ import com.kingsrook.qqq.backend.core.model.metadata.security.RecordSecurityLock
 import com.kingsrook.qqq.backend.core.model.session.QSession;
 import com.kingsrook.qqq.backend.module.rdbms.TestUtils;
 import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.Assertions;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
 import static org.assertj.core.api.Assertions.assertThat;
 import static org.junit.jupiter.api.Assertions.assertEquals;
-import static org.junit.jupiter.api.Assertions.assertFalse;
-import static org.junit.jupiter.api.Assertions.assertTrue;
 
 
 /*******************************************************************************
@@ -168,7 +166,7 @@ public class RDBMSQueryActionTest extends RDBMSActionTest
       );
       QueryOutput queryOutput = new RDBMSQueryAction().execute(queryInput);
       assertEquals(4, queryOutput.getRecords().size(), "Expected # of rows");
-      assertTrue(queryOutput.getRecords().stream().noneMatch(r -> r.getValueString("email").equals(email)), "Should NOT find expected email address");
+      Assertions.assertTrue(queryOutput.getRecords().stream().noneMatch(r -> r.getValueString("email").equals(email)), "Should NOT find expected email address");
    }
 
 
@@ -201,7 +199,7 @@ public class RDBMSQueryActionTest extends RDBMSActionTest
             .withValues(List.of(1_000_000))));
       queryOutput = new RDBMSQueryAction().execute(queryInput);
       assertEquals(4, queryOutput.getRecords().size(), "Expected # of rows");
-      assertTrue(queryOutput.getRecords().stream().noneMatch(r -> Objects.equals(1_000_000, r.getValueInteger("annualSalary"))), "Should NOT find expected salary");
+      Assertions.assertTrue(queryOutput.getRecords().stream().noneMatch(r -> Objects.equals(1_000_000, r.getValueInteger("annualSalary"))), "Should NOT find expected salary");
    }
 
 
@@ -221,7 +219,7 @@ public class RDBMSQueryActionTest extends RDBMSActionTest
       );
       QueryOutput queryOutput = new RDBMSQueryAction().execute(queryInput);
       assertEquals(2, queryOutput.getRecords().size(), "Expected # of rows");
-      assertTrue(queryOutput.getRecords().stream().allMatch(r -> r.getValueInteger("id").equals(2) || r.getValueInteger("id").equals(4)), "Should find expected ids");
+      Assertions.assertTrue(queryOutput.getRecords().stream().allMatch(r -> r.getValueInteger("id").equals(2) || r.getValueInteger("id").equals(4)), "Should find expected ids");
    }
 
 
@@ -241,7 +239,7 @@ public class RDBMSQueryActionTest extends RDBMSActionTest
       );
       QueryOutput queryOutput = new RDBMSQueryAction().execute(queryInput);
       assertEquals(2, queryOutput.getRecords().size(), "Expected # of rows");
-      assertTrue(queryOutput.getRecords().stream().allMatch(r -> r.getValueInteger("id").equals(1) || r.getValueInteger("id").equals(5)), "Should find expected ids");
+      Assertions.assertTrue(queryOutput.getRecords().stream().allMatch(r -> r.getValueInteger("id").equals(1) || r.getValueInteger("id").equals(5)), "Should find expected ids");
    }
 
 
@@ -261,7 +259,7 @@ public class RDBMSQueryActionTest extends RDBMSActionTest
       );
       QueryOutput queryOutput = new RDBMSQueryAction().execute(queryInput);
       assertEquals(1, queryOutput.getRecords().size(), "Expected # of rows");
-      assertTrue(queryOutput.getRecords().stream().allMatch(r -> r.getValueString("email").matches("darin.*")), "Should find matching email address");
+      Assertions.assertTrue(queryOutput.getRecords().stream().allMatch(r -> r.getValueString("email").matches("darin.*")), "Should find matching email address");
    }
 
 
@@ -281,7 +279,7 @@ public class RDBMSQueryActionTest extends RDBMSActionTest
       );
       QueryOutput queryOutput = new RDBMSQueryAction().execute(queryInput);
       assertEquals(1, queryOutput.getRecords().size(), "Expected # of rows");
-      assertTrue(queryOutput.getRecords().stream().allMatch(r -> r.getValueString("email").matches(".*kelkhoff.*")), "Should find matching email address");
+      Assertions.assertTrue(queryOutput.getRecords().stream().allMatch(r -> r.getValueString("email").matches(".*kelkhoff.*")), "Should find matching email address");
    }
 
 
@@ -301,7 +299,7 @@ public class RDBMSQueryActionTest extends RDBMSActionTest
       );
       QueryOutput queryOutput = new RDBMSQueryAction().execute(queryInput);
       assertEquals(1, queryOutput.getRecords().size(), "Expected # of rows");
-      assertTrue(queryOutput.getRecords().stream().allMatch(r -> r.getValueString("email").matches(".*kelkhoff.*")), "Should find matching email address");
+      Assertions.assertTrue(queryOutput.getRecords().stream().allMatch(r -> r.getValueString("email").matches(".*kelkhoff.*")), "Should find matching email address");
    }
 
 
@@ -321,7 +319,7 @@ public class RDBMSQueryActionTest extends RDBMSActionTest
       );
       QueryOutput queryOutput = new RDBMSQueryAction().execute(queryInput);
       assertEquals(4, queryOutput.getRecords().size(), "Expected # of rows");
-      assertTrue(queryOutput.getRecords().stream().noneMatch(r -> r.getValueString("email").matches(".*kelkhoff.*")), "Should find matching email address");
+      Assertions.assertTrue(queryOutput.getRecords().stream().noneMatch(r -> r.getValueString("email").matches(".*kelkhoff.*")), "Should find matching email address");
    }
 
 
@@ -341,7 +339,7 @@ public class RDBMSQueryActionTest extends RDBMSActionTest
       );
       QueryOutput queryOutput = new RDBMSQueryAction().execute(queryInput);
       assertEquals(1, queryOutput.getRecords().size(), "Expected # of rows");
-      assertTrue(queryOutput.getRecords().stream().allMatch(r -> r.getValueString("email").matches(".*gmail.com")), "Should find matching email address");
+      Assertions.assertTrue(queryOutput.getRecords().stream().allMatch(r -> r.getValueString("email").matches(".*gmail.com")), "Should find matching email address");
    }
 
 
@@ -361,7 +359,7 @@ public class RDBMSQueryActionTest extends RDBMSActionTest
       );
       QueryOutput queryOutput = new RDBMSQueryAction().execute(queryInput);
       assertEquals(4, queryOutput.getRecords().size(), "Expected # of rows");
-      assertTrue(queryOutput.getRecords().stream().noneMatch(r -> r.getValueString("email").matches("darin.*")), "Should find matching email address");
+      Assertions.assertTrue(queryOutput.getRecords().stream().noneMatch(r -> r.getValueString("email").matches("darin.*")), "Should find matching email address");
    }
 
 
@@ -381,7 +379,7 @@ public class RDBMSQueryActionTest extends RDBMSActionTest
       );
       QueryOutput queryOutput = new RDBMSQueryAction().execute(queryInput);
       assertEquals(4, queryOutput.getRecords().size(), "Expected # of rows");
-      assertTrue(queryOutput.getRecords().stream().noneMatch(r -> r.getValueString("email").matches(".*kelkhoff.*")), "Should find matching email address");
+      Assertions.assertTrue(queryOutput.getRecords().stream().noneMatch(r -> r.getValueString("email").matches(".*kelkhoff.*")), "Should find matching email address");
    }
 
 
@@ -401,7 +399,7 @@ public class RDBMSQueryActionTest extends RDBMSActionTest
       );
       QueryOutput queryOutput = new RDBMSQueryAction().execute(queryInput);
       assertEquals(4, queryOutput.getRecords().size(), "Expected # of rows");
-      assertTrue(queryOutput.getRecords().stream().noneMatch(r -> r.getValueString("email").matches(".*gmail.com")), "Should find matching email address");
+      Assertions.assertTrue(queryOutput.getRecords().stream().noneMatch(r -> r.getValueString("email").matches(".*gmail.com")), "Should find matching email address");
    }
 
 
@@ -421,7 +419,7 @@ public class RDBMSQueryActionTest extends RDBMSActionTest
       );
       QueryOutput queryOutput = new RDBMSQueryAction().execute(queryInput);
       assertEquals(2, queryOutput.getRecords().size(), "Expected # of rows");
-      assertTrue(queryOutput.getRecords().stream().allMatch(r -> r.getValueInteger("id").equals(1) || r.getValueInteger("id").equals(2)), "Should find expected ids");
+      Assertions.assertTrue(queryOutput.getRecords().stream().allMatch(r -> r.getValueInteger("id").equals(1) || r.getValueInteger("id").equals(2)), "Should find expected ids");
    }
 
 
@@ -441,7 +439,7 @@ public class RDBMSQueryActionTest extends RDBMSActionTest
       );
       QueryOutput queryOutput = new RDBMSQueryAction().execute(queryInput);
       assertEquals(2, queryOutput.getRecords().size(), "Expected # of rows");
-      assertTrue(queryOutput.getRecords().stream().allMatch(r -> r.getValueInteger("id").equals(1) || r.getValueInteger("id").equals(2)), "Should find expected ids");
+      Assertions.assertTrue(queryOutput.getRecords().stream().allMatch(r -> r.getValueInteger("id").equals(1) || r.getValueInteger("id").equals(2)), "Should find expected ids");
    }
 
 
@@ -461,7 +459,7 @@ public class RDBMSQueryActionTest extends RDBMSActionTest
       );
       QueryOutput queryOutput = new RDBMSQueryAction().execute(queryInput);
       assertEquals(2, queryOutput.getRecords().size(), "Expected # of rows");
-      assertTrue(queryOutput.getRecords().stream().allMatch(r -> r.getValueInteger("id").equals(4) || r.getValueInteger("id").equals(5)), "Should find expected ids");
+      Assertions.assertTrue(queryOutput.getRecords().stream().allMatch(r -> r.getValueInteger("id").equals(4) || r.getValueInteger("id").equals(5)), "Should find expected ids");
    }
 
 
@@ -481,7 +479,7 @@ public class RDBMSQueryActionTest extends RDBMSActionTest
       );
       QueryOutput queryOutput = new RDBMSQueryAction().execute(queryInput);
       assertEquals(2, queryOutput.getRecords().size(), "Expected # of rows");
-      assertTrue(queryOutput.getRecords().stream().allMatch(r -> r.getValueInteger("id").equals(4) || r.getValueInteger("id").equals(5)), "Should find expected ids");
+      Assertions.assertTrue(queryOutput.getRecords().stream().allMatch(r -> r.getValueInteger("id").equals(4) || r.getValueInteger("id").equals(5)), "Should find expected ids");
    }
 
 
@@ -500,7 +498,7 @@ public class RDBMSQueryActionTest extends RDBMSActionTest
          ));
       QueryOutput queryOutput = new RDBMSQueryAction().execute(queryInput);
       assertEquals(1, queryOutput.getRecords().size(), "Expected # of rows");
-      assertTrue(queryOutput.getRecords().stream().allMatch(r -> r.getValue("birthDate") == null), "Should find expected row");
+      Assertions.assertTrue(queryOutput.getRecords().stream().allMatch(r -> r.getValue("birthDate") == null), "Should find expected row");
    }
 
 
@@ -519,7 +517,7 @@ public class RDBMSQueryActionTest extends RDBMSActionTest
          ));
       QueryOutput queryOutput = new RDBMSQueryAction().execute(queryInput);
       assertEquals(5, queryOutput.getRecords().size(), "Expected # of rows");
-      assertTrue(queryOutput.getRecords().stream().allMatch(r -> r.getValue("firstName") != null), "Should find expected rows");
+      Assertions.assertTrue(queryOutput.getRecords().stream().allMatch(r -> r.getValue("firstName") != null), "Should find expected rows");
    }
 
 
@@ -539,7 +537,7 @@ public class RDBMSQueryActionTest extends RDBMSActionTest
          ));
       QueryOutput queryOutput = new RDBMSQueryAction().execute(queryInput);
       assertEquals(3, queryOutput.getRecords().size(), "Expected # of rows");
-      assertTrue(queryOutput.getRecords().stream().allMatch(r -> r.getValueInteger("id").equals(2) || r.getValueInteger("id").equals(3) || r.getValueInteger("id").equals(4)), "Should find expected ids");
+      Assertions.assertTrue(queryOutput.getRecords().stream().allMatch(r -> r.getValueInteger("id").equals(2) || r.getValueInteger("id").equals(3) || r.getValueInteger("id").equals(4)), "Should find expected ids");
    }
 
 
@@ -559,7 +557,7 @@ public class RDBMSQueryActionTest extends RDBMSActionTest
          ));
       QueryOutput queryOutput = new RDBMSQueryAction().execute(queryInput);
       assertEquals(2, queryOutput.getRecords().size(), "Expected # of rows");
-      assertTrue(queryOutput.getRecords().stream().allMatch(r -> r.getValueInteger("id").equals(1) || r.getValueInteger("id").equals(5)), "Should find expected ids");
+      Assertions.assertTrue(queryOutput.getRecords().stream().allMatch(r -> r.getValueInteger("id").equals(1) || r.getValueInteger("id").equals(5)), "Should find expected ids");
    }
 
 
@@ -585,7 +583,7 @@ public class RDBMSQueryActionTest extends RDBMSActionTest
             .withCriteria(new QFilterCriteria().withFieldName("birthDate").withOperator(QCriteriaOperator.LESS_THAN).withValues(List.of(new Now()))));
          QueryOutput queryOutput = new RDBMSQueryAction().execute(queryInput);
          assertEquals(1, queryOutput.getRecords().size(), "Expected # of rows");
-         assertTrue(queryOutput.getRecords().stream().anyMatch(r -> r.getValue("firstName").equals("past")), "Should find expected row");
+         Assertions.assertTrue(queryOutput.getRecords().stream().anyMatch(r -> r.getValue("firstName").equals("past")), "Should find expected row");
       }
 
       {
@@ -595,7 +593,7 @@ public class RDBMSQueryActionTest extends RDBMSActionTest
             .withCriteria(new QFilterCriteria().withFieldName("birthDate").withOperator(QCriteriaOperator.LESS_THAN).withValues(List.of(NowWithOffset.plus(2, ChronoUnit.DAYS)))));
          QueryOutput queryOutput = new RDBMSQueryAction().execute(queryInput);
          assertEquals(1, queryOutput.getRecords().size(), "Expected # of rows");
-         assertTrue(queryOutput.getRecords().stream().anyMatch(r -> r.getValue("firstName").equals("past")), "Should find expected row");
+         Assertions.assertTrue(queryOutput.getRecords().stream().anyMatch(r -> r.getValue("firstName").equals("past")), "Should find expected row");
       }
 
       {
@@ -605,8 +603,8 @@ public class RDBMSQueryActionTest extends RDBMSActionTest
             .withCriteria(new QFilterCriteria().withFieldName("birthDate").withOperator(QCriteriaOperator.GREATER_THAN).withValues(List.of(NowWithOffset.minus(5, ChronoUnit.DAYS)))));
          QueryOutput queryOutput = new RDBMSQueryAction().execute(queryInput);
          assertEquals(2, queryOutput.getRecords().size(), "Expected # of rows");
-         assertTrue(queryOutput.getRecords().stream().anyMatch(r -> r.getValue("firstName").equals("past")), "Should find expected row");
-         assertTrue(queryOutput.getRecords().stream().anyMatch(r -> r.getValue("firstName").equals("future")), "Should find expected row");
+         Assertions.assertTrue(queryOutput.getRecords().stream().anyMatch(r -> r.getValue("firstName").equals("past")), "Should find expected row");
+         Assertions.assertTrue(queryOutput.getRecords().stream().anyMatch(r -> r.getValue("firstName").equals("future")), "Should find expected row");
       }
    }
 
@@ -1007,36 +1005,7 @@ public class RDBMSQueryActionTest extends RDBMSActionTest
       queryInput.setShouldFetchHeavyFields(true);
       records = new QueryAction().execute(queryInput).getRecords();
       assertThat(records).describedAs("Some records should have the heavy homeTown field set when heavies are requested").anyMatch(r -> r.getValue("homeTown") != null);
-   }
 
-
-
-   /*******************************************************************************
-    **
-    *******************************************************************************/
-   @Test
-   void testFieldNamesToInclude() throws QException
-   {
-      QQueryFilter  filter     = new QQueryFilter().withCriteria("id", QCriteriaOperator.EQUALS, 1);
-      QueryInput    queryInput = new QueryInput(TestUtils.TABLE_NAME_PERSON).withFilter(filter);
-
-      QRecord record = new QueryAction().execute(queryInput.withFieldNamesToInclude(null)).getRecords().get(0);
-      assertTrue(record.getValues().containsKey("id"));
-      assertTrue(record.getValues().containsKey("firstName"));
-      assertTrue(record.getValues().containsKey("createDate"));
-      assertEquals(QContext.getQInstance().getTable(TestUtils.TABLE_NAME_PERSON).getFields().size(), record.getValues().size());
-
-      record = new QueryAction().execute(queryInput.withFieldNamesToInclude(Set.of("id", "firstName"))).getRecords().get(0);
-      assertTrue(record.getValues().containsKey("id"));
-      assertTrue(record.getValues().containsKey("firstName"));
-      assertFalse(record.getValues().containsKey("createDate"));
-      assertEquals(2, record.getValues().size());
-
-      record = new QueryAction().execute(queryInput.withFieldNamesToInclude(Set.of("homeTown"))).getRecords().get(0);
-      assertFalse(record.getValues().containsKey("id"));
-      assertFalse(record.getValues().containsKey("firstName"));
-      assertFalse(record.getValues().containsKey("createDate"));
-      assertEquals(1, record.getValues().size());
    }
 
 }

qqq-middleware-api/src/main/java/com/kingsrook/qqq/api/model/metadata/processes/ApiProcessInputFieldsContainer.java

@@ -43,6 +43,32 @@ public class ApiProcessInputFieldsContainer
 
 
 
+   /***************************************************************************
+    ** factory method to build one of these containers using all of the input fields
+    ** in a process
+    ***************************************************************************/
+   public static ApiProcessInputFieldsContainer forAllInputFields(QProcessMetaData process)
+   {
+      return forFields(process.getInputFields());
+   }
+
+
+
+   /***************************************************************************
+    ** factory method to build one of these containers using a list of fields.
+    ***************************************************************************/
+   public static ApiProcessInputFieldsContainer forFields(List<QFieldMetaData> fields)
+   {
+      ApiProcessInputFieldsContainer container = new ApiProcessInputFieldsContainer();
+      for(QFieldMetaData inputField : CollectionUtils.nonNullList(fields))
+      {
+         container.withField(inputField);
+      }
+      return (container);
+   }
+
+
+
    /*******************************************************************************
     ** find all input fields in frontend steps of the process, and add them as fields
     ** in this container.

qqq-middleware-api/src/main/java/com/kingsrook/qqq/api/model/metadata/processes/ApiProcessSummaryListOutput.java

@@ -48,7 +48,9 @@ import org.eclipse.jetty.http.HttpStatus;
 
 
 /*******************************************************************************
- **
+ ** For a process that puts "processResults" in its output (as a list of
+ ** ProcessSummaryLineInterface objects) - this class converts such an object
+ ** to a suitable ApiProcessOutput.
  *******************************************************************************/
 public class ApiProcessSummaryListOutput implements ApiProcessOutputInterface
 {
@@ -143,22 +145,31 @@ public class ApiProcessSummaryListOutput implements ApiProcessOutputInterface
          {
             if(processSummaryLineInterface instanceof ProcessSummaryLine processSummaryLine)
             {
-               processSummaryLine.setCount(1);
-               processSummaryLine.pickMessage(true);
-
                List<Serializable> primaryKeys = processSummaryLine.getPrimaryKeys();
                if(CollectionUtils.nullSafeHasContents(primaryKeys))
                {
+                  ////////////////////////////////////////////////////////////////////////////
+                  // if there are primary keys in the line, then we'll loop over those, and //
+                  // output an object in the API output for each one - and we'll make the   //
+                  // line appear to be a singular-past-tense line about that individual key //
+                  ////////////////////////////////////////////////////////////////////////////
+                  processSummaryLine.setCount(1);
+                  processSummaryLine.pickMessage(true);
+
                   for(Serializable primaryKey : primaryKeys)
                   {
-                     HashMap<String, Serializable> map = toMap(processSummaryLine);
+                     HashMap<String, Serializable> map = toMap(processSummaryLine, false);
                      map.put("id", primaryKey);
                      apiOutput.add(map);
                   }
                }
                else
                {
-                  apiOutput.add(toMap(processSummaryLine));
+                  //////////////////////////////////////////////////////////////////////////
+                  // otherwise, handle a line without pkeys as a single output map/object //
+                  //////////////////////////////////////////////////////////////////////////
+                  HashMap<String, Serializable> map = toMap(processSummaryLine, true);
+                  apiOutput.add(map);
                }
             }
             else if(processSummaryLineInterface instanceof ProcessSummaryRecordLink processSummaryRecordLink)
@@ -219,12 +230,19 @@ public class ApiProcessSummaryListOutput implements ApiProcessOutputInterface
    /*******************************************************************************
     **
     *******************************************************************************/
-   private static HashMap<String, Serializable> toMap(ProcessSummaryLine processSummaryLine)
+   private static HashMap<String, Serializable> toMap(ProcessSummaryLine processSummaryLine, boolean tryToIncludeCount)
    {
       HashMap<String, Serializable> map = initResultMapForProcessSummaryLine(processSummaryLine);
 
       String messagePrefix = getResultMapMessagePrefix(processSummaryLine);
-      map.put("message", messagePrefix + processSummaryLine.getMessage());
+
+      String messageSuffix = processSummaryLine.getMessage();
+      if(tryToIncludeCount && processSummaryLine.getCount() != null)
+      {
+         messageSuffix = processSummaryLine.getCount() + " " + messageSuffix;
+      }
+
+      map.put("message", messagePrefix + messageSuffix);
 
       return (map);
    }