CE-1328 Add job status messages

hotfix: allow basepull override values from jobs

docs/actions/QueryAction.adoc

@@ -33,9 +33,6 @@ If the {link-table} has a `POST_QUERY_CUSTOMIZER` defined, then after records ar
 * `table` - *String, Required* - Name of the table being queried against.
 * `filter` - *<<QQueryFilter>> object* - Specification for what records should be returned, based on *<<QFilterCriteria>>* objects, and how they should be sorted, based on *<<QFilterOrderBy>>* objects.
 If a `filter` is not given, then all rows in the table will be returned by the query.
-* `skip` - *Integer* - Optional number of records to be skipped at the beginning of the result set.
-e.g., for implementing pagination.
-* `limit` - *Integer* - Optional maximum number of records to be returned by the query.
 * `transaction` - *QBackendTransaction object* - Optional transaction object.
 ** Behavior for this object is backend-dependant.
 In an RDBMS backend, this object is generally needed if you want your query to see data that may have been modified within the same transaction.
@@ -55,6 +52,14 @@ But if running a query to provide data as part of a process, then this can gener
 * `shouldMaskPassword` - *boolean, default: true* - Controls whether or not fields with `type` = `PASSWORD` should be masked, or if their actual values should be returned.
 * `queryJoins` - *List of <<QueryJoin>> objects* - Optional list of tables to be joined with the main table being queried.
 See QueryJoin below for further details.
+* `fieldNamesToInclude` - *Set of String* - Optional set of field names to be included in the records.
+** Fields from a queryJoin must be prefixed by the join table's name or alias, and a period.
+Field names from the table being queried should not have any sort of prefix.
+** A `null` set here (default) means to include all fields from the table and any queryJoins set as select=true.
+** An empty set will cause an error, as well any unrecognized field names.
+** `QueryAction` will validate the set of field names, and throw an exception if any unrecognized names are given.
+** _Note that this is an optional feature, which some backend modules may not implement.
+Meaning, they would always return all fields._
 
 ==== QQueryFilter
 A key component of *<<QueryInput>>*, a *QQueryFilter* defines both what records should be included in a query's results (e.g., an SQL `WHERE`), as well as how those results should be sorted (SQL `ORDER BY`).
@@ -68,6 +73,9 @@ In general, multiple *orderBys* can be given (depending on backend implementatio
 ** Each *subFilter* can include its own additional *subFilters*.
 ** Each *subFilter* can specify a different *booleanOperator*.
 ** For example, consider the following *QQueryFilter*, that uses two *subFilters*, and a mix of *booleanOperators*
+* `skip` - *Integer* - Optional number of records to be skipped at the beginning of the result set.
+e.g., for implementing pagination.
+* `limit` - *Integer* - Optional maximum number of records to be returned by the query.
 
 [source,java]
 ----

pom.xml

@@ -46,7 +46,7 @@
    </modules>
 
    <properties>
-      <revision>0.21.0-SNAPSHOT</revision>
+      <revision>0.23.0-SNAPSHOT</revision>
 
       <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
       <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/audits/AuditAction.java

@@ -225,7 +225,13 @@ public class AuditAction extends AbstractQActionFunction<AuditInput, AuditOutput
                {
                   if(auditSingleInput.getSecurityKeyValues() == null || !auditSingleInput.getSecurityKeyValues().containsKey(recordSecurityLock.getSecurityKeyType()))
                   {
-                     throw (new QException("Missing securityKeyValue [" + recordSecurityLock.getSecurityKeyType() + "] in audit request for table " + auditSingleInput.getAuditTableName()));
+                     ///////////////////////////////////////////////////////
+                     // originally, this case threw...                    //
+                     // but i think it's better to record the audit, just //
+                     // missing its security key value, then to fail...   //
+                     ///////////////////////////////////////////////////////
+                     // throw (new QException("Missing securityKeyValue [" + recordSecurityLock.getSecurityKeyType() + "] in audit request for table " + auditSingleInput.getAuditTableName()));
+                     LOG.info("Missing securityKeyValue in audit request", logPair("table", auditSingleInput.getAuditTableName()), logPair("securityKey", recordSecurityLock.getSecurityKeyType()));
                   }
                }
 
@@ -272,7 +278,7 @@ public class AuditAction extends AbstractQActionFunction<AuditInput, AuditOutput
             List<QRecord> auditDetailRecords = new ArrayList<>();
             for(AuditSingleInput auditSingleInput : CollectionUtils.nonNullList(input.getAuditSingleInputList()))
             {
-               Integer auditId = insertOutput.getRecords().get(i++).getValueInteger("id");
+               Long auditId = insertOutput.getRecords().get(i++).getValueLong("id");
                if(auditId == null)
                {
                   LOG.warn("Missing an id for inserted audit - so won't be able to store its child details...");

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/RecordCustomizerUtilityInterface.java

@@ -24,10 +24,12 @@ package com.kingsrook.qqq.backend.core.actions.customizers;
 
 import java.io.Serializable;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 import java.util.Objects;
 import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.logging.QLogger;
+import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
@@ -143,4 +145,19 @@ public interface RecordCustomizerUtilityInterface
       }
    }
 
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   default Map<Serializable, QRecord> getOldRecordMap(List<QRecord> oldRecordList, UpdateInput updateInput)
+   {
+      Map<Serializable, QRecord> oldRecordMap = new HashMap<>();
+      for(QRecord qRecord : oldRecordList)
+      {
+         oldRecordMap.put(qRecord.getValue(updateInput.getTable().getPrimaryKeyField()), qRecord);
+      }
+
+      return (oldRecordMap);
+   }
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/processes/RunProcessAction.java

@@ -79,6 +79,7 @@ public class RunProcessAction
 {
    private static final QLogger LOG = QLogger.getLogger(RunProcessAction.class);
 
+   public static final String BASEPULL_KEY_VALUE        = "basepullKeyValue";
    public static final String BASEPULL_THIS_RUNTIME_KEY = "basepullThisRuntimeKey";
    public static final String BASEPULL_LAST_RUNTIME_KEY = "basepullLastRuntimeKey";
    public static final String BASEPULL_TIMESTAMP_FIELD  = "basepullTimestampField";
@@ -517,9 +518,13 @@ public class RunProcessAction
    /*******************************************************************************
     **
     *******************************************************************************/
-   protected String determineBasepullKeyValue(QProcessMetaData process, BasepullConfiguration basepullConfiguration) throws QException
+   protected String determineBasepullKeyValue(QProcessMetaData process, RunProcessInput runProcessInput, BasepullConfiguration basepullConfiguration) throws QException
    {
       String basepullKeyValue = (basepullConfiguration.getKeyValue() != null) ? basepullConfiguration.getKeyValue() : process.getName();
+      if(runProcessInput.getValueString(BASEPULL_KEY_VALUE) != null)
+      {
+         basepullKeyValue = runProcessInput.getValueString(BASEPULL_KEY_VALUE);
+      }
 
       //////////////////////////////////////////////////////////////////////////////////////////////////////////////////
       // if process specifies that it uses variants, look for that data in the session and append to our basepull key //
@@ -551,7 +556,7 @@ public class RunProcessAction
       String basepullTableName            = basepullConfiguration.getTableName();
       String basepullKeyFieldName         = basepullConfiguration.getKeyField();
       String basepullLastRunTimeFieldName = basepullConfiguration.getLastRunTimeFieldName();
-      String basepullKeyValue             = determineBasepullKeyValue(process, basepullConfiguration);
+      String basepullKeyValue             = determineBasepullKeyValue(process, runProcessInput, basepullConfiguration);
 
       ///////////////////////////////////////
       // get the stored basepull timestamp //
@@ -631,7 +636,7 @@ public class RunProcessAction
       String  basepullKeyFieldName                 = basepullConfiguration.getKeyField();
       String  basepullLastRunTimeFieldName         = basepullConfiguration.getLastRunTimeFieldName();
       Integer basepullHoursBackForInitialTimestamp = basepullConfiguration.getHoursBackForInitialTimestamp();
-      String  basepullKeyValue                     = determineBasepullKeyValue(process, basepullConfiguration);
+      String  basepullKeyValue                     = determineBasepullKeyValue(process, runProcessInput, basepullConfiguration);
 
       ///////////////////////////////////////
       // get the stored basepull timestamp //

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/GenerateReportAction.java

@@ -42,6 +42,7 @@ import com.kingsrook.qqq.backend.core.actions.AbstractQActionFunction;
 import com.kingsrook.qqq.backend.core.actions.async.AsyncRecordPipeLoop;
 import com.kingsrook.qqq.backend.core.actions.customizers.QCodeLoader;
 import com.kingsrook.qqq.backend.core.actions.reporting.customizers.DataSourceQueryInputCustomizer;
+import com.kingsrook.qqq.backend.core.actions.reporting.customizers.ReportCustomRecordSourceInterface;
 import com.kingsrook.qqq.backend.core.actions.reporting.customizers.ReportViewCustomizer;
 import com.kingsrook.qqq.backend.core.actions.tables.CountAction;
 import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
@@ -62,6 +63,8 @@ import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportOutput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.QueryHint;
 import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountOutput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.CriteriaMissingInputValueBehavior;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.FilterUseCase;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.JoinsContext;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
@@ -302,10 +305,19 @@ public class GenerateReportAction extends AbstractQActionFunction<ReportInput, R
       JoinsContext joinsContext = null;
       if(dataSource != null)
       {
+         ///////////////////////////////////////////////////////////////////////////////////////
+         // count records, if applicable, from the data source - for populating into the      //
+         // countByDataSource map, as well as for checking if too many rows (e.g., for excel) //
+         ///////////////////////////////////////////////////////////////////////////////////////
+         countDataSourceRecords(reportInput, dataSource, reportFormat);
+
+         ///////////////////////////////////////////////////////////////////////////////////////////
+         // if there's a source table, set up a joins context, to use below for looking up fields //
+         ///////////////////////////////////////////////////////////////////////////////////////////
          if(StringUtils.hasContent(dataSource.getSourceTable()))
          {
-            joinsContext = new JoinsContext(QContext.getQInstance(), dataSource.getSourceTable(), cloneDataSourceQueryJoins(dataSource), dataSource.getQueryFilter() == null ? null : dataSource.getQueryFilter().clone());
-            countDataSourceRecords(reportInput, dataSource, reportFormat);
+            QQueryFilter queryFilter = dataSource.getQueryFilter() == null ? new QQueryFilter() : dataSource.getQueryFilter().clone();
+            joinsContext = new JoinsContext(QContext.getQInstance(), dataSource.getSourceTable(), dataSource.getQueryJoins(), queryFilter);
          }
       }
 
@@ -329,6 +341,7 @@ public class GenerateReportAction extends AbstractQActionFunction<ReportInput, R
             field.setName(column.getName());
             if(StringUtils.hasContent(column.getLabel()))
             {
+
                field.setLabel(column.getLabel());
             }
             fields.add(field);
@@ -346,23 +359,33 @@ public class GenerateReportAction extends AbstractQActionFunction<ReportInput, R
     *******************************************************************************/
    private void countDataSourceRecords(ReportInput reportInput, QReportDataSource dataSource, ReportFormat reportFormat) throws QException
    {
-      QQueryFilter queryFilter = dataSource.getQueryFilter() == null ? new QQueryFilter() : dataSource.getQueryFilter().clone();
-      setInputValuesInQueryFilter(reportInput, queryFilter);
-
-      CountInput countInput = new CountInput();
-      countInput.setTableName(dataSource.getSourceTable());
-      countInput.setFilter(queryFilter);
-      countInput.setQueryJoins(cloneDataSourceQueryJoins(dataSource));
-      CountOutput countOutput = new CountAction().execute(countInput);
-
-      if(countOutput.getCount() != null)
+      Integer count = null;
+      if(dataSource.getCustomRecordSource() != null)
       {
-         countByDataSource.put(dataSource.getName(), countOutput.getCount());
+         // todo - add `count` method to interface?
+      }
+      else if(StringUtils.hasContent(dataSource.getSourceTable()))
+      {
+         QQueryFilter queryFilter = dataSource.getQueryFilter() == null ? new QQueryFilter() : dataSource.getQueryFilter().clone();
+         setInputValuesInQueryFilter(reportInput, queryFilter);
 
-         if(reportFormat.getMaxRows() != null && countOutput.getCount() > reportFormat.getMaxRows())
+         CountInput countInput = new CountInput();
+         countInput.setTableName(dataSource.getSourceTable());
+         countInput.setFilter(queryFilter);
+         countInput.setQueryJoins(cloneDataSourceQueryJoins(dataSource));
+         CountOutput countOutput = new CountAction().execute(countInput);
+
+         count = countOutput.getCount();
+      }
+
+      if(count != null)
+      {
+         countByDataSource.put(dataSource.getName(), count);
+
+         if(reportFormat.getMaxRows() != null && count > reportFormat.getMaxRows())
          {
             throw (new QUserFacingException("The requested report would include more rows ("
-               + String.format("%,d", countOutput.getCount()) + ") than the maximum allowed ("
+               + String.format("%,d", count) + ") than the maximum allowed ("
                + String.format("%,d", reportFormat.getMaxRows()) + ") for the selected file format (" + reportFormat + ")."));
          }
       }
@@ -423,13 +446,19 @@ public class GenerateReportAction extends AbstractQActionFunction<ReportInput, R
       String        tableLabel    = ObjectUtils.tryElse(() -> QContext.getQInstance().getTable(dataSource.getSourceTable()).getLabel(), Objects.requireNonNullElse(dataSource.getSourceTable(), ""));
       AtomicInteger consumedCount = new AtomicInteger(0);
 
-      /////////////////////////////////////////////////////////////////
-      // run a record pipe loop, over the query for this data source //
-      /////////////////////////////////////////////////////////////////
+      /////////////////////////////////////////////////////////////////////////////////////////////////
+      // run a record pipe loop, over the query (or other data-supplier/source) for this data source //
+      /////////////////////////////////////////////////////////////////////////////////////////////////
       RecordPipe recordPipe = new BufferedRecordPipe(1000);
       new AsyncRecordPipeLoop().run("Report[" + reportInput.getReportName() + "]", null, recordPipe, (callback) ->
       {
-         if(dataSource.getSourceTable() != null)
+         if(dataSource.getCustomRecordSource() != null)
+         {
+            ReportCustomRecordSourceInterface recordSource = QCodeLoader.getAdHoc(ReportCustomRecordSourceInterface.class, dataSource.getCustomRecordSource());
+            recordSource.execute(reportInput, dataSource, recordPipe);
+            return (true);
+         }
+         else if(dataSource.getSourceTable() != null)
          {
             QQueryFilter queryFilter = dataSource.getQueryFilter() == null ? new QQueryFilter() : dataSource.getQueryFilter().clone();
             setInputValuesInQueryFilter(reportInput, queryFilter);
@@ -587,7 +616,56 @@ public class GenerateReportAction extends AbstractQActionFunction<ReportInput, R
          return;
       }
 
-      queryFilter.interpretValues(reportInput.getInputValues());
+      //////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      // for reports defined in meta-data, the established rule is, that missing input variable values are discarded. //
+      // but for non-meta-data reports (e.g., user-saved), we expect an exception for missing values.                 //
+      // so, set those use-cases up.                                                                                  //
+      //////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      FilterUseCase filterUseCase;
+      if(StringUtils.hasContent(reportInput.getReportName()) && QContext.getQInstance().getReport(reportInput.getReportName()) != null)
+      {
+         filterUseCase = new ReportFromMetaDataFilterUseCase();
+      }
+      else
+      {
+         filterUseCase = new ReportNotFromMetaDataFilterUseCase();
+      }
+
+      queryFilter.interpretValues(reportInput.getInputValues(), filterUseCase);
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   private static class ReportFromMetaDataFilterUseCase implements FilterUseCase
+   {
+      /***************************************************************************
+       **
+       ***************************************************************************/
+      @Override
+      public CriteriaMissingInputValueBehavior getDefaultCriteriaMissingInputValueBehavior()
+      {
+         return CriteriaMissingInputValueBehavior.REMOVE_FROM_FILTER;
+      }
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   private static class ReportNotFromMetaDataFilterUseCase implements FilterUseCase
+   {
+      /***************************************************************************
+       **
+       ***************************************************************************/
+      @Override
+      public CriteriaMissingInputValueBehavior getDefaultCriteriaMissingInputValueBehavior()
+      {
+         return CriteriaMissingInputValueBehavior.THROW_EXCEPTION;
+      }
    }
 
 

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/customizers/ReportCustomRecordSourceInterface.java

@@ -19,27 +19,25 @@
  * along with this program.  If not, see <https://www.gnu.org/licenses/>.
  */
 
-package com.kingsrook.qqq.backend.module.api.processes;
+package com.kingsrook.qqq.backend.core.actions.reporting.customizers;
 
 
-import com.kingsrook.qqq.backend.core.model.actions.tables.QueryHint;
-import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
-import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.ExtractViaQueryStep;
+import com.kingsrook.qqq.backend.core.actions.reporting.RecordPipe;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportInput;
+import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportDataSource;
 
 
 /*******************************************************************************
- **
+ ** Interface to be implemented to do a custom source of data for a report
+ ** (instead of just a query against a table).
  *******************************************************************************/
-public class MigrateOutboundAPILogExtractStep extends ExtractViaQueryStep
+public interface ReportCustomRecordSourceInterface
 {
 
-   /*******************************************************************************
-    **
-    *******************************************************************************/
-   @Override
-   protected void customizeInputPreQuery(QueryInput queryInput)
-   {
-      queryInput.withQueryHint(QueryHint.POTENTIALLY_LARGE_NUMBER_OF_RESULTS);
-   }
+   /***************************************************************************
+    ** Given the report input, put records into the pipe, for the report.
+    ***************************************************************************/
+   void execute(ReportInput reportInput, QReportDataSource reportDataSource, RecordPipe recordPipe) throws QException;
 
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/excel/poi/ExcelPoiBasedStreamingExportStreamer.java

@@ -124,10 +124,11 @@ public class ExcelPoiBasedStreamingExportStreamer implements ExportStreamerInter
    private Writer              activeSheetWriter = null;
    private StreamedSheetWriter sheetWriter       = null;
 
-   private QReportView                       currentView      = null;
-   private Map<String, List<QFieldMetaData>> fieldsPerView    = new HashMap<>();
-   private Map<String, Integer>              rowsPerView      = new HashMap<>();
-   private Map<String, String>               labelViewsByName = new HashMap<>();
+   private QReportView                       currentView              = null;
+   private Map<String, List<QFieldMetaData>> fieldsPerView            = new HashMap<>();
+   private Map<String, Integer>              rowsPerView              = new HashMap<>();
+   private Map<String, String>               labelViewsByName         = new HashMap<>();
+   private Map<String, String>               sheetReferenceByViewName = new HashMap<>();
 
 
 
@@ -180,6 +181,7 @@ public class ExcelPoiBasedStreamingExportStreamer implements ExportStreamerInter
             String    sheetReference = sheet.getPackagePart().getPartName().getName().substring(1);
             sheetMapByExcelReference.put(sheetReference, sheet);
             sheetMapByViewName.put(view.getName(), sheet);
+            sheetReferenceByViewName.put(view.getName(), sheetReference);
             sheetCounter++;
          }
 
@@ -446,7 +448,7 @@ public class ExcelPoiBasedStreamingExportStreamer implements ExportStreamerInter
          // - with a new output stream writer    //
          // - and with a SpreadsheetWriter       //
          //////////////////////////////////////////
-         zipOutputStream.putNextEntry(new ZipEntry("xl/worksheets/sheet" + this.sheetIndex++ + ".xml"));
+         zipOutputStream.putNextEntry(new ZipEntry(sheetReferenceByViewName.get(view.getName())));
          activeSheetWriter = new OutputStreamWriter(zipOutputStream);
          sheetWriter = new StreamedSheetWriter(activeSheetWriter);
 

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/QueryAction.java

@@ -26,6 +26,7 @@ import java.io.Serializable;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
+import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
@@ -50,8 +51,10 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperat
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryJoin;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.data.QRecordEntity;
 import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.AdornmentType;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
@@ -64,6 +67,7 @@ import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleDispatcher;
 import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleInterface;
 import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
 import com.kingsrook.qqq.backend.core.utils.ListingHash;
+import com.kingsrook.qqq.backend.core.utils.StringUtils;
 import com.kingsrook.qqq.backend.core.utils.ValueUtils;
 
 
@@ -101,6 +105,8 @@ public class QueryAction
          throw (new QException("A table named [" + queryInput.getTableName() + "] was not found in the active QInstance"));
       }
 
+      validateFieldNamesToInclude(queryInput);
+
       QBackendMetaData backend = queryInput.getBackend();
       postQueryRecordCustomizer = QCodeLoader.getTableCustomizer(table, TableCustomizers.POST_QUERY_RECORD.getRole());
       this.queryInput = queryInput;
@@ -158,6 +164,125 @@ public class QueryAction
 
 
 
+   /***************************************************************************
+    ** if QueryInput contains a set of FieldNamesToInclude, then validate that
+    ** those are known field names in the table being queried, or a selected
+    ** queryJoin.
+    ***************************************************************************/
+   static void validateFieldNamesToInclude(QueryInput queryInput) throws QException
+   {
+      Set<String> fieldNamesToInclude = queryInput.getFieldNamesToInclude();
+      if(fieldNamesToInclude == null)
+      {
+         ////////////////////////////////
+         // null set means select all. //
+         ////////////////////////////////
+         return;
+      }
+
+      if(fieldNamesToInclude.isEmpty())
+      {
+         /////////////////////////////////////
+         // empty set, however, is an error //
+         /////////////////////////////////////
+         throw (new QException("An empty set of fieldNamesToInclude was given as queryInput, which is not allowed."));
+      }
+
+      List<String>                unrecognizedFieldNames = new ArrayList<>();
+      Map<String, QTableMetaData> selectedQueryJoins     = null;
+      for(String fieldName : fieldNamesToInclude)
+      {
+         if(fieldName.contains("."))
+         {
+            ////////////////////////////////////////////////
+            // handle names with dots - fields from joins //
+            ////////////////////////////////////////////////
+            String[] parts = fieldName.split("\\.");
+            if(parts.length != 2)
+            {
+               unrecognizedFieldNames.add(fieldName);
+            }
+            else
+            {
+               String tableOrAlias  = parts[0];
+               String fieldNamePart = parts[1];
+
+               ////////////////////////////////////////////
+               // build map of queryJoins being selected //
+               ////////////////////////////////////////////
+               if(selectedQueryJoins == null)
+               {
+                  selectedQueryJoins = new HashMap<>();
+                  for(QueryJoin queryJoin : CollectionUtils.nonNullList(queryInput.getQueryJoins()))
+                  {
+                     if(queryJoin.getSelect())
+                     {
+                        String         joinTableOrAlias = queryJoin.getJoinTableOrItsAlias();
+                        QTableMetaData joinTable        = QContext.getQInstance().getTable(queryJoin.getJoinTable());
+                        if(joinTable != null)
+                        {
+                           selectedQueryJoins.put(joinTableOrAlias, joinTable);
+                        }
+                     }
+                  }
+               }
+
+               if(!selectedQueryJoins.containsKey(tableOrAlias))
+               {
+                  ///////////////////////////////////////////
+                  // unrecognized tableOrAlias is an error //
+                  ///////////////////////////////////////////
+                  unrecognizedFieldNames.add(fieldName);
+               }
+               else
+               {
+                  QTableMetaData joinTable = selectedQueryJoins.get(tableOrAlias);
+                  if(!joinTable.getFields().containsKey(fieldNamePart))
+                  {
+                     //////////////////////////////////////////////////////////
+                     // unrecognized field within the join table is an error //
+                     //////////////////////////////////////////////////////////
+                     unrecognizedFieldNames.add(fieldName);
+                  }
+               }
+            }
+         }
+         else
+         {
+            ///////////////////////////////////////////////////////////////////////
+            // non-join fields - just ensure field name is in table's fields map //
+            ///////////////////////////////////////////////////////////////////////
+            if(!queryInput.getTable().getFields().containsKey(fieldName))
+            {
+               unrecognizedFieldNames.add(fieldName);
+            }
+         }
+      }
+
+      if(!unrecognizedFieldNames.isEmpty())
+      {
+         throw (new QException("QueryInput contained " + unrecognizedFieldNames.size() + " unrecognized field name" + StringUtils.plural(unrecognizedFieldNames) + ": " + StringUtils.join(",", unrecognizedFieldNames)));
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** shorthand way to call for the most common use-case, when you just want the
+    ** entities to be returned, and you just want to pass in a table name and filter.
+    *******************************************************************************/
+   public static <T extends QRecordEntity> List<T> execute(String tableName, Class<T> entityClass, QQueryFilter filter) throws QException
+   {
+      QueryAction queryAction = new QueryAction();
+      QueryInput  queryInput  = new QueryInput();
+      queryInput.setTableName(tableName);
+      queryInput.setFilter(filter);
+      QueryOutput queryOutput = queryAction.execute(queryInput);
+      return (queryOutput.getRecordEntities(entityClass));
+   }
+
+
+
    /*******************************************************************************
     ** shorthand way to call for the most common use-case, when you just want the
     ** records to be returned, and you just want to pass in a table name and filter.

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/SearchPossibleValueSourceAction.java

@@ -260,9 +260,6 @@ public class SearchPossibleValueSourceAction
          }
       }
 
-      // todo - skip & limit as params
-      queryFilter.setLimit(250);
-
       ///////////////////////////////////////////////////////////////////////////////////////////////////////
       // if given a default filter, make it the 'top level' filter and the one we just created a subfilter //
       ///////////////////////////////////////////////////////////////////////////////////////////////////////
@@ -272,6 +269,9 @@ public class SearchPossibleValueSourceAction
          queryFilter = input.getDefaultQueryFilter();
       }
 
+      // todo - skip & limit as params
+      queryFilter.setLimit(250);
+
       queryFilter.setOrderBys(possibleValueSource.getOrderByFields());
 
       queryInput.setFilter(queryFilter);

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidator.java

@@ -44,6 +44,7 @@ import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizers;
 import com.kingsrook.qqq.backend.core.actions.dashboard.widgets.AbstractWidgetRenderer;
 import com.kingsrook.qqq.backend.core.actions.metadata.JoinGraph;
 import com.kingsrook.qqq.backend.core.actions.processes.BackendStep;
+import com.kingsrook.qqq.backend.core.actions.reporting.customizers.ReportCustomRecordSourceInterface;
 import com.kingsrook.qqq.backend.core.actions.scripts.TestScriptActionInterface;
 import com.kingsrook.qqq.backend.core.actions.values.QCustomPossibleValueProvider;
 import com.kingsrook.qqq.backend.core.exceptions.QInstanceValidationException;
@@ -1660,9 +1661,12 @@ public class QInstanceValidator
 
                   String dataSourceErrorPrefix = "Report " + reportName + " data source " + dataSource.getName() + " ";
 
+                  boolean hasASource = false;
+
                   if(StringUtils.hasContent(dataSource.getSourceTable()))
                   {
-                     assertCondition(dataSource.getStaticDataSupplier() == null, dataSourceErrorPrefix + "has both a sourceTable and a staticDataSupplier (exactly 1 is required).");
+                     hasASource = true;
+                     assertCondition(dataSource.getStaticDataSupplier() == null, dataSourceErrorPrefix + "has both a sourceTable and a staticDataSupplier (not compatible together).");
                      if(assertCondition(qInstance.getTable(dataSource.getSourceTable()) != null, dataSourceErrorPrefix + "source table " + dataSource.getSourceTable() + " is not a table in this instance."))
                      {
                         if(dataSource.getQueryFilter() != null)
@@ -1671,14 +1675,21 @@ public class QInstanceValidator
                         }
                      }
                   }
-                  else if(dataSource.getStaticDataSupplier() != null)
+
+                  if(dataSource.getStaticDataSupplier() != null)
                   {
+                     assertCondition(dataSource.getCustomRecordSource() == null, dataSourceErrorPrefix + "has both a staticDataSupplier and a customRecordSource (not compatible together).");
+                     hasASource = true;
                      validateSimpleCodeReference(dataSourceErrorPrefix, dataSource.getStaticDataSupplier(), Supplier.class);
                   }
-                  else
+
+                  if(dataSource.getCustomRecordSource() != null)
                   {
-                     errors.add(dataSourceErrorPrefix + "does not have a sourceTable or a staticDataSupplier (exactly 1 is required).");
+                     hasASource = true;
+                     validateSimpleCodeReference(dataSourceErrorPrefix, dataSource.getCustomRecordSource(), ReportCustomRecordSourceInterface.class);
                   }
+
+                  assertCondition(hasASource, dataSourceErrorPrefix + "does not have a sourceTable, customRecordSource, or a staticDataSupplier.");
                }
             }
 

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/CriteriaMissingInputValueBehavior.java

@@ -0,0 +1,66 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.model.actions.tables.query;
+
+
+/***************************************************************************
+ ** Possible behaviors for doing interpretValues on a filter, and a criteria
+ ** has a variable value (either as a string-that-looks-like-a-variable,
+ ** as in ${input.foreignId} for a PVS filter, or a FilterVariableExpression),
+ ** and a value for that variable isn't available.
+ **
+ ** Used in conjunction with FilterUseCase and its implementations, e.g.,
+ ** PossibleValueSearchFilterUseCase.
+ ***************************************************************************/
+public enum CriteriaMissingInputValueBehavior
+{
+   //////////////////////////////////////////////////////////////////////
+   // this was the original behavior, before we added this enum.  but, //
+   // it doesn't ever seem entirely valid, and isn't currently used.   //
+   //////////////////////////////////////////////////////////////////////
+   INTERPRET_AS_NULL_VALUE,
+
+   //////////////////////////////////////////////////////////////////////////
+   // make the criteria behave as though it's not in the filter at all.    //
+   // effectively by changing its operator to TRUE, so it always matches.  //
+   // original intended use is for possible-values on  query screens,      //
+   // where a foreign-id isn't present, so we want to show all PV options. //
+   //////////////////////////////////////////////////////////////////////////
+   REMOVE_FROM_FILTER,
+
+   //////////////////////////////////////////////////////////////////////////////////////
+   // make the criteria such that it makes no rows ever match.                         //
+   // e.g., changes it to a FALSE.  I suppose, within an OR, that might                //
+   // not be powerful enough...  but, it solves the immediate use-case in              //
+   // front of us, which is forms, where a PV field should show no values              //
+   // until a foreign key field has a value.                                           //
+   // Note that this use-case used to have the same end-effect by such                 //
+   // variables being interpreted as nulls - but this approach feels more intentional. //
+   //////////////////////////////////////////////////////////////////////////////////////
+   MAKE_NO_MATCHES,
+
+   ///////////////////////////////////////////////////////////////////////////////////////////
+   // throw an exception if a value isn't available.  This is the overall default,          //
+   // and originally was what we did for FilterVariableExpressions, e.g., for saved reports //
+   ///////////////////////////////////////////////////////////////////////////////////////////
+   THROW_EXCEPTION
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/FilterUseCase.java

@@ -19,36 +19,40 @@
  * along with this program.  If not, see <https://www.gnu.org/licenses/>.
  */
 
-package com.kingsrook.qqq.backend.module.api.processes;
-
-
-import com.kingsrook.qqq.backend.core.exceptions.QException;
-import com.kingsrook.qqq.backend.core.logging.QLogger;
-import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
-import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
-import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.LoadViaInsertStep;
+package com.kingsrook.qqq.backend.core.model.actions.tables.query;
 
 
 /*******************************************************************************
- ** store outboundApiLogHeaders and maybe more...
+ ** Interface where we can associate behaviors with various use cases for
+ ** QQueryFilters - the original being, how to handle (in the interpretValues
+ ** method) how to handle missing input values.
+ **
+ ** Includes a default implementation, with a default behavior - which is to
+ ** throw an exception upon missing criteria variable values.
  *******************************************************************************/
-public class MigrateOutboundAPILogLoadStep extends LoadViaInsertStep
+public interface FilterUseCase
 {
-   private static final QLogger LOG = QLogger.getLogger(MigrateOutboundAPILogLoadStep.class);
+   FilterUseCase DEFAULT = new DefaultFilterUseCase();
 
-
-
-   /*******************************************************************************
+   /***************************************************************************
     **
-    *******************************************************************************/
-   @Override
-   public void runOnePage(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException
+    ***************************************************************************/
+   CriteriaMissingInputValueBehavior getDefaultCriteriaMissingInputValueBehavior();
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   class DefaultFilterUseCase implements FilterUseCase
    {
-      super.runOnePage(runBackendStepInput, runBackendStepOutput);
 
-      ///////////////////////////////////////
-      // todo - track what we've migrated? //
-      ///////////////////////////////////////
+      /***************************************************************************
+       **
+       ***************************************************************************/
+      @Override
+      public CriteriaMissingInputValueBehavior getDefaultCriteriaMissingInputValueBehavior()
+      {
+         return CriteriaMissingInputValueBehavior.THROW_EXCEPTION;
+      }
    }
-
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/JoinsContext.java

@@ -82,7 +82,7 @@ public class JoinsContext
    /////////////////////////////////////////////////////////////////////////////
    // we will get a TON of more output if this gets turned up, so be cautious //
    /////////////////////////////////////////////////////////////////////////////
-   private Level logLevel = Level.OFF;
+   private Level logLevel          = Level.OFF;
    private Level logLevelForFilter = Level.OFF;
 
 
@@ -404,6 +404,12 @@ public class JoinsContext
                chainIsInner = false;
             }
 
+            if(hasAllAccessKey(recordSecurityLock))
+            {
+               queryJoin.withType(QueryJoin.Type.LEFT);
+               chainIsInner = false;
+            }
+
             addQueryJoin(queryJoin, "forRecordSecurityLock (non-flipped)", "- ");
             addedQueryJoins.add(queryJoin);
             tmpTable = instance.getTable(join.getRightTable());
@@ -423,6 +429,12 @@ public class JoinsContext
                chainIsInner = false;
             }
 
+            if(hasAllAccessKey(recordSecurityLock))
+            {
+               queryJoin.withType(QueryJoin.Type.LEFT);
+               chainIsInner = false;
+            }
+
             addQueryJoin(queryJoin, "forRecordSecurityLock (flipped)", "- ");
             addedQueryJoins.add(queryJoin);
             tmpTable = instance.getTable(join.getLeftTable());
@@ -456,44 +468,53 @@ public class JoinsContext
 
 
 
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   private boolean hasAllAccessKey(RecordSecurityLock recordSecurityLock)
+   {
+      //////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      // check if the key type has an all-access key, and if so, if it's set to true for the current user/session //
+      //////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      QSecurityKeyType securityKeyType = instance.getSecurityKeyType(recordSecurityLock.getSecurityKeyType());
+      if(StringUtils.hasContent(securityKeyType.getAllAccessKeyName()))
+      {
+         QSession session = QContext.getQSession();
+         if(session.hasSecurityKeyValue(securityKeyType.getAllAccessKeyName(), true, QFieldType.BOOLEAN))
+         {
+            return (true);
+         }
+      }
+
+      return (false);
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/
    private void addSubFilterForRecordSecurityLock(RecordSecurityLock recordSecurityLock, QTableMetaData table, String tableNameOrAlias, boolean isOuter, QueryJoin sourceQueryJoin)
    {
-      QSession session = QContext.getQSession();
-
-      //////////////////////////////////////////////////////////////////////////////////////////////////////////////
-      // check if the key type has an all-access key, and if so, if it's set to true for the current user/session //
-      //////////////////////////////////////////////////////////////////////////////////////////////////////////////
-      QSecurityKeyType securityKeyType  = instance.getSecurityKeyType(recordSecurityLock.getSecurityKeyType());
-      boolean          haveAllAccessKey = false;
-      if(StringUtils.hasContent(securityKeyType.getAllAccessKeyName()))
+      boolean haveAllAccessKey = hasAllAccessKey(recordSecurityLock);
+      if(haveAllAccessKey)
       {
-         //////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-         // if we have all-access on this key, then we don't need a criterion for it (as long as we're in an AND filter) //
-         //////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-         if(session.hasSecurityKeyValue(securityKeyType.getAllAccessKeyName(), true, QFieldType.BOOLEAN))
+         if(sourceQueryJoin != null)
          {
-            haveAllAccessKey = true;
+            ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+            // in case the queryJoin object is re-used between queries, and its security criteria need to be different (!!), reset it //
+            // this can be exposed in tests - maybe not entirely expected in real-world, but seems safe enough                        //
+            ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+            sourceQueryJoin.withSecurityCriteria(new ArrayList<>());
+         }
 
-            if(sourceQueryJoin != null)
-            {
-               ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-               // in case the queryJoin object is re-used between queries, and its security criteria need to be different (!!), reset it //
-               // this can be exposed in tests - maybe not entirely expected in real-world, but seems safe enough                        //
-               ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-               sourceQueryJoin.withSecurityCriteria(new ArrayList<>());
-            }
-
-            ////////////////////////////////////////////////////////////////////////////////////////
-            // if we're in an AND filter, then we don't need a criteria for this lock, so return. //
-            ////////////////////////////////////////////////////////////////////////////////////////
-            boolean inAnAndFilter = securityFilterCursor.getBooleanOperator() == QQueryFilter.BooleanOperator.AND;
-            if(inAnAndFilter)
-            {
-               return;
-            }
+         ////////////////////////////////////////////////////////////////////////////////////////
+         // if we're in an AND filter, then we don't need a criteria for this lock, so return. //
+         ////////////////////////////////////////////////////////////////////////////////////////
+         boolean inAnAndFilter = securityFilterCursor.getBooleanOperator() == QQueryFilter.BooleanOperator.AND;
+         if(inAnAndFilter)
+         {
+            return;
          }
       }
 
@@ -545,7 +566,7 @@ public class JoinsContext
       }
       else
       {
-         List<Serializable> securityKeyValues = session.getSecurityKeyValues(recordSecurityLock.getSecurityKeyType(), type);
+         List<Serializable> securityKeyValues = QContext.getQSession().getSecurityKeyValues(recordSecurityLock.getSecurityKeyType(), type);
          if(CollectionUtils.nullSafeIsEmpty(securityKeyValues))
          {
             ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QQueryFilter.java

@@ -528,8 +528,27 @@ public class QQueryFilter implements Serializable, Cloneable
     ** Note - it may be very important that you call this method on a clone of a
     ** QQueryFilter - e.g., if it's one that defined in metaData, and that we don't
     ** want to be (permanently) changed!!
-    *******************************************************************************/
+    **
+    ** This overload does not take in a FilterUseCase - it uses FilterUseCase.DEFAULT
+    ******************************************************************************/
    public void interpretValues(Map<String, Serializable> inputValues) throws QException
+   {
+      interpretValues(inputValues, FilterUseCase.DEFAULT);
+   }
+
+
+
+   /*******************************************************************************
+    ** Replace any criteria values that look like ${input.XXX} with the value of XXX
+    ** from the supplied inputValues map - where the handling of missing values
+    ** is specified in the inputted FilterUseCase parameter
+    **
+    ** Note - it may be very important that you call this method on a clone of a
+    ** QQueryFilter - e.g., if it's one that defined in metaData, and that we don't
+    ** want to be (permanently) changed!!
+    **
+    *******************************************************************************/
+   public void interpretValues(Map<String, Serializable> inputValues, FilterUseCase useCase) throws QException
    {
       List<Exception> caughtExceptions = new ArrayList<>();
 
@@ -545,6 +564,9 @@ public class QQueryFilter implements Serializable, Cloneable
             {
                try
                {
+                  Serializable interpretedValue = value;
+                  Exception    caughtException  = null;
+
                   if(value instanceof AbstractFilterExpression<?>)
                   {
                      ///////////////////////////////////////////////////////////////////////
@@ -553,17 +575,54 @@ public class QQueryFilter implements Serializable, Cloneable
                      ///////////////////////////////////////////////////////////////////////
                      if(value instanceof FilterVariableExpression filterVariableExpression)
                      {
-                        newValues.add(filterVariableExpression.evaluateInputValues(inputValues));
-                     }
-                     else
-                     {
-                        newValues.add(value);
+                        try
+                        {
+                           interpretedValue = filterVariableExpression.evaluateInputValues(inputValues);
+                        }
+                        catch(Exception e)
+                        {
+                           caughtException = e;
+                           interpretedValue = InputNotFound.instance;
+                        }
+                     }
+                  }
+                  else
+                  {
+                     /////////////////////////////////////////////////////////////////////////////////////////////////////////
+                     // for non-expressions, cast the value to a string, and see if it can be resolved a variable.          //
+                     // there are 3 possible cases here:                                                                    //
+                     // 1: it doesn't look like a variable, so it just comes back as a string version of whatever went in.  //
+                     // 2: it was resolved from a variable to a value, e.g., ${input.someVar} => someValue                  //
+                     // 3: it looked like a variable, but no value for that variable was present in the interpreter's value //
+                     // map - so we'll get back the InputNotFound.instance.                                                 //
+                     /////////////////////////////////////////////////////////////////////////////////////////////////////////
+                     String valueAsString = ValueUtils.getValueAsString(value);
+                     interpretedValue = variableInterpreter.interpretForObject(valueAsString, InputNotFound.instance);
+                  }
+
+                  //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+                  // if interpreting a value returned the not-found value, or an empty string,                                            //
+                  // then decide how to handle the missing value, based on the use-case input                                             //
+                  // Note: questionable, using "" here, but that's what reality is passing a lot for cases we want to treat as missing... //
+                  //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+                  if(interpretedValue == InputNotFound.instance || "".equals(interpretedValue))
+                  {
+                     CriteriaMissingInputValueBehavior missingInputValueBehavior = getMissingInputValueBehavior(useCase);
+
+                     switch(missingInputValueBehavior)
+                     {
+                        case REMOVE_FROM_FILTER -> criterion.setOperator(QCriteriaOperator.TRUE);
+                        case MAKE_NO_MATCHES -> criterion.setOperator(QCriteriaOperator.FALSE);
+                        case INTERPRET_AS_NULL_VALUE -> newValues.add(null);
+
+                        /////////////////////////////////////////////////
+                        // handle case in the default: THROW_EXCEPTION //
+                        /////////////////////////////////////////////////
+                        default -> throw (Objects.requireNonNullElseGet(caughtException, () -> new QUserFacingException("Missing value for criteria on field: " + criterion.getFieldName())));
                      }
                   }
                   else
                   {
-                     String       valueAsString    = ValueUtils.getValueAsString(value);
-                     Serializable interpretedValue = variableInterpreter.interpretForObject(valueAsString);
                      newValues.add(interpretedValue);
                   }
                }
@@ -586,6 +645,44 @@ public class QQueryFilter implements Serializable, Cloneable
 
 
 
+   /***************************************************************************
+    ** Note:  in the original build of this, it felt like we *might* want to be
+    ** able to specify these behaviors at the individual criteria level, where
+    ** the implementation would be to add to QFilterCriteria:
+    ** - Map<FilterUseCase, CriteriaMissingInputValueBehavior> missingInputValueBehaviors;
+    ** - CriteriaMissingInputValueBehavior getMissingInputValueBehaviorForUseCase(FilterUseCase useCase) {}
+    *
+    ** (and maybe do that in a sub-class of QFilterCriteria, so it isn't always
+    ** there?  idk...) and then here we'd call:
+    ** - CriteriaMissingInputValueBehavior missingInputValueBehavior = criterion.getMissingInputValueBehaviorForUseCase(useCase);
+    *
+    ** But, we don't actually have that use-case at hand now, so - let's keep it
+    ** just at the level we need for now.
+    **
+    ***************************************************************************/
+   private CriteriaMissingInputValueBehavior getMissingInputValueBehavior(FilterUseCase useCase)
+   {
+      if(useCase == null)
+      {
+         useCase = FilterUseCase.DEFAULT;
+      }
+
+      CriteriaMissingInputValueBehavior missingInputValueBehavior = useCase.getDefaultCriteriaMissingInputValueBehavior();
+      if(missingInputValueBehavior == null)
+      {
+         missingInputValueBehavior = useCase.getDefaultCriteriaMissingInputValueBehavior();
+      }
+
+      if(missingInputValueBehavior == null)
+      {
+         missingInputValueBehavior = FilterUseCase.DEFAULT.getDefaultCriteriaMissingInputValueBehavior();
+      }
+
+      return (missingInputValueBehavior);
+   }
+
+
+
    /*******************************************************************************
     ** Getter for skip
     *******************************************************************************/
@@ -678,4 +775,28 @@ public class QQueryFilter implements Serializable, Cloneable
    {
       return Objects.hash(criteria, orderBys, booleanOperator, subFilters, skip, limit);
    }
+
+
+
+   /***************************************************************************
+    ** "Token" object to be used as the defaultIfLooksLikeVariableButNotFound
+    ** parameter to variableInterpreter.interpretForObject, so we can be
+    ** very clear that we got this default back (e.g., instead of a null,
+    ** which could maybe mean something else?)
+    ***************************************************************************/
+   private static final class InputNotFound implements Serializable
+   {
+      private static InputNotFound instance = new InputNotFound();
+
+
+
+      /*******************************************************************************
+       ** private singleton constructor
+       *******************************************************************************/
+      private InputNotFound()
+      {
+
+      }
+   }
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QueryInput.java

@@ -66,6 +66,14 @@ public class QueryInput extends AbstractTableActionInput implements QueryOrGetIn
    private List<QueryJoin> queryJoins     = null;
    private boolean         selectDistinct = false;
 
+   /////////////////////////////////////////////////////////////////////////////
+   // if this set is null, then the default (all fields) should be included   //
+   // if it's an empty set, that should throw an error                        //
+   // or if there are any fields in it that aren't valid fields on the table, //
+   // or in a selected queryJoin.                                             //
+   /////////////////////////////////////////////////////////////////////////////
+   private Set<String> fieldNamesToInclude;
+
    ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
    // if you say you want to includeAssociations, you can limit which ones by passing them in associationNamesToInclude. //
    // if you leave it null, you get all associations defined on the table.  if you pass it as empty, you get none.       //
@@ -686,4 +694,35 @@ public class QueryInput extends AbstractTableActionInput implements QueryOrGetIn
       return (queryHints.contains(queryHint));
    }
 
+
+
+   /*******************************************************************************
+    ** Getter for fieldNamesToInclude
+    *******************************************************************************/
+   public Set<String> getFieldNamesToInclude()
+   {
+      return (this.fieldNamesToInclude);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for fieldNamesToInclude
+    *******************************************************************************/
+   public void setFieldNamesToInclude(Set<String> fieldNamesToInclude)
+   {
+      this.fieldNamesToInclude = fieldNamesToInclude;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for fieldNamesToInclude
+    *******************************************************************************/
+   public QueryInput withFieldNamesToInclude(Set<String> fieldNamesToInclude)
+   {
+      this.fieldNamesToInclude = fieldNamesToInclude;
+      return (this);
+   }
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/expressions/AbstractFilterExpression.java

@@ -25,6 +25,7 @@ package com.kingsrook.qqq.backend.core.model.actions.tables.query.expressions;
 import java.io.Serializable;
 import java.util.Map;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
 
 
 /*******************************************************************************
@@ -35,7 +36,7 @@ public abstract class AbstractFilterExpression<T extends Serializable> implement
    /*******************************************************************************
     **
     *******************************************************************************/
-   public abstract T evaluate() throws QException;
+   public abstract T evaluate(QFieldMetaData field) throws QException;
 
 
 
@@ -47,7 +48,7 @@ public abstract class AbstractFilterExpression<T extends Serializable> implement
     *******************************************************************************/
    public T evaluateInputValues(Map<String, Serializable> inputValues) throws QException
    {
-      return evaluate();
+      return evaluate(null);
    }
 
 

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/expressions/FilterVariableExpression.java

@@ -26,6 +26,7 @@ import java.io.Serializable;
 import java.util.Map;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.exceptions.QUserFacingException;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
 import com.kingsrook.qqq.backend.core.utils.ValueUtils;
 
 
@@ -45,7 +46,7 @@ public class FilterVariableExpression extends AbstractFilterExpression<Serializa
     **
     *******************************************************************************/
    @Override
-   public Serializable evaluate() throws QException
+   public Serializable evaluate(QFieldMetaData field) throws QException
    {
       throw (new QUserFacingException("Missing variable value."));
    }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/expressions/Now.java

@@ -22,23 +22,42 @@
 package com.kingsrook.qqq.backend.core.model.actions.tables.query.expressions;
 
 
+import java.io.Serializable;
 import java.time.Instant;
+import java.time.ZoneId;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
+import com.kingsrook.qqq.backend.core.utils.ValueUtils;
 
 
 /*******************************************************************************
  **
  *******************************************************************************/
-public class Now extends AbstractFilterExpression<Instant>
+public class Now extends AbstractFilterExpression<Serializable>
 {
 
    /*******************************************************************************
     **
     *******************************************************************************/
    @Override
-   public Instant evaluate() throws QException
+   public Serializable evaluate(QFieldMetaData field) throws QException
    {
-      return (Instant.now());
+      QFieldType type = field == null ? QFieldType.DATE_TIME : field.getType();
+
+      if(type.equals(QFieldType.DATE_TIME))
+      {
+         return (Instant.now());
+      }
+      else if(type.equals(QFieldType.DATE))
+      {
+         ZoneId zoneId = ValueUtils.getSessionOrInstanceZoneId();
+         return (Instant.now().atZone(zoneId).toLocalDate());
+      }
+      else
+      {
+         throw (new QException("Unsupported field type [" + type + "]"));
+      }
    }
 
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/expressions/NowWithOffset.java

@@ -22,19 +22,24 @@
 package com.kingsrook.qqq.backend.core.model.actions.tables.query.expressions;
 
 
+import java.io.Serializable;
 import java.time.Instant;
+import java.time.LocalDate;
 import java.time.LocalDateTime;
 import java.time.ZoneId;
 import java.time.ZoneOffset;
 import java.time.temporal.ChronoUnit;
 import java.util.concurrent.TimeUnit;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
+import com.kingsrook.qqq.backend.core.utils.ValueUtils;
 
 
 /*******************************************************************************
  **
  *******************************************************************************/
-public class NowWithOffset extends AbstractFilterExpression<Instant>
+public class NowWithOffset extends AbstractFilterExpression<Serializable>
 {
    private Operator   operator;
    private int        amount;
@@ -123,7 +128,30 @@ public class NowWithOffset extends AbstractFilterExpression<Instant>
     **
     *******************************************************************************/
    @Override
-   public Instant evaluate() throws QException
+   public Serializable evaluate(QFieldMetaData field) throws QException
+   {
+      QFieldType type = field == null ? QFieldType.DATE_TIME : field.getType();
+
+      if(type.equals(QFieldType.DATE_TIME))
+      {
+         return (evaluateForDateTime());
+      }
+      else if(type.equals(QFieldType.DATE))
+      {
+         return (evaluateForDate());
+      }
+      else
+      {
+         throw (new QException("Unsupported field type [" + type + "]"));
+      }
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   private Instant evaluateForDateTime()
    {
       /////////////////////////////////////////////////////////////////////////////
       // Instant doesn't let us plus/minus WEEK, MONTH, or YEAR...               //
@@ -147,6 +175,26 @@ public class NowWithOffset extends AbstractFilterExpression<Instant>
 
 
 
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   private LocalDate evaluateForDate()
+   {
+      ZoneId    zoneId = ValueUtils.getSessionOrInstanceZoneId();
+      LocalDate now    = Instant.now().atZone(zoneId).toLocalDate();
+
+      if(operator.equals(Operator.PLUS))
+      {
+         return (now.plus(amount, timeUnit));
+      }
+      else
+      {
+         return (now.minus(amount, timeUnit));
+      }
+   }
+
+
+
    /*******************************************************************************
     ** Getter for operator
     **

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/expressions/ThisOrLastPeriod.java

@@ -22,27 +22,32 @@
 package com.kingsrook.qqq.backend.core.model.actions.tables.query.expressions;
 
 
+import java.io.Serializable;
 import java.time.DayOfWeek;
 import java.time.Instant;
+import java.time.LocalDate;
 import java.time.LocalDateTime;
 import java.time.ZoneId;
 import java.time.temporal.ChronoField;
 import java.time.temporal.ChronoUnit;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.exceptions.QRuntimeException;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
 import com.kingsrook.qqq.backend.core.utils.ValueUtils;
 
 
 /*******************************************************************************
  **
  *******************************************************************************/
-public class ThisOrLastPeriod extends AbstractFilterExpression<Instant>
+public class ThisOrLastPeriod extends AbstractFilterExpression<Serializable>
 {
    private Operator   operator;
    private ChronoUnit timeUnit;
 
 
 
+
    /***************************************************************************
     **
     ***************************************************************************/
@@ -88,7 +93,7 @@ public class ThisOrLastPeriod extends AbstractFilterExpression<Instant>
     ** Factory
     **
     *******************************************************************************/
-   public static ThisOrLastPeriod last(int amount, ChronoUnit timeUnit)
+   public static ThisOrLastPeriod last(ChronoUnit timeUnit)
    {
       return (new ThisOrLastPeriod(Operator.LAST, timeUnit));
    }
@@ -99,7 +104,31 @@ public class ThisOrLastPeriod extends AbstractFilterExpression<Instant>
     **
     *******************************************************************************/
    @Override
-   public Instant evaluate() throws QException
+   public Serializable evaluate(QFieldMetaData field) throws QException
+   {
+      QFieldType type = field == null ? QFieldType.DATE_TIME : field.getType();
+
+      if(type.equals(QFieldType.DATE_TIME))
+      {
+         return (evaluateForDateTime());
+      }
+      else if(type.equals(QFieldType.DATE))
+      {
+         // return (evaluateForDateTime());
+         return (evaluateForDate());
+      }
+      else
+      {
+         throw (new QException("Unsupported field type [" + type + "]"));
+      }
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   private Instant evaluateForDateTime()
    {
       ZoneId zoneId = ValueUtils.getSessionOrInstanceZoneId();
 
@@ -154,7 +183,57 @@ public class ThisOrLastPeriod extends AbstractFilterExpression<Instant>
 
             return operator.equals(Operator.THIS) ? startOfThisYear : startOfLastYear;
          }
-         default -> throw (new QRuntimeException("Unsupported timeUnit: " + timeUnit));
+         default -> throw (new QRuntimeException("Unsupported unit: " + timeUnit));
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public LocalDate evaluateForDate()
+   {
+      ZoneId    zoneId = ValueUtils.getSessionOrInstanceZoneId();
+      LocalDate today  = Instant.now().atZone(zoneId).toLocalDate();
+
+      switch(timeUnit)
+      {
+         case DAYS ->
+         {
+            return operator.equals(Operator.THIS) ? today : today.minusDays(1);
+         }
+         case WEEKS ->
+         {
+            LocalDate startOfThisWeek = today;
+            while(startOfThisWeek.get(ChronoField.DAY_OF_WEEK) != DayOfWeek.SUNDAY.getValue())
+            {
+               ////////////////////////////////////////
+               // go backwards until sunday is found //
+               ////////////////////////////////////////
+               startOfThisWeek = startOfThisWeek.minusDays(1);
+            }
+            return operator.equals(Operator.THIS) ? startOfThisWeek : startOfThisWeek.minusDays(7);
+         }
+         case MONTHS ->
+         {
+            Instant       startOfThisMonth    = ValueUtils.getStartOfMonthInZoneId(zoneId.getId());
+            LocalDateTime startOfThisMonthLDT = LocalDateTime.ofInstant(startOfThisMonth, ZoneId.of(zoneId.getId()));
+            LocalDateTime startOfLastMonthLDT = startOfThisMonthLDT.minusMonths(1);
+            Instant       startOfLastMonth    = startOfLastMonthLDT.toInstant(ZoneId.of(zoneId.getId()).getRules().getOffset(Instant.now()));
+
+            return (operator.equals(Operator.THIS) ? startOfThisMonth : startOfLastMonth).atZone(zoneId).toLocalDate();
+         }
+         case YEARS ->
+         {
+            Instant       startOfThisYear    = ValueUtils.getStartOfYearInZoneId(zoneId.getId());
+            LocalDateTime startOfThisYearLDT = LocalDateTime.ofInstant(startOfThisYear, zoneId);
+            LocalDateTime startOfLastYearLDT = startOfThisYearLDT.minusYears(1);
+            Instant       startOfLastYear    = startOfLastYearLDT.toInstant(zoneId.getRules().getOffset(Instant.now()));
+
+            return (operator.equals(Operator.THIS) ? startOfThisYear : startOfLastYear).atZone(zoneId).toLocalDate();
+         }
+         default -> throw (new QRuntimeException("Unsupported unit: " + timeUnit));
       }
    }
 

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/audits/AuditsMetaDataProvider.java

@@ -220,7 +220,7 @@ public class AuditsMetaDataProvider
          .withRecordLabelFormat("%s %s")
          .withRecordLabelFields("auditTableId", "recordId")
          .withPrimaryKeyField("id")
-         .withField(new QFieldMetaData("id", QFieldType.INTEGER))
+         .withField(new QFieldMetaData("id", QFieldType.LONG))
          .withField(new QFieldMetaData("auditTableId", QFieldType.INTEGER).withPossibleValueSourceName(TABLE_NAME_AUDIT_TABLE))
          .withField(new QFieldMetaData("auditUserId", QFieldType.INTEGER).withPossibleValueSourceName(TABLE_NAME_AUDIT_USER))
          .withField(new QFieldMetaData("recordId", QFieldType.INTEGER))
@@ -243,8 +243,8 @@ public class AuditsMetaDataProvider
          .withRecordLabelFormat("%s")
          .withRecordLabelFields("id")
          .withPrimaryKeyField("id")
-         .withField(new QFieldMetaData("id", QFieldType.INTEGER))
-         .withField(new QFieldMetaData("auditId", QFieldType.INTEGER).withPossibleValueSourceName(TABLE_NAME_AUDIT))
+         .withField(new QFieldMetaData("id", QFieldType.LONG))
+         .withField(new QFieldMetaData("auditId", QFieldType.LONG).withPossibleValueSourceName(TABLE_NAME_AUDIT))
          .withField(new QFieldMetaData("message", QFieldType.STRING).withMaxLength(250).withBehavior(ValueTooLongBehavior.TRUNCATE_ELLIPSIS))
          .withField(new QFieldMetaData("fieldName", QFieldType.STRING).withMaxLength(100).withBehavior(ValueTooLongBehavior.TRUNCATE_ELLIPSIS))
          .withField(new QFieldMetaData("oldValue", QFieldType.STRING).withMaxLength(250).withBehavior(ValueTooLongBehavior.TRUNCATE_ELLIPSIS))

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/dashboard/widgets/CompositeWidgetData.java

@@ -40,9 +40,10 @@ public class CompositeWidgetData extends AbstractBlockWidgetData<CompositeWidget
 {
    private List<AbstractBlockWidgetData<?, ?, ?, ?>> blocks = new ArrayList<>();
 
-   private Map<String, Serializable> styleOverrides = new HashMap<>();
-
-   private Layout layout;
+   private Layout                    layout;
+   private Map<String, Serializable> styleOverrides        = new HashMap<>();
+   private String                    overlayHtml;
+   private Map<String, Serializable> overlayStyleOverrides = new HashMap<>();
 
 
 
@@ -218,4 +219,91 @@ public class CompositeWidgetData extends AbstractBlockWidgetData<CompositeWidget
       return (this);
    }
 
+
+
+   /*******************************************************************************
+    ** Getter for overlayHtml
+    *******************************************************************************/
+   public String getOverlayHtml()
+   {
+      return (this.overlayHtml);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for overlayHtml
+    *******************************************************************************/
+   public void setOverlayHtml(String overlayHtml)
+   {
+      this.overlayHtml = overlayHtml;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for overlayHtml
+    *******************************************************************************/
+   public CompositeWidgetData withOverlayHtml(String overlayHtml)
+   {
+      this.overlayHtml = overlayHtml;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for overlayStyleOverrides
+    *******************************************************************************/
+   public Map<String, Serializable> getOverlayStyleOverrides()
+   {
+      return (this.overlayStyleOverrides);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for overlayStyleOverrides
+    *******************************************************************************/
+   public void setOverlayStyleOverrides(Map<String, Serializable> overlayStyleOverrides)
+   {
+      this.overlayStyleOverrides = overlayStyleOverrides;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for overlayStyleOverrides
+    *******************************************************************************/
+   public CompositeWidgetData withOverlayStyleOverrides(Map<String, Serializable> overlayStyleOverrides)
+   {
+      this.overlayStyleOverrides = overlayStyleOverrides;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public CompositeWidgetData withOverlayStyleOverride(String key, Serializable value)
+   {
+      addOverlayStyleOverride(key, value);
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public void addOverlayStyleOverride(String key, Serializable value)
+   {
+      if(this.overlayStyleOverrides == null)
+      {
+         this.overlayStyleOverrides = new HashMap<>();
+      }
+      this.overlayStyleOverrides.put(key, value);
+   }
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/dashboard/widgets/blocks/AbstractBlockWidgetData.java

@@ -24,6 +24,7 @@ package com.kingsrook.qqq.backend.core.model.dashboard.widgets.blocks;
 
 import java.util.HashMap;
 import java.util.Map;
+import com.kingsrook.qqq.backend.core.model.dashboard.widgets.CompositeWidgetData;
 import com.kingsrook.qqq.backend.core.model.dashboard.widgets.QWidgetData;
 
 
@@ -203,6 +204,19 @@ public abstract class AbstractBlockWidgetData<
 
 
 
+   /*******************************************************************************
+    ** Fluent setter for tooltip
+    **
+    *******************************************************************************/
+   @SuppressWarnings("unchecked")
+   public T withTooltip(CompositeWidgetData data)
+   {
+      this.tooltip = new BlockTooltip(data);
+      return (T) (this);
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/
@@ -398,6 +412,7 @@ public abstract class AbstractBlockWidgetData<
    }
 
 
+
    /*******************************************************************************
     ** Getter for blockId
     *******************************************************************************/
@@ -428,5 +443,4 @@ public abstract class AbstractBlockWidgetData<
       return (T) this;
    }
 
-
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/dashboard/widgets/blocks/BlockTooltip.java

@@ -22,14 +22,18 @@
 package com.kingsrook.qqq.backend.core.model.dashboard.widgets.blocks;
 
 
+import com.kingsrook.qqq.backend.core.model.dashboard.widgets.CompositeWidgetData;
+
+
 /*******************************************************************************
  ** A tooltip used within a (widget) block.
  **
  *******************************************************************************/
 public class BlockTooltip
 {
-   private String    title;
-   private Placement placement = Placement.BOTTOM;
+   private CompositeWidgetData blockData;
+   private String              title;
+   private Placement           placement = Placement.BOTTOM;
 
 
 
@@ -62,6 +66,17 @@ public class BlockTooltip
 
 
 
+   /*******************************************************************************
+    ** Constructor
+    **
+    *******************************************************************************/
+   public BlockTooltip(CompositeWidgetData blockData)
+   {
+      this.blockData = blockData;
+   }
+
+
+
    /*******************************************************************************
     ** Getter for title
     *******************************************************************************/
@@ -122,4 +137,35 @@ public class BlockTooltip
       return (this);
    }
 
+
+
+   /*******************************************************************************
+    ** Getter for blockData
+    *******************************************************************************/
+   public CompositeWidgetData getBlockData()
+   {
+      return (this.blockData);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for blockData
+    *******************************************************************************/
+   public void setBlockData(CompositeWidgetData blockData)
+   {
+      this.blockData = blockData;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for blockData
+    *******************************************************************************/
+   public BlockTooltip withBlockData(CompositeWidgetData blockData)
+   {
+      this.blockData = blockData;
+      return (this);
+   }
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/QBackendMetaData.java

@@ -53,6 +53,8 @@ public class QBackendMetaData implements TopLevelMetaDataInterface
    private String  variantOptionsTableUsernameField;
    private String  variantOptionsTablePasswordField;
    private String  variantOptionsTableApiKeyField;
+   private String  variantOptionsTableClientIdField;
+   private String  variantOptionsTableClientSecretField;
    private String  variantOptionsTableName;
 
    // todo - at some point, we may want to apply this to secret properties on subclasses?
@@ -648,4 +650,66 @@ public class QBackendMetaData implements TopLevelMetaDataInterface
    {
       qInstance.addBackend(this);
    }
+
+   /*******************************************************************************
+    ** Getter for variantOptionsTableClientIdField
+    *******************************************************************************/
+   public String getVariantOptionsTableClientIdField()
+   {
+      return (this.variantOptionsTableClientIdField);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for variantOptionsTableClientIdField
+    *******************************************************************************/
+   public void setVariantOptionsTableClientIdField(String variantOptionsTableClientIdField)
+   {
+      this.variantOptionsTableClientIdField = variantOptionsTableClientIdField;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for variantOptionsTableClientIdField
+    *******************************************************************************/
+   public QBackendMetaData withVariantOptionsTableClientIdField(String variantOptionsTableClientIdField)
+   {
+      this.variantOptionsTableClientIdField = variantOptionsTableClientIdField;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for variantOptionsTableClientSecretField
+    *******************************************************************************/
+   public String getVariantOptionsTableClientSecretField()
+   {
+      return (this.variantOptionsTableClientSecretField);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for variantOptionsTableClientSecretField
+    *******************************************************************************/
+   public void setVariantOptionsTableClientSecretField(String variantOptionsTableClientSecretField)
+   {
+      this.variantOptionsTableClientSecretField = variantOptionsTableClientSecretField;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for variantOptionsTableClientSecretField
+    *******************************************************************************/
+   public QBackendMetaData withVariantOptionsTableClientSecretField(String variantOptionsTableClientSecretField)
+   {
+      this.variantOptionsTableClientSecretField = variantOptionsTableClientSecretField;
+      return (this);
+   }
+
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/possiblevalues/PossibleValueSearchFilterUseCase.java

@@ -0,0 +1,72 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.model.metadata.possiblevalues;
+
+
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.CriteriaMissingInputValueBehavior;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.FilterUseCase;
+
+
+/*******************************************************************************
+ ** FilterUseCase implementation for the ways that possible value searches
+ ** are performed, and where we want to have different behaviors for criteria
+ ** that are missing an input value.  That is, either for a:
+ **
+ ** - FORM - e.g., creating a new record, or in a process - where we want a
+ ** missing filter value to basically block you from selecting a value in the
+ ** PVS field - e.g., you must enter some other foreign-key value before choosing
+ ** from this possible value - at least that's the use-case we know of now.
+ **
+ ** - FILTER - e.g., a query screen - where there isn't really quite the same
+ ** scenario of choosing that foreign-key value first - so, such a PVS should
+ ** list all its values (e.g., a criteria missing an input value should be
+ ** removed from the filter).
+ *******************************************************************************/
+public enum PossibleValueSearchFilterUseCase implements FilterUseCase
+{
+   FORM(CriteriaMissingInputValueBehavior.MAKE_NO_MATCHES),
+   FILTER(CriteriaMissingInputValueBehavior.REMOVE_FROM_FILTER);
+
+
+   private final CriteriaMissingInputValueBehavior defaultCriteriaMissingInputValueBehavior;
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   PossibleValueSearchFilterUseCase(CriteriaMissingInputValueBehavior defaultCriteriaMissingInputValueBehavior)
+   {
+      this.defaultCriteriaMissingInputValueBehavior = defaultCriteriaMissingInputValueBehavior;
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for defaultCriteriaMissingInputValueBehavior
+    **
+    *******************************************************************************/
+   public CriteriaMissingInputValueBehavior getDefaultCriteriaMissingInputValueBehavior()
+   {
+      return defaultCriteriaMissingInputValueBehavior;
+   }
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/reporting/QReportDataSource.java

@@ -32,6 +32,13 @@ import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
 /*******************************************************************************
  ** Meta-data definition of a source of data for a report (e.g., a table and query
  ** filter or custom-code reference).
+ **
+ ** Runs in 3 modes:
+ **
+ ** - If a customRecordSource is specified, then that code is executed to get the records.
+ ** - else, if a sourceTable is specified, then the corresponding queryFilter
+ ** (optionally along with queryJoins and queryInputCustomizer) is used.
+ ** - else a staticDataSupplier is used.
  *******************************************************************************/
 public class QReportDataSource
 {
@@ -44,6 +51,7 @@ public class QReportDataSource
 
    private QCodeReference queryInputCustomizer;
    private QCodeReference staticDataSupplier;
+   private QCodeReference customRecordSource;
 
 
 
@@ -265,4 +273,35 @@ public class QReportDataSource
       return (this);
    }
 
+
+   /*******************************************************************************
+    ** Getter for customRecordSource
+    *******************************************************************************/
+   public QCodeReference getCustomRecordSource()
+   {
+      return (this.customRecordSource);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for customRecordSource
+    *******************************************************************************/
+   public void setCustomRecordSource(QCodeReference customRecordSource)
+   {
+      this.customRecordSource = customRecordSource;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for customRecordSource
+    *******************************************************************************/
+   public QReportDataSource withCustomRecordSource(QCodeReference customRecordSource)
+   {
+      this.customRecordSource = customRecordSource;
+      return (this);
+   }
+
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/authentication/implementations/Auth0AuthenticationModule.java

@@ -247,10 +247,7 @@ public class Auth0AuthenticationModule implements QAuthenticationModuleInterface
             //////////////////////////////////////////////////////////////
             // allow customizer to do custom things here, if so desired //
             //////////////////////////////////////////////////////////////
-            if(getCustomizer() != null)
-            {
-               getCustomizer().finalCustomizeSession(qInstance, qSession);
-            }
+            finalCustomizeSession(qInstance, qSession);
 
             return (qSession);
          }
@@ -311,10 +308,7 @@ public class Auth0AuthenticationModule implements QAuthenticationModuleInterface
          //////////////////////////////////////////////////////////////
          // allow customizer to do custom things here, if so desired //
          //////////////////////////////////////////////////////////////
-         if(getCustomizer() != null)
-         {
-            getCustomizer().finalCustomizeSession(qInstance, qSession);
-         }
+         finalCustomizeSession(qInstance, qSession);
 
          return (qSession);
       }
@@ -360,6 +354,23 @@ public class Auth0AuthenticationModule implements QAuthenticationModuleInterface
 
 
 
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   private void finalCustomizeSession(QInstance qInstance, QSession qSession)
+   {
+      if(getCustomizer() != null)
+      {
+         QContext.withTemporaryContext(QContext.capture(), () ->
+         {
+            QContext.setQSession(getChickenAndEggSession());
+            getCustomizer().finalCustomizeSession(qInstance, qSession);
+         });
+      }
+   }
+
+
+
    /*******************************************************************************
     ** Insert a session as a new record into userSession table
     *******************************************************************************/

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/memory/MemoryRecordStore.java

@@ -58,6 +58,7 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.JoinsContext;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryJoin;
 import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
@@ -170,6 +171,8 @@ public class MemoryRecordStore
       Collection<QRecord> tableData = getTableData(input.getTable()).values();
       List<QRecord>       records   = new ArrayList<>();
 
+      QQueryFilter filter       = clonedOrNewFilter(input.getFilter());
+      JoinsContext joinsContext = new JoinsContext(QContext.getQInstance(), input.getTableName(), input.getQueryJoins(), filter);
       if(CollectionUtils.nullSafeHasContents(input.getQueryJoins()))
       {
          tableData = buildJoinCrossProduct(input);
@@ -185,7 +188,7 @@ public class MemoryRecordStore
             qRecord.setTableName(input.getTableName());
          }
 
-         boolean recordMatches = BackendQueryFilterUtils.doesRecordMatch(input.getFilter(), qRecord);
+         boolean recordMatches = BackendQueryFilterUtils.doesRecordMatch(input.getFilter(), joinsContext, qRecord);
 
          if(recordMatches)
          {
@@ -224,8 +227,7 @@ public class MemoryRecordStore
     *******************************************************************************/
    private Collection<QRecord> buildJoinCrossProduct(QueryInput input) throws QException
    {
-      QInstance    qInstance    = QContext.getQInstance();
-      JoinsContext joinsContext = new JoinsContext(qInstance, input.getTableName(), input.getQueryJoins(), input.getFilter());
+      QInstance qInstance = QContext.getQInstance();
 
       List<QRecord>  crossProduct = new ArrayList<>();
       QTableMetaData leftTable    = input.getTable();
@@ -373,7 +375,14 @@ public class MemoryRecordStore
          /////////////////////////////////////////////////
          if(recordToInsert.getValue(primaryKeyField.getName()) == null && (primaryKeyField.getType().equals(QFieldType.INTEGER) || primaryKeyField.getType().equals(QFieldType.LONG)))
          {
-            recordToInsert.setValue(primaryKeyField.getName(), nextSerial++);
+            if(primaryKeyField.getType().equals(QFieldType.LONG))
+            {
+               recordToInsert.setValue(primaryKeyField.getName(), (nextSerial++).longValue());
+            }
+            else
+            {
+               recordToInsert.setValue(primaryKeyField.getName(), nextSerial++);
+            }
          }
 
          ///////////////////////////////////////////////////////////////////////////////////////////////////
@@ -383,7 +392,7 @@ public class MemoryRecordStore
          {
             nextSerial = recordToInsert.getValueInteger(primaryKeyField.getName()) + 1;
          }
-         else if(primaryKeyField.getType().equals(QFieldType.LONG) && recordToInsert.getValueLong(primaryKeyField.getName()) > nextSerial)
+         else if(primaryKeyField.getType().equals(QFieldType.LONG) && recordToInsert.getValueInteger(primaryKeyField.getName()) > nextSerial)
          {
             //////////////////////////////////////
             // todo - mmm, could overflow here? //
@@ -901,4 +910,21 @@ public class MemoryRecordStore
 
       return ValueUtils.getValueAsFieldType(fieldType, aggregateValue);
    }
+
+
+
+   /*******************************************************************************
+    ** Either clone the input filter (so we can change it safely), or return a new blank filter.
+    *******************************************************************************/
+   protected QQueryFilter clonedOrNewFilter(QQueryFilter filter)
+   {
+      if(filter == null)
+      {
+         return (new QQueryFilter());
+      }
+      else
+      {
+         return (filter.clone());
+      }
+   }
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/utils/BackendQueryFilterUtils.java

@@ -34,14 +34,18 @@ import java.util.Objects;
 import java.util.concurrent.atomic.AtomicBoolean;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.logging.QLogger;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.JoinsContext;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.expressions.AbstractFilterExpression;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
 import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
 import com.kingsrook.qqq.backend.core.utils.ValueUtils;
 import org.apache.commons.lang.NotImplementedException;
+import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
 
 
 /*******************************************************************************
@@ -58,8 +62,22 @@ public class BackendQueryFilterUtils
 
    /*******************************************************************************
     ** Test if record matches filter.
-    *******************************************************************************/
+    ******************************************************************************/
    public static boolean doesRecordMatch(QQueryFilter filter, QRecord qRecord)
+   {
+      return doesRecordMatch(filter, null, qRecord);
+   }
+
+
+
+   /*******************************************************************************
+    ** Test if record matches filter - where we are executing a QueryAction, and
+    ** we have a JoinsContext.  Note, if you don't have one of those, you can call
+    ** the overload of this method that doesn't take one, and everything downstream
+    ** /should/ be tolerant of that being absent...  You just might not have the
+    ** benefit of things like knowing field-meta-data associated with criteria...
+    *******************************************************************************/
+   public static boolean doesRecordMatch(QQueryFilter filter, JoinsContext joinsContext, QRecord qRecord)
    {
       if(filter == null || !filter.hasAnyCriteria())
       {
@@ -97,7 +115,36 @@ public class BackendQueryFilterUtils
             }
          }
 
-         boolean criterionMatches = doesCriteriaMatch(criterion, fieldName, value);
+         ///////////////////////////////////////////////////////////////////////////////////////////////
+         // Test if this criteria(on) matches the record.                                             //
+         // As criteria have become more sophisticated over time, we would like to be able to know    //
+         // what field they are for. In general, we'll try to get that from the query's JoinsContext. //
+         // But, in some scenarios, that isn't available - so - be safe and defer to simpler methods  //
+         // that might not have the full field, when necessary.                                       //
+         ///////////////////////////////////////////////////////////////////////////////////////////////
+         Boolean criterionMatches = null;
+         if(joinsContext != null)
+         {
+            JoinsContext.FieldAndTableNameOrAlias fieldAndTableNameOrAlias = null;
+            try
+            {
+               fieldAndTableNameOrAlias = joinsContext.getFieldAndTableNameOrAlias(criterion.getFieldName());
+            }
+            catch(Exception e)
+            {
+               LOG.debug("Exception getting field from joinsContext", e, logPair("fieldName", criterion.getFieldName()));
+            }
+
+            if(fieldAndTableNameOrAlias != null)
+            {
+               criterionMatches = doesCriteriaMatch(criterion, fieldAndTableNameOrAlias.field(), value);
+            }
+         }
+
+         if(criterionMatches == null)
+         {
+            criterionMatches = doesCriteriaMatch(criterion, criterion.getFieldName(), value);
+         }
 
          ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
          // add this new value to the existing recordMatches value - and if we can short circuit the remaining checks, do so. //
@@ -131,11 +178,24 @@ public class BackendQueryFilterUtils
 
 
 
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   public static boolean doesCriteriaMatch(QFilterCriteria criterion, String fieldName, Serializable value)
+   {
+      QFieldMetaData field = new QFieldMetaData(fieldName, ValueUtils.inferQFieldTypeFromValue(value, QFieldType.STRING));
+      return doesCriteriaMatch(criterion, field, value);
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/
-   public static boolean doesCriteriaMatch(QFilterCriteria criterion, String fieldName, Serializable value)
+   private static boolean doesCriteriaMatch(QFilterCriteria criterion, QFieldMetaData field, Serializable value)
    {
+      String fieldName = field == null ? "__unknownField" : field.getName();
+
       ListIterator<Serializable> valueListIterator = criterion.getValues().listIterator();
       while(valueListIterator.hasNext())
       {
@@ -144,7 +204,7 @@ public class BackendQueryFilterUtils
          {
             try
             {
-               valueListIterator.set(expression.evaluate());
+               valueListIterator.set(expression.evaluate(field));
             }
             catch(QException qe)
             {

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/basepull/ExtractViaBasepullQueryStep.java

@@ -40,6 +40,10 @@ import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwith
  *******************************************************************************/
 public class ExtractViaBasepullQueryStep extends ExtractViaQueryStep implements BasepullExtractStepInterface
 {
+   protected static final String SECONDS_TO_SUBTRACT_FROM_THIS_RUN_TIME_KEY = "secondsToSubtractFromThisRunTimeForTimestampQuery";
+   protected static final String SECONDS_TO_SUBTRACT_FROM_LAST_RUN_TIME_KEY = "secondsToSubtractFromLastRunTimeForTimestampQuery";
+
+
 
    /*******************************************************************************
     **
@@ -124,7 +128,8 @@ public class ExtractViaBasepullQueryStep extends ExtractViaQueryStep implements
     *******************************************************************************/
    protected String getLastRunTimeString(RunBackendStepInput runBackendStepInput) throws QException
    {
-      Instant lastRunTime = runBackendStepInput.getBasepullLastRunTime();
+      Instant lastRunTime    = runBackendStepInput.getBasepullLastRunTime();
+      Instant updatedRunTime = lastRunTime;
 
       //////////////////////////////////////////////////////////////////////////////////////////////
       // allow the timestamps to be adjusted by the specified number of seconds.                  //
@@ -135,10 +140,19 @@ public class ExtractViaBasepullQueryStep extends ExtractViaQueryStep implements
       Serializable basepullConfigurationValue = runBackendStepInput.getValue(RunProcessAction.BASEPULL_CONFIGURATION);
       if(basepullConfigurationValue instanceof BasepullConfiguration basepullConfiguration && basepullConfiguration.getSecondsToSubtractFromLastRunTimeForTimestampQuery() != null)
       {
-         lastRunTime = lastRunTime.minusSeconds(basepullConfiguration.getSecondsToSubtractFromLastRunTimeForTimestampQuery());
+         updatedRunTime = lastRunTime.minusSeconds(basepullConfiguration.getSecondsToSubtractFromLastRunTimeForTimestampQuery());
       }
 
-      return (lastRunTime.toString());
+      //////////////////////////////////////////////////////////////
+      // if an override was found in the params, use that instead //
+      //////////////////////////////////////////////////////////////
+      if(runBackendStepInput.getValueString(SECONDS_TO_SUBTRACT_FROM_LAST_RUN_TIME_KEY) != null)
+      {
+         int secondsBack = Integer.parseInt(runBackendStepInput.getValueString(SECONDS_TO_SUBTRACT_FROM_LAST_RUN_TIME_KEY));
+         updatedRunTime = lastRunTime.minusSeconds(secondsBack);
+      }
+
+      return (updatedRunTime.toString());
    }
 
 
@@ -148,14 +162,24 @@ public class ExtractViaBasepullQueryStep extends ExtractViaQueryStep implements
     *******************************************************************************/
    protected String getThisRunTimeString(RunBackendStepInput runBackendStepInput) throws QException
    {
-      Instant thisRunTime = runBackendStepInput.getValueInstant(RunProcessAction.BASEPULL_THIS_RUNTIME_KEY);
+      Instant thisRunTime    = runBackendStepInput.getValueInstant(RunProcessAction.BASEPULL_THIS_RUNTIME_KEY);
+      Instant updatedRunTime = thisRunTime;
 
       Serializable basepullConfigurationValue = runBackendStepInput.getValue(RunProcessAction.BASEPULL_CONFIGURATION);
       if(basepullConfigurationValue instanceof BasepullConfiguration basepullConfiguration && basepullConfiguration.getSecondsToSubtractFromThisRunTimeForTimestampQuery() != null)
       {
-         thisRunTime = thisRunTime.minusSeconds(basepullConfiguration.getSecondsToSubtractFromThisRunTimeForTimestampQuery());
+         updatedRunTime = thisRunTime.minusSeconds(basepullConfiguration.getSecondsToSubtractFromThisRunTimeForTimestampQuery());
       }
 
-      return (thisRunTime.toString());
+      //////////////////////////////////////////////////////////////
+      // if an override was found in the params, use that instead //
+      //////////////////////////////////////////////////////////////
+      if(runBackendStepInput.getValueString(SECONDS_TO_SUBTRACT_FROM_THIS_RUN_TIME_KEY) != null)
+      {
+         int secondsBack = Integer.parseInt(runBackendStepInput.getValueString(SECONDS_TO_SUBTRACT_FROM_THIS_RUN_TIME_KEY));
+         updatedRunTime = thisRunTime.minusSeconds(secondsBack);
+      }
+
+      return (updatedRunTime.toString());
    }
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/reports/BasicRunReportProcess.java

@@ -46,7 +46,8 @@ public class BasicRunReportProcess
    public static final String STEP_NAME_EXECUTE = "execute";
    public static final String STEP_NAME_ACCESS  = "accessReport";
 
-   public static final String FIELD_REPORT_NAME = "reportName";
+   public static final String FIELD_REPORT_NAME   = "reportName";
+   public static final String FIELD_REPORT_FORMAT = "reportFormat";
 
 
 

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/reports/ExecuteReportStep.java

@@ -33,6 +33,7 @@ import com.kingsrook.qqq.backend.core.actions.processes.BackendStep;
 import com.kingsrook.qqq.backend.core.actions.reporting.GenerateReportAction;
 import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.exceptions.QUserFacingException;
 import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
 import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
 import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportDestination;
@@ -50,6 +51,7 @@ import com.kingsrook.qqq.backend.core.utils.StringUtils;
 public class ExecuteReportStep implements BackendStep
 {
 
+
    /*******************************************************************************
     **
     *******************************************************************************/
@@ -58,9 +60,10 @@ public class ExecuteReportStep implements BackendStep
    {
       try
       {
-         String          reportName = runBackendStepInput.getValueString("reportName");
-         QReportMetaData report     = QContext.getQInstance().getReport(reportName);
-         File            tmpFile    = File.createTempFile(reportName, ".xlsx", new File("/tmp/"));
+         ReportFormat    reportFormat = getReportFormat(runBackendStepInput);
+         String          reportName   = runBackendStepInput.getValueString("reportName");
+         QReportMetaData report       = QContext.getQInstance().getReport(reportName);
+         File            tmpFile      = File.createTempFile(reportName, "." + reportFormat.getExtension());
 
          runBackendStepInput.getAsyncJobCallback().updateStatus("Generating Report");
 
@@ -69,7 +72,7 @@ public class ExecuteReportStep implements BackendStep
             ReportInput reportInput = new ReportInput();
             reportInput.setReportName(reportName);
             reportInput.setReportDestination(new ReportDestination()
-               .withReportFormat(ReportFormat.XLSX) // todo - variable
+               .withReportFormat(reportFormat)
                .withReportOutputStream(reportOutputStream));
 
             Map<String, Serializable> values = runBackendStepInput.getValues();
@@ -79,7 +82,7 @@ public class ExecuteReportStep implements BackendStep
 
             String downloadFileBaseName = getDownloadFileBaseName(runBackendStepInput, report);
 
-            runBackendStepOutput.addValue("downloadFileName", downloadFileBaseName + ".xlsx");
+            runBackendStepOutput.addValue("downloadFileName", downloadFileBaseName + "." + reportFormat.getExtension());
             runBackendStepOutput.addValue("serverFilePath", tmpFile.getCanonicalPath());
          }
       }
@@ -91,6 +94,22 @@ public class ExecuteReportStep implements BackendStep
 
 
 
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   private ReportFormat getReportFormat(RunBackendStepInput runBackendStepInput) throws QUserFacingException
+   {
+      String reportFormatInput = runBackendStepInput.getValueString(BasicRunReportProcess.FIELD_REPORT_FORMAT);
+      if(StringUtils.hasContent(reportFormatInput))
+      {
+         return (ReportFormat.fromString(reportFormatInput));
+      }
+
+      return (ReportFormat.XLSX);
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/tablesync/AbstractTableSyncTransformStep.java

@@ -23,6 +23,9 @@ package com.kingsrook.qqq.backend.core.processes.implementations.tablesync;
 
 
 import java.io.Serializable;
+import java.time.Instant;
+import java.time.ZoneId;
+import java.time.ZonedDateTime;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
@@ -35,6 +38,7 @@ import java.util.Set;
 import java.util.stream.Collectors;
 import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
 import com.kingsrook.qqq.backend.core.actions.values.QPossibleValueTranslator;
+import com.kingsrook.qqq.backend.core.actions.values.QValueFormatter;
 import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.logging.QLogger;
@@ -53,6 +57,7 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryJoin;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
+import com.kingsrook.qqq.backend.core.model.session.QSession;
 import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.AbstractTransformStep;
 import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess;
 import com.kingsrook.qqq.backend.core.processes.implementations.general.StandardProcessSummaryLineProducer;
@@ -72,33 +77,33 @@ import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
  *******************************************************************************/
 public abstract class AbstractTableSyncTransformStep extends AbstractTransformStep
 {
-   private static final QLogger LOG = QLogger.getLogger(AbstractTableSyncTransformStep.class);
+   protected static final QLogger LOG = QLogger.getLogger(AbstractTableSyncTransformStep.class);
 
-   private ProcessSummaryLine okToInsert = StandardProcessSummaryLineProducer.getOkToInsertLine();
-   private ProcessSummaryLine okToUpdate = StandardProcessSummaryLineProducer.getOkToUpdateLine();
+   protected ProcessSummaryLine okToInsert = StandardProcessSummaryLineProducer.getOkToInsertLine();
+   protected ProcessSummaryLine okToUpdate = StandardProcessSummaryLineProducer.getOkToUpdateLine();
 
-   private ProcessSummaryLine willNotInsert = new ProcessSummaryLine(Status.INFO)
+   protected ProcessSummaryLine willNotInsert = new ProcessSummaryLine(Status.INFO)
       .withMessageSuffix("because this process is not configured to insert records.")
       .withSingularFutureMessage("will not be inserted ")
       .withPluralFutureMessage("will not be inserted ")
       .withSingularPastMessage("was not inserted ")
       .withPluralPastMessage("were not inserted ");
 
-   private ProcessSummaryLine willNotUpdate = new ProcessSummaryLine(Status.INFO)
+   protected ProcessSummaryLine willNotUpdate = new ProcessSummaryLine(Status.INFO)
       .withMessageSuffix("because this process is not configured to update records.")
       .withSingularFutureMessage("will not be updated ")
       .withPluralFutureMessage("will not be updated ")
       .withSingularPastMessage("was not updated ")
       .withPluralPastMessage("were not updated ");
 
-   private ProcessSummaryLine errorMissingKeyField = new ProcessSummaryLine(Status.ERROR)
+   protected ProcessSummaryLine errorMissingKeyField = new ProcessSummaryLine(Status.ERROR)
       .withMessageSuffix("missing a value for the key field.")
       .withSingularFutureMessage("will not be synced, because it is ")
       .withPluralFutureMessage("will not be synced, because they are ")
       .withSingularPastMessage("was not synced, because it is ")
       .withPluralPastMessage("were not synced, because they are ");
 
-   private ProcessSummaryLine unspecifiedError = new ProcessSummaryLine(Status.ERROR)
+   protected ProcessSummaryLine unspecifiedError = new ProcessSummaryLine(Status.ERROR)
       .withMessageSuffix("of an unexpected error: ")
       .withSingularFutureMessage("will not be synced, ")
       .withPluralFutureMessage("will not be synced, ")
@@ -109,7 +114,11 @@ public abstract class AbstractTableSyncTransformStep extends AbstractTransformSt
    protected RunBackendStepOutput runBackendStepOutput = null;
    protected RecordLookupHelper   recordLookupHelper   = null;
 
-   private QPossibleValueTranslator possibleValueTranslator;
+   protected QPossibleValueTranslator possibleValueTranslator;
+
+   protected static final String SYNC_TABLE_PERFORM_INSERTS_KEY = "syncTablePerformInsertsKey";
+   protected static final String SYNC_TABLE_PERFORM_UPDATES_KEY = "syncTablePerformUpdatesKey";
+   protected static final String LOG_TRANSFORM_RESULTS          = "logTransformResults";
 
 
 
@@ -214,6 +223,7 @@ public abstract class AbstractTableSyncTransformStep extends AbstractTransformSt
    {
       if(CollectionUtils.nullSafeIsEmpty(runBackendStepInput.getRecords()))
       {
+         LOG.info("No input records were found.");
          return;
       }
 
@@ -222,6 +232,19 @@ public abstract class AbstractTableSyncTransformStep extends AbstractTransformSt
 
       SyncProcessConfig config = getSyncProcessConfig();
 
+      ////////////////////////////////////////////////////////////
+      // see if these fields have been updated via input fields //
+      ////////////////////////////////////////////////////////////
+      if(runBackendStepInput.getValueString(SYNC_TABLE_PERFORM_INSERTS_KEY) != null)
+      {
+         boolean performInserts = Boolean.parseBoolean(runBackendStepInput.getValueString(SYNC_TABLE_PERFORM_INSERTS_KEY));
+         config = new SyncProcessConfig(config.sourceTable, config.sourceTableKeyField, config.destinationTable, config.destinationTableForeignKey, performInserts, config.performUpdates);
+      }
+      if(runBackendStepInput.getValueString(SYNC_TABLE_PERFORM_UPDATES_KEY) != null)
+      {
+         boolean performUpdates = Boolean.parseBoolean(runBackendStepInput.getValueString(SYNC_TABLE_PERFORM_UPDATES_KEY));
+         config = new SyncProcessConfig(config.sourceTable, config.sourceTableKeyField, config.destinationTable, config.destinationTableForeignKey, config.performUpdates, performUpdates);
+      }
       String sourceTableKeyField             = config.sourceTableKeyField;
       String destinationTableForeignKeyField = config.destinationTableForeignKey;
       String destinationTableName            = config.destinationTable;
@@ -371,9 +394,63 @@ public abstract class AbstractTableSyncTransformStep extends AbstractTransformSt
             possibleValueTranslator.translatePossibleValuesInRecords(QContext.getQInstance().getTable(destinationTableName), runBackendStepOutput.getRecords());
          }
       }
+
+      if(Boolean.parseBoolean(runBackendStepInput.getValueString(LOG_TRANSFORM_RESULTS)))
+      {
+         logResults(runBackendStepInput, config);
+      }
    }
 
 
+
+   /*******************************************************************************
+    ** Log results of transformation
+    **
+    *******************************************************************************/
+   protected void logResults(RunBackendStepInput runBackendStepInput, SyncProcessConfig syncProcessConfig)
+   {
+      String timezone = QContext.getQSession().getValue(QSession.VALUE_KEY_USER_TIMEZONE);
+      if(timezone == null)
+      {
+         timezone = QContext.getQInstance().getDefaultTimeZoneId();
+      }
+      Instant lastRunTime = Instant.now();
+      if(runBackendStepInput.getBasepullLastRunTime() != null)
+      {
+         lastRunTime = runBackendStepInput.getBasepullLastRunTime();
+      }
+
+      ZonedDateTime dateTime = lastRunTime.atZone(ZoneId.of(timezone));
+
+      if(syncProcessConfig.performInserts)
+      {
+         if(okToInsert.getCount() == 0)
+         {
+            LOG.info("No Records were found to insert since " + QValueFormatter.formatDateTimeWithZone(dateTime) + ".");
+         }
+         else
+         {
+            String pluralized = okToInsert.getCount() > 1 ? " Records were " : " Record was ";
+            LOG.info(okToInsert.getCount() + pluralized + " found to insert since " + QValueFormatter.formatDateTimeWithZone(dateTime) + ".", logPair("primaryKeys", okToInsert.getPrimaryKeys()));
+         }
+      }
+
+      if(syncProcessConfig.performUpdates)
+      {
+         if(okToUpdate.getCount() == 0)
+         {
+            LOG.info("No Records were found to update since " + QValueFormatter.formatDateTimeWithZone(dateTime) + ".");
+         }
+         else
+         {
+            String pluralized = okToUpdate.getCount() > 1 ? " Records were " : " Record was ";
+            LOG.info(okToUpdate.getCount() + pluralized + " found to update since " + QValueFormatter.formatDateTimeWithZone(dateTime) + ".", logPair("primaryKeys", okToInsert.getPrimaryKeys()));
+         }
+      }
+   }
+
+
+
    /*******************************************************************************
     ** Given a source record, extract what we'll use as its key from it.
     **

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/ValueUtils.java

@@ -41,6 +41,7 @@ import java.util.List;
 import java.util.TimeZone;
 import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.exceptions.QValueException;
+import com.kingsrook.qqq.backend.core.logging.QLogger;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
 import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.PossibleValueEnum;
 import com.kingsrook.qqq.backend.core.model.session.QSession;
@@ -51,6 +52,8 @@ import com.kingsrook.qqq.backend.core.model.session.QSession;
  *******************************************************************************/
 public class ValueUtils
 {
+   private static final QLogger LOG = QLogger.getLogger(ValueUtils.class);
+
    private static final DateTimeFormatter dateTimeFormatter_yyyyMMddWithDashes = DateTimeFormatter.ofPattern("yyyy-MM-dd");
    private static final DateTimeFormatter dateTimeFormatter_MdyyyyWithSlashes  = DateTimeFormatter.ofPattern("M/d/yyyy");
    private static final DateTimeFormatter dateTimeFormatter_yyyyMMdd           = DateTimeFormatter.ofPattern("yyyyMMdd");
@@ -931,4 +934,48 @@ public class ValueUtils
       return (ZoneId.of(QContext.getQInstance().getDefaultTimeZoneId()));
    }
 
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   public static QFieldType inferQFieldTypeFromValue(Serializable value, QFieldType defaultIfCannotInfer)
+   {
+      if(value instanceof String)
+      {
+         return QFieldType.STRING;
+      }
+      else if(value instanceof Integer)
+      {
+         return QFieldType.INTEGER;
+      }
+      else if(value instanceof Long)
+      {
+         return QFieldType.LONG;
+      }
+      else if(value instanceof BigDecimal)
+      {
+         return QFieldType.DECIMAL;
+      }
+      else if(value instanceof Boolean)
+      {
+         return QFieldType.BOOLEAN;
+      }
+      else if(value instanceof Instant)
+      {
+         return QFieldType.DATE_TIME;
+      }
+      else if(value instanceof LocalDate)
+      {
+         return QFieldType.DATE;
+      }
+      else if(value instanceof LocalTime)
+      {
+         return QFieldType.TIME;
+      }
+
+      LOG.debug("Could not infer QFieldType from value [" + (value == null ? "null" : value.getClass().getSimpleName()) + "]");
+
+      return defaultIfCannotInfer;
+   }
 }

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/audits/AuditActionTest.java

@@ -30,6 +30,8 @@ import java.util.Optional;
 import com.kingsrook.qqq.backend.core.BaseTest;
 import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.logging.QCollectingLogger;
+import com.kingsrook.qqq.backend.core.logging.QLogger;
 import com.kingsrook.qqq.backend.core.model.actions.audits.AuditInput;
 import com.kingsrook.qqq.backend.core.model.actions.audits.AuditSingleInput;
 import com.kingsrook.qqq.backend.core.model.audits.AuditsMetaDataProvider;
@@ -40,6 +42,7 @@ import com.kingsrook.qqq.backend.core.model.session.QUser;
 import com.kingsrook.qqq.backend.core.processes.utils.GeneralProcessUtils;
 import com.kingsrook.qqq.backend.core.utils.TestUtils;
 import com.kingsrook.qqq.backend.core.utils.collections.MapBuilder;
+import org.junit.jupiter.api.Disabled;
 import org.junit.jupiter.api.Test;
 import static org.assertj.core.api.Assertions.assertThat;
 import static org.junit.jupiter.api.Assertions.assertEquals;
@@ -83,6 +86,7 @@ class AuditActionTest extends BaseTest
     **
     *******************************************************************************/
    @Test
+   @Disabled("this behavior has been changed to just log... should this be a setting?")
    void testFailWithoutSecurityKey() throws QException
    {
       QInstance qInstance = TestUtils.defineInstance();
@@ -117,6 +121,50 @@ class AuditActionTest extends BaseTest
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testLogWithoutSecurityKey() throws QException
+   {
+      int recordId = 1701;
+
+      QInstance qInstance = TestUtils.defineInstance();
+      new AuditsMetaDataProvider().defineAll(qInstance, TestUtils.MEMORY_BACKEND_NAME, null);
+
+      String userName = "John Doe";
+      QContext.init(qInstance, new QSession().withUser(new QUser().withFullName(userName)));
+
+      QCollectingLogger collectingLogger = QLogger.activateCollectingLoggerForClass(AuditAction.class);
+      AuditAction.execute(TestUtils.TABLE_NAME_ORDER, recordId, Map.of(), "Test Audit");
+
+      ///////////////////////////////////////////////////////////////////
+      // it should not throw, but it should also not insert the audit. //
+      ///////////////////////////////////////////////////////////////////
+      Optional<QRecord> auditRecord = GeneralProcessUtils.getRecordByField("audit", "recordId", recordId);
+      assertTrue(auditRecord.isPresent());
+
+      assertThat(collectingLogger.getCollectedMessages()).anyMatch(m -> m.getMessage().contains("Missing securityKeyValue in audit request"));
+      collectingLogger.clear();
+
+      ////////////////////////////////////////////////////////////////////////////////////////////////
+      // try again with a null value in the key - that should be ok - as at least you were thinking //
+      // about the key and put in SOME value (null has its own semantics in security keys)          //
+      ////////////////////////////////////////////////////////////////////////////////////////////////
+      Map<String, Serializable> securityKeys = new HashMap<>();
+      securityKeys.put(TestUtils.SECURITY_KEY_TYPE_STORE, null);
+      AuditAction.execute(TestUtils.TABLE_NAME_ORDER, recordId, securityKeys, "Test Audit");
+
+      /////////////////////////////////////
+      // now the audit should be stored. //
+      /////////////////////////////////////
+      auditRecord = GeneralProcessUtils.getRecordByField("audit", "recordId", recordId);
+      assertTrue(auditRecord.isPresent());
+      assertThat(collectingLogger.getCollectedMessages()).noneMatch(m -> m.getMessage().contains("Missing securityKeyValue in audit request"));
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/
@@ -180,7 +228,7 @@ class AuditActionTest extends BaseTest
       QRecord auditRecord = GeneralProcessUtils.getRecordByFieldOrElseThrow("audit", "recordId", recordId1);
       assertEquals("Test Audit", auditRecord.getValueString("message"));
 
-      List<QRecord> auditDetails = GeneralProcessUtils.getRecordListByField("auditDetail", "auditId", auditRecord.getValue("id"));
+      List<QRecord> auditDetails = GeneralProcessUtils.getRecordListByField("auditDetail", "auditId", auditRecord.getValueLong("id"));
       assertEquals(2, auditDetails.size());
       assertThat(auditDetails).anyMatch(r -> r.getValueString("message").equals("Detail1"));
       assertThat(auditDetails).anyMatch(r -> r.getValueString("message").equals("Detail2"));
@@ -188,13 +236,13 @@ class AuditActionTest extends BaseTest
       auditRecord = GeneralProcessUtils.getRecordByFieldOrElseThrow("audit", "recordId", recordId2);
       assertEquals("Test Another Audit", auditRecord.getValueString("message"));
       assertEquals(47, auditRecord.getValueInteger(TestUtils.SECURITY_KEY_TYPE_STORE));
-      auditDetails = GeneralProcessUtils.getRecordListByField("auditDetail", "auditId", auditRecord.getValue("id"));
+      auditDetails = GeneralProcessUtils.getRecordListByField("auditDetail", "auditId", auditRecord.getValueLong("id"));
       assertEquals(0, auditDetails.size());
 
       auditRecord = GeneralProcessUtils.getRecordByFieldOrElseThrow("audit", "recordId", recordId3);
       assertEquals("Audit 3", auditRecord.getValueString("message"));
       assertEquals(42, auditRecord.getValueInteger(TestUtils.SECURITY_KEY_TYPE_STORE));
-      auditDetails = GeneralProcessUtils.getRecordListByField("auditDetail", "auditId", auditRecord.getValue("id"));
+      auditDetails = GeneralProcessUtils.getRecordListByField("auditDetail", "auditId", auditRecord.getValueLong("id"));
       assertEquals(1, auditDetails.size());
       assertThat(auditDetails).anyMatch(r -> r.getValueString("message").equals("Detail3"));
    }

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/tables/QQueryFilterTest.java

@@ -29,6 +29,8 @@ import java.util.Map;
 import com.kingsrook.qqq.backend.core.BaseTest;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.exceptions.QUserFacingException;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.CriteriaMissingInputValueBehavior;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.FilterUseCase;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.expressions.AbstractFilterExpression;
@@ -36,9 +38,12 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.query.expressions.Fil
 import org.junit.jupiter.api.Test;
 import static com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator.BETWEEN;
 import static com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator.EQUALS;
+import static com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator.FALSE;
 import static com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator.IS_BLANK;
+import static com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator.TRUE;
 import static org.assertj.core.api.Assertions.assertThatThrownBy;
 import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNull;
 
 
 /*******************************************************************************
@@ -140,4 +145,230 @@ class QQueryFilterTest extends BaseTest
       assertEquals("joinTableSomeFieldIdEquals", fve7.getVariableName());
    }
 
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testInterpretValueVariableExpressionNotFoundUseCases() throws QException
+   {
+      Map<String, Serializable> inputValues = new HashMap<>();
+
+      AbstractFilterExpression<Serializable> expression = new FilterVariableExpression()
+         .withVariableName("clientId");
+
+      ////////////////////////////////////////
+      // Control - where the value IS found //
+      ////////////////////////////////////////
+      inputValues.put("clientId", 47);
+      {
+         QQueryFilter filter = new QQueryFilter(new QFilterCriteria("id", EQUALS, expression));
+         filter.interpretValues(inputValues);
+         assertEquals(47, filter.getCriteria().get(0).getValues().get(0));
+         assertEquals(EQUALS, filter.getCriteria().get(0).getOperator());
+      }
+
+      //////////////////////////////////////////////////////
+      // now - remove the value for the next set of cases //
+      //////////////////////////////////////////////////////
+      inputValues.remove("clientId");
+
+      ////////////////////////////////////////////////////////////////////////////////////////////////
+      // a use-case that says to remove-from-filter, which, means translate to a criteria of "TRUE" //
+      ////////////////////////////////////////////////////////////////////////////////////////////////
+      {
+         QQueryFilter filter = new QQueryFilter(new QFilterCriteria("id", EQUALS, expression));
+         filter.interpretValues(inputValues, new RemoveFromFilterUseCase());
+         assertEquals(0, filter.getCriteria().get(0).getValues().size());
+         assertEquals(TRUE, filter.getCriteria().get(0).getOperator());
+      }
+
+      //////////////////////////////////////////////////////////////////////////////////////////////
+      // a use-case that says to make-no-matches, which, means translate to a criteria of "FALSE" //
+      //////////////////////////////////////////////////////////////////////////////////////////////
+      {
+         QQueryFilter filter = new QQueryFilter(new QFilterCriteria("id", EQUALS, expression));
+         filter.interpretValues(inputValues, new MakeNoMatchesUseCase());
+         assertEquals(0, filter.getCriteria().get(0).getValues().size());
+         assertEquals(FALSE, filter.getCriteria().get(0).getOperator());
+      }
+
+      ///////////////////////////////////////////
+      // a use-case that says to treat as null //
+      ///////////////////////////////////////////
+      {
+         QQueryFilter filter = new QQueryFilter(new QFilterCriteria("id", EQUALS, expression));
+         filter.interpretValues(inputValues, new InterpretAsNullValueUseCase());
+         assertNull(filter.getCriteria().get(0).getValues().get(0));
+         assertEquals(EQUALS, filter.getCriteria().get(0).getOperator());
+      }
+
+      ///////////////////////////////////
+      // a use-case that says to throw //
+      ///////////////////////////////////
+      {
+         QQueryFilter filter = new QQueryFilter(new QFilterCriteria("id", EQUALS, expression));
+         assertThatThrownBy(() -> filter.interpretValues(inputValues, new ThrowExceptionUseCase()))
+            .isInstanceOf(QUserFacingException.class)
+            .hasMessageContaining("Missing value for variable: clientId");
+      }
+
+      //////////////////////////////////////////////////////////
+      // verify that empty-string is treated as not-found too //
+      //////////////////////////////////////////////////////////
+      inputValues.put("clientId", "");
+      {
+         QQueryFilter filter = new QQueryFilter(new QFilterCriteria("id", EQUALS, expression));
+         assertThatThrownBy(() -> filter.interpretValues(inputValues, new ThrowExceptionUseCase()))
+            .isInstanceOf(QUserFacingException.class)
+            .hasMessageContaining("Missing value for variable: clientId");
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testInterpretValueStringStyleNotFoundUseCases() throws QException
+   {
+      Map<String, Serializable> inputValues = new HashMap<>();
+
+      ////////////////////////////////////////
+      // Control - where the value IS found //
+      ////////////////////////////////////////
+      inputValues.put("clientId", 47);
+      {
+         QQueryFilter filter = new QQueryFilter(new QFilterCriteria("id", EQUALS, "${input.clientId}"));
+         filter.interpretValues(inputValues);
+         assertEquals(47, filter.getCriteria().get(0).getValues().get(0));
+         assertEquals(EQUALS, filter.getCriteria().get(0).getOperator());
+      }
+
+      //////////////////////////////////////////////////////
+      // now - remove the value for the next set of cases //
+      //////////////////////////////////////////////////////
+      inputValues.remove("clientId");
+
+      ////////////////////////////////////////////////////////////////////////////////////////////////
+      // a use-case that says to remove-from-filter, which, means translate to a criteria of "TRUE" //
+      ////////////////////////////////////////////////////////////////////////////////////////////////
+      {
+         QQueryFilter filter = new QQueryFilter(new QFilterCriteria("id", EQUALS, "${input.clientId}"));
+         filter.interpretValues(inputValues, new RemoveFromFilterUseCase());
+         assertEquals(0, filter.getCriteria().get(0).getValues().size());
+         assertEquals(TRUE, filter.getCriteria().get(0).getOperator());
+      }
+
+      //////////////////////////////////////////////////////////////////////////////////////////////
+      // a use-case that says to make-no-matches, which, means translate to a criteria of "FALSE" //
+      //////////////////////////////////////////////////////////////////////////////////////////////
+      {
+         QQueryFilter filter = new QQueryFilter(new QFilterCriteria("id", EQUALS, "${input.clientId}"));
+         filter.interpretValues(inputValues, new MakeNoMatchesUseCase());
+         assertEquals(0, filter.getCriteria().get(0).getValues().size());
+         assertEquals(FALSE, filter.getCriteria().get(0).getOperator());
+      }
+
+      ///////////////////////////////////////////
+      // a use-case that says to treat as null //
+      ///////////////////////////////////////////
+      {
+         QQueryFilter filter = new QQueryFilter(new QFilterCriteria("id", EQUALS, "${input.clientId}"));
+         filter.interpretValues(inputValues, new InterpretAsNullValueUseCase());
+         assertNull(filter.getCriteria().get(0).getValues().get(0));
+         assertEquals(EQUALS, filter.getCriteria().get(0).getOperator());
+      }
+
+      ///////////////////////////////////
+      // a use-case that says to throw //
+      ///////////////////////////////////
+      {
+         QQueryFilter filter = new QQueryFilter(new QFilterCriteria("id", EQUALS, "${input.clientId}"));
+         assertThatThrownBy(() -> filter.interpretValues(inputValues, new ThrowExceptionUseCase()))
+            .isInstanceOf(QUserFacingException.class)
+            .hasMessageContaining("Missing value for criteria on field: id");
+      }
+
+      //////////////////////////////////////////////////////////
+      // verify that empty-string is treated as not-found too //
+      //////////////////////////////////////////////////////////
+      inputValues.put("clientId", "");
+      {
+         QQueryFilter filter = new QQueryFilter(new QFilterCriteria("id", EQUALS, "${input.clientId}"));
+         assertThatThrownBy(() -> filter.interpretValues(inputValues, new ThrowExceptionUseCase()))
+            .isInstanceOf(QUserFacingException.class)
+            .hasMessageContaining("Missing value for criteria on field: id");
+      }
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   private static class RemoveFromFilterUseCase implements FilterUseCase
+   {
+      /***************************************************************************
+       **
+       ***************************************************************************/
+      @Override
+      public CriteriaMissingInputValueBehavior getDefaultCriteriaMissingInputValueBehavior()
+      {
+         return CriteriaMissingInputValueBehavior.REMOVE_FROM_FILTER;
+      }
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   private static class MakeNoMatchesUseCase implements FilterUseCase
+   {
+      /***************************************************************************
+       **
+       ***************************************************************************/
+      @Override
+      public CriteriaMissingInputValueBehavior getDefaultCriteriaMissingInputValueBehavior()
+      {
+         return CriteriaMissingInputValueBehavior.MAKE_NO_MATCHES;
+      }
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   private static class InterpretAsNullValueUseCase implements FilterUseCase
+   {
+      /***************************************************************************
+       **
+       ***************************************************************************/
+      @Override
+      public CriteriaMissingInputValueBehavior getDefaultCriteriaMissingInputValueBehavior()
+      {
+         return CriteriaMissingInputValueBehavior.INTERPRET_AS_NULL_VALUE;
+      }
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   private static class ThrowExceptionUseCase implements FilterUseCase
+   {
+      /***************************************************************************
+       **
+       ***************************************************************************/
+      @Override
+      public CriteriaMissingInputValueBehavior getDefaultCriteriaMissingInputValueBehavior()
+      {
+         return CriteriaMissingInputValueBehavior.THROW_EXCEPTION;
+      }
+   }
 }

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/tables/QueryActionTest.java

@@ -24,6 +24,8 @@ package com.kingsrook.qqq.backend.core.actions.tables;
 
 import java.io.Serializable;
 import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.LinkedHashSet;
 import java.util.List;
 import java.util.Set;
 import com.kingsrook.qqq.backend.core.BaseTest;
@@ -38,6 +40,7 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryJoin;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
 import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
@@ -54,6 +57,7 @@ import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
 import com.kingsrook.qqq.backend.core.utils.TestUtils;
 import org.junit.jupiter.api.Test;
 import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertFalse;
 import static org.junit.jupiter.api.Assertions.assertNotNull;
@@ -541,4 +545,89 @@ class QueryActionTest extends BaseTest
       assertEquals(1, QueryAction.execute(TestUtils.TABLE_NAME_SHAPE, new QQueryFilter(new QFilterCriteria("name", QCriteriaOperator.EQUALS, "SqUaRe"))).size());
    }
 
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testValidateFieldNamesToInclude() throws QException
+   {
+      /////////////////////////////
+      // cases that do not throw //
+      /////////////////////////////
+      QueryAction.validateFieldNamesToInclude(new QueryInput(TestUtils.TABLE_NAME_PERSON)
+         .withFieldNamesToInclude(null));
+
+      QueryAction.validateFieldNamesToInclude(new QueryInput(TestUtils.TABLE_NAME_PERSON)
+         .withFieldNamesToInclude(Set.of("id")));
+
+      QueryAction.validateFieldNamesToInclude(new QueryInput(TestUtils.TABLE_NAME_PERSON)
+         .withFieldNamesToInclude(Set.of("id", "firstName", "lastName", "birthDate", "modifyDate", "createDate")));
+
+      QueryAction.validateFieldNamesToInclude(new QueryInput(TestUtils.TABLE_NAME_PERSON)
+         .withQueryJoin(new QueryJoin(TestUtils.TABLE_NAME_SHAPE).withSelect(true))
+         .withFieldNamesToInclude(Set.of("id", "firstName", "lastName", "birthDate", "modifyDate", "createDate")));
+
+      QueryAction.validateFieldNamesToInclude(new QueryInput(TestUtils.TABLE_NAME_PERSON)
+         .withQueryJoin(new QueryJoin(TestUtils.TABLE_NAME_SHAPE).withSelect(true))
+         .withFieldNamesToInclude(Set.of("id", "firstName", TestUtils.TABLE_NAME_SHAPE + ".id", TestUtils.TABLE_NAME_SHAPE + ".noOfSides")));
+
+      QueryAction.validateFieldNamesToInclude(new QueryInput(TestUtils.TABLE_NAME_PERSON)
+         .withQueryJoin(new QueryJoin(TestUtils.TABLE_NAME_SHAPE).withSelect(true).withAlias("s"))
+         .withFieldNamesToInclude(Set.of("id", "s.id", "s.noOfSides")));
+
+      //////////////////////////
+      // cases that do throw! //
+      //////////////////////////
+      assertThatThrownBy(() -> QueryAction.validateFieldNamesToInclude(new QueryInput(TestUtils.TABLE_NAME_PERSON)
+         .withFieldNamesToInclude(new HashSet<>())))
+         .hasMessageContaining("empty set of fieldNamesToInclude");
+
+      assertThatThrownBy(() -> QueryAction.validateFieldNamesToInclude(new QueryInput(TestUtils.TABLE_NAME_PERSON)
+         .withFieldNamesToInclude(Set.of("notAField"))))
+         .hasMessageContaining("1 unrecognized field name: notAField");
+
+      assertThatThrownBy(() -> QueryAction.validateFieldNamesToInclude(new QueryInput(TestUtils.TABLE_NAME_PERSON)
+         .withFieldNamesToInclude(new LinkedHashSet<>(List.of("notAField", "alsoNot")))))
+         .hasMessageContaining("2 unrecognized field names: notAField,alsoNot");
+
+      assertThatThrownBy(() -> QueryAction.validateFieldNamesToInclude(new QueryInput(TestUtils.TABLE_NAME_PERSON)
+         .withFieldNamesToInclude(new LinkedHashSet<>(List.of("notAField", "alsoNot", "join.not")))))
+         .hasMessageContaining("3 unrecognized field names: notAField,alsoNot,join.not");
+
+      assertThatThrownBy(() -> QueryAction.validateFieldNamesToInclude(new QueryInput(TestUtils.TABLE_NAME_PERSON)
+         .withQueryJoin(new QueryJoin(TestUtils.TABLE_NAME_SHAPE)) // oops, didn't select it!
+         .withFieldNamesToInclude(new LinkedHashSet<>(List.of("id", "firstName", TestUtils.TABLE_NAME_SHAPE + ".id", TestUtils.TABLE_NAME_SHAPE + ".noOfSides")))))
+         .hasMessageContaining("2 unrecognized field names: shape.id,shape.noOfSides");
+
+      assertThatThrownBy(() -> QueryAction.validateFieldNamesToInclude(new QueryInput(TestUtils.TABLE_NAME_PERSON)
+         .withQueryJoin(new QueryJoin(TestUtils.TABLE_NAME_SHAPE).withSelect(true))
+         .withFieldNamesToInclude(Set.of(TestUtils.TABLE_NAME_SHAPE + ".noField"))))
+         .hasMessageContaining("1 unrecognized field name: shape.noField");
+
+      assertThatThrownBy(() -> QueryAction.validateFieldNamesToInclude(new QueryInput(TestUtils.TABLE_NAME_PERSON)
+         .withQueryJoin(new QueryJoin(TestUtils.TABLE_NAME_SHAPE).withSelect(true).withAlias("s")) // oops, not using alias
+         .withFieldNamesToInclude(new LinkedHashSet<>(List.of("id", "firstName", TestUtils.TABLE_NAME_SHAPE + ".id", "s.noOfSides")))))
+         .hasMessageContaining("1 unrecognized field name: shape.id");
+
+      assertThatThrownBy(() -> QueryAction.validateFieldNamesToInclude(new QueryInput(TestUtils.TABLE_NAME_PERSON)
+         .withQueryJoin(new QueryJoin(TestUtils.TABLE_NAME_SHAPE).withSelect(true))
+         .withFieldNamesToInclude(new LinkedHashSet<>(List.of("id", "firstName", TestUtils.TABLE_NAME_SHAPE + ".id", "noOfSides")))))
+         .hasMessageContaining("1 unrecognized field name: noOfSides");
+
+      assertThatThrownBy(() -> QueryAction.validateFieldNamesToInclude(new QueryInput(TestUtils.TABLE_NAME_PERSON)
+         .withQueryJoin(new QueryJoin("noJoinTable").withSelect(true))
+         .withFieldNamesToInclude(Set.of("noJoinTable.id"))))
+         .hasMessageContaining("1 unrecognized field name: noJoinTable.id");
+
+      assertThatThrownBy(() -> QueryAction.validateFieldNamesToInclude(new QueryInput(TestUtils.TABLE_NAME_PERSON)
+         .withFieldNamesToInclude(Set.of("noJoin.id"))))
+         .hasMessageContaining("1 unrecognized field name: noJoin.id");
+
+      assertThatThrownBy(() -> QueryAction.validateFieldNamesToInclude(new QueryInput(TestUtils.TABLE_NAME_PERSON)
+         .withFieldNamesToInclude(Set.of("noDb.noJoin.id"))))
+         .hasMessageContaining("1 unrecognized field name: noDb.noJoin.id");
+   }
+
 }

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidatorTest.java

@@ -22,6 +22,7 @@
 package com.kingsrook.qqq.backend.core.instances;
 
 
+import java.io.Serializable;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashMap;
@@ -38,6 +39,8 @@ import com.kingsrook.qqq.backend.core.actions.dashboard.PersonsByCreateDateBarCh
 import com.kingsrook.qqq.backend.core.actions.dashboard.widgets.AbstractWidgetRenderer;
 import com.kingsrook.qqq.backend.core.actions.dashboard.widgets.ParentWidgetRenderer;
 import com.kingsrook.qqq.backend.core.actions.processes.CancelProcessActionTest;
+import com.kingsrook.qqq.backend.core.actions.reporting.RecordPipe;
+import com.kingsrook.qqq.backend.core.actions.reporting.customizers.ReportCustomRecordSourceInterface;
 import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.exceptions.QInstanceValidationException;
@@ -45,6 +48,7 @@ import com.kingsrook.qqq.backend.core.instances.validation.plugins.AlwaysFailsPr
 import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLineInterface;
 import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
 import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
+import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy;
@@ -1712,7 +1716,7 @@ public class QInstanceValidatorTest extends BaseTest
    @Test
    void testReportDataSourceStaticDataSupplier()
    {
-      assertValidationFailureReasons((qInstance) -> qInstance.getReport(TestUtils.REPORT_NAME_SHAPES_PERSON).getDataSources().get(0).withStaticDataSupplier(new QCodeReference()),
+      assertValidationFailureReasons((qInstance) -> qInstance.getReport(TestUtils.REPORT_NAME_SHAPES_PERSON).getDataSources().get(0).withStaticDataSupplier(new QCodeReference(TestReportStaticDataSupplier.class)),
          "has both a sourceTable and a staticDataSupplier");
 
       assertValidationFailureReasons((qInstance) ->
@@ -1720,16 +1724,43 @@ public class QInstanceValidatorTest extends BaseTest
          QReportDataSource dataSource = qInstance.getReport(TestUtils.REPORT_NAME_SHAPES_PERSON).getDataSources().get(0);
          dataSource.setSourceTable(null);
          dataSource.setStaticDataSupplier(new QCodeReference(null, QCodeType.JAVA));
-      },
-         "missing a code reference name");
+      }, "missing a code reference name");
 
       assertValidationFailureReasons((qInstance) ->
       {
          QReportDataSource dataSource = qInstance.getReport(TestUtils.REPORT_NAME_SHAPES_PERSON).getDataSources().get(0);
          dataSource.setSourceTable(null);
          dataSource.setStaticDataSupplier(new QCodeReference(ArrayList.class));
-      },
-         "is not of the expected type");
+      }, "is not of the expected type");
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testReportDataSourceCustomRecordSource()
+   {
+      assertValidationFailureReasons((qInstance) -> qInstance.getReport(TestUtils.REPORT_NAME_SHAPES_PERSON).getDataSources().get(0)
+            .withSourceTable(null)
+            .withStaticDataSupplier(new QCodeReference(TestReportStaticDataSupplier.class))
+            .withCustomRecordSource(new QCodeReference(TestReportCustomRecordSource.class)),
+         "has both a staticDataSupplier and a customRecordSource");
+
+      assertValidationFailureReasons((qInstance) ->
+      {
+         QReportDataSource dataSource = qInstance.getReport(TestUtils.REPORT_NAME_SHAPES_PERSON).getDataSources().get(0);
+         dataSource.setSourceTable(null);
+         dataSource.setCustomRecordSource(new QCodeReference(null, QCodeType.JAVA));
+      }, "missing a code reference name");
+
+      assertValidationFailureReasons((qInstance) ->
+      {
+         QReportDataSource dataSource = qInstance.getReport(TestUtils.REPORT_NAME_SHAPES_PERSON).getDataSources().get(0);
+         dataSource.setSourceTable(null);
+         dataSource.setCustomRecordSource(new QCodeReference(ArrayList.class));
+      }, "is not of the expected type");
    }
 
 
@@ -2371,5 +2402,38 @@ public class QInstanceValidatorTest extends BaseTest
     *******************************************************************************/
    public static class ValidAuthCustomizer implements QAuthenticationModuleCustomizerInterface {}
 
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   public static class TestReportStaticDataSupplier implements Supplier<List<List<Serializable>>>
+   {
+
+      /***************************************************************************
+       **
+       ***************************************************************************/
+      @Override
+      public List<List<Serializable>> get()
+      {
+         return List.of();
+      }
+   }
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   public static class TestReportCustomRecordSource implements ReportCustomRecordSourceInterface
+   {
+
+      /***************************************************************************
+       **
+       ***************************************************************************/
+      @Override
+      public void execute(ReportInput reportInput, QReportDataSource reportDataSource, RecordPipe recordPipe) throws QException
+      {
+
+      }
+   }
 }
 

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/expressions/NowWithOffsetTest.java

@@ -22,11 +22,16 @@
 package com.kingsrook.qqq.backend.core.model.actions.tables.query.expressions;
 
 
+import java.time.Instant;
+import java.time.LocalDate;
 import java.time.temporal.ChronoUnit;
 import com.kingsrook.qqq.backend.core.BaseTest;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
 import org.assertj.core.data.Offset;
 import org.junit.jupiter.api.Test;
 import static org.assertj.core.api.Assertions.assertThat;
+import static org.junit.jupiter.api.Assertions.assertEquals;
 
 
 /*******************************************************************************
@@ -46,23 +51,39 @@ class NowWithOffsetTest extends BaseTest
    {
       long now = System.currentTimeMillis();
 
-      long oneWeekAgoMillis = NowWithOffset.minus(1, ChronoUnit.WEEKS).evaluate().toEpochMilli();
-      assertThat(oneWeekAgoMillis).isCloseTo(now - (7 * DAY_IN_MILLIS), Offset.offset(10_000L));
+      QFieldMetaData dateTimeField = new QFieldMetaData("myDateTime", QFieldType.DATE_TIME);
+      QFieldMetaData dateField     = new QFieldMetaData("myDate", QFieldType.DATE);
 
-      long oneWeekFromNowMillis = NowWithOffset.plus(2, ChronoUnit.WEEKS).evaluate().toEpochMilli();
-      assertThat(oneWeekFromNowMillis).isCloseTo(now + (14 * DAY_IN_MILLIS), Offset.offset(10_000L));
+      {
+         Offset<Long> allowedDiff            = Offset.offset(100L);
+         Offset<Long> allowedDiffPlusOneDay  = Offset.offset(100L + DAY_IN_MILLIS);
+         Offset<Long> allowedDiffPlusTwoDays = Offset.offset(100L + 2 * DAY_IN_MILLIS);
 
-      long oneMonthAgoMillis = NowWithOffset.minus(1, ChronoUnit.MONTHS).evaluate().toEpochMilli();
-      assertThat(oneMonthAgoMillis).isCloseTo(now - (30 * DAY_IN_MILLIS), Offset.offset(10_000L + 2 * DAY_IN_MILLIS));
+         long oneWeekAgoMillis = ((Instant) NowWithOffset.minus(1, ChronoUnit.WEEKS).evaluate(dateTimeField)).toEpochMilli();
+         assertThat(oneWeekAgoMillis).isCloseTo(now - (7 * DAY_IN_MILLIS), allowedDiff);
 
-      long oneMonthFromNowMillis = NowWithOffset.plus(2, ChronoUnit.MONTHS).evaluate().toEpochMilli();
-      assertThat(oneMonthFromNowMillis).isCloseTo(now + (60 * DAY_IN_MILLIS), Offset.offset(10_000L + 3 * DAY_IN_MILLIS));
+         long twoWeeksFromNowMillis = ((Instant) NowWithOffset.plus(2, ChronoUnit.WEEKS).evaluate(dateTimeField)).toEpochMilli();
+         assertThat(twoWeeksFromNowMillis).isCloseTo(now + (14 * DAY_IN_MILLIS), allowedDiff);
 
-      long oneYearAgoMillis = NowWithOffset.minus(1, ChronoUnit.YEARS).evaluate().toEpochMilli();
-      assertThat(oneYearAgoMillis).isCloseTo(now - (365 * DAY_IN_MILLIS), Offset.offset(10_000L + 2 * DAY_IN_MILLIS));
+         long oneMonthAgoMillis = ((Instant) NowWithOffset.minus(1, ChronoUnit.MONTHS).evaluate(dateTimeField)).toEpochMilli();
+         assertThat(oneMonthAgoMillis).isCloseTo(now - (30 * DAY_IN_MILLIS), allowedDiffPlusOneDay);
 
-      long oneYearFromNowMillis = NowWithOffset.plus(2, ChronoUnit.YEARS).evaluate().toEpochMilli();
-      assertThat(oneYearFromNowMillis).isCloseTo(now + (730 * DAY_IN_MILLIS), Offset.offset(10_000L + 3 * DAY_IN_MILLIS));
+         long twoMonthsFromNowMillis = ((Instant) NowWithOffset.plus(2, ChronoUnit.MONTHS).evaluate(dateTimeField)).toEpochMilli();
+         assertThat(twoMonthsFromNowMillis).isCloseTo(now + (60 * DAY_IN_MILLIS), allowedDiffPlusTwoDays);
+
+         long oneYearAgoMillis = ((Instant) NowWithOffset.minus(1, ChronoUnit.YEARS).evaluate(dateTimeField)).toEpochMilli();
+         assertThat(oneYearAgoMillis).isCloseTo(now - (365 * DAY_IN_MILLIS), allowedDiffPlusOneDay);
+
+         long twoYearsFromNowMillis = ((Instant) NowWithOffset.plus(2, ChronoUnit.YEARS).evaluate(dateTimeField)).toEpochMilli();
+         assertThat(twoYearsFromNowMillis).isCloseTo(now + (730 * DAY_IN_MILLIS), allowedDiffPlusTwoDays);
+      }
+
+      {
+         assertThat(NowWithOffset.minus(1, ChronoUnit.WEEKS).evaluate(dateField)).isInstanceOf(LocalDate.class);
+
+         assertEquals(LocalDate.now().minusDays(1), NowWithOffset.minus(1, ChronoUnit.DAYS).evaluate(dateField));
+         assertEquals(LocalDate.now().minusDays(7), NowWithOffset.minus(1, ChronoUnit.WEEKS).evaluate(dateField));
+      }
    }
 
 }

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/reports/BasicRunReportProcessTest.java

@@ -30,6 +30,7 @@ import com.kingsrook.qqq.backend.core.actions.reporting.GenerateReportActionTest
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput;
 import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessOutput;
+import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportFormat;
 import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
 import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportMetaData;
@@ -43,6 +44,7 @@ import static org.assertj.core.api.Assertions.assertThat;
  *******************************************************************************/
 class BasicRunReportProcessTest extends BaseTest
 {
+
    /*******************************************************************************
     **
     *******************************************************************************/
@@ -74,6 +76,18 @@ class BasicRunReportProcessTest extends BaseTest
       runProcessOutput = new RunProcessAction().execute(runProcessInput);
       assertThat(runProcessOutput.getProcessState().getNextStepName()).isPresent().get().isEqualTo(BasicRunReportProcess.STEP_NAME_ACCESS);
       assertThat(runProcessOutput.getValues()).containsKeys("downloadFileName", "serverFilePath");
+
+      ///////////////////////////////////
+      // assert we get xlsx by default //
+      ///////////////////////////////////
+      assertThat(runProcessOutput.getValueString("downloadFileName")).endsWith(".xlsx");
+
+      /////////////////////////////////////////////////////
+      // re-run, requesting CSV, then assert we get that //
+      /////////////////////////////////////////////////////
+      runProcessInput.addValue(BasicRunReportProcess.FIELD_REPORT_FORMAT, ReportFormat.CSV.name());
+      runProcessOutput = new RunProcessAction().execute(runProcessInput);
+      assertThat(runProcessOutput.getValueString("downloadFileName")).endsWith(".csv");
    }
 
 }

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/savedviews/SavedViewProcessTests.java

@@ -192,7 +192,6 @@ class SavedViewProcessTests extends BaseTest
     **
     *******************************************************************************/
    @Test
-   @SuppressWarnings("unchecked")
    void testNotFoundThrowsProperly() throws QException
    {
       QInstance qInstance = QContext.getQInstance();
@@ -245,4 +244,4 @@ class SavedViewProcessTests extends BaseTest
       }
    }
 
-}
+}

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/ValueUtilsTest.java

@@ -32,12 +32,14 @@ import java.time.LocalDateTime;
 import java.time.LocalTime;
 import java.time.Month;
 import java.time.ZoneId;
+import java.util.ArrayList;
 import java.util.Calendar;
 import java.util.GregorianCalendar;
 import com.kingsrook.qqq.backend.core.BaseTest;
 import com.kingsrook.qqq.backend.core.actions.automation.AutomationStatus;
 import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.exceptions.QValueException;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
 import com.kingsrook.qqq.backend.core.model.session.QSession;
 import org.junit.jupiter.api.Test;
 import static org.assertj.core.api.Assertions.assertThat;
@@ -333,4 +335,28 @@ class ValueUtilsTest extends BaseTest
       assertEquals(ZoneId.of("UTC-05:00"), ValueUtils.getSessionOrInstanceZoneId());
    }
 
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testInferQFieldTypeFromValue()
+   {
+      assertNull(ValueUtils.inferQFieldTypeFromValue(null, null));
+      assertNull(ValueUtils.inferQFieldTypeFromValue(new ArrayList<>(), null));
+      assertEquals(QFieldType.HTML, ValueUtils.inferQFieldTypeFromValue(new ArrayList<>(), QFieldType.HTML));
+
+      assertEquals(QFieldType.STRING, ValueUtils.inferQFieldTypeFromValue("value", null));
+      assertEquals(QFieldType.INTEGER, ValueUtils.inferQFieldTypeFromValue(1, null));
+      assertEquals(QFieldType.INTEGER, ValueUtils.inferQFieldTypeFromValue(Integer.valueOf("1"), null));
+      assertEquals(QFieldType.LONG, ValueUtils.inferQFieldTypeFromValue(10_000_000_000L, null));
+      assertEquals(QFieldType.DECIMAL, ValueUtils.inferQFieldTypeFromValue(BigDecimal.ZERO, null));
+      assertEquals(QFieldType.BOOLEAN, ValueUtils.inferQFieldTypeFromValue(true, null));
+      assertEquals(QFieldType.BOOLEAN, ValueUtils.inferQFieldTypeFromValue(Boolean.FALSE, null));
+      assertEquals(QFieldType.DATE_TIME, ValueUtils.inferQFieldTypeFromValue(Instant.now(), null));
+      assertEquals(QFieldType.DATE, ValueUtils.inferQFieldTypeFromValue(LocalDate.now(), null));
+      assertEquals(QFieldType.TIME, ValueUtils.inferQFieldTypeFromValue(LocalTime.now(), null));
+   }
+
 }

qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/actions/BaseAPIActionUtil.java

@@ -35,6 +35,7 @@ import java.util.Base64;
 import java.util.List;
 import java.util.Map;
 import java.util.concurrent.TimeUnit;
+import java.util.function.Supplier;
 import com.kingsrook.qqq.backend.core.actions.tables.GetAction;
 import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
 import com.kingsrook.qqq.backend.core.context.QContext;
@@ -64,6 +65,7 @@ import com.kingsrook.qqq.backend.core.model.session.QSession;
 import com.kingsrook.qqq.backend.core.model.statusmessages.SystemErrorStatusMessage;
 import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
 import com.kingsrook.qqq.backend.core.utils.JsonUtils;
+import com.kingsrook.qqq.backend.core.utils.Pair;
 import com.kingsrook.qqq.backend.core.utils.SleepUtils;
 import com.kingsrook.qqq.backend.core.utils.StringUtils;
 import com.kingsrook.qqq.backend.core.utils.ValueUtils;
@@ -87,6 +89,7 @@ import org.apache.http.client.methods.CloseableHttpResponse;
 import org.apache.http.client.methods.HttpDelete;
 import org.apache.http.client.methods.HttpEntityEnclosingRequestBase;
 import org.apache.http.client.methods.HttpGet;
+import org.apache.http.client.methods.HttpPatch;
 import org.apache.http.client.methods.HttpPost;
 import org.apache.http.client.methods.HttpPut;
 import org.apache.http.client.methods.HttpRequestBase;
@@ -118,10 +121,36 @@ public class BaseAPIActionUtil
 
 
    /***************************************************************************
-    **
+    ** enum of which HTTP Method the backend uses for Updates.
     ***************************************************************************/
    public enum UpdateHttpMethod
-   {PUT, POST}
+   {
+      PUT(HttpPut::new),
+      POST(HttpPost::new),
+      PATCH(HttpPatch::new);
+
+      private Supplier<HttpEntityEnclosingRequestBase> httpEntitySupplier;
+
+
+
+      /***************************************************************************
+       **
+       ***************************************************************************/
+      UpdateHttpMethod(Supplier<HttpEntityEnclosingRequestBase> httpEnttySupplier)
+      {
+         this.httpEntitySupplier = httpEnttySupplier;
+      }
+
+
+
+      /***************************************************************************
+       **
+       ***************************************************************************/
+      public HttpEntityEnclosingRequestBase newRequest()
+      {
+         return (this.httpEntitySupplier.get());
+      }
+   }
 
 
 
@@ -350,7 +379,9 @@ public class BaseAPIActionUtil
             {
                String                         paramString = buildQueryStringForUpdate(table, recordList);
                String                         url         = buildTableUrl(table) + paramString;
-               HttpEntityEnclosingRequestBase request     = getUpdateMethod().equals(UpdateHttpMethod.PUT) ? new HttpPut(url) : new HttpPost(url);
+               HttpEntityEnclosingRequestBase request     = getUpdateMethod().newRequest();
+
+               request.setURI(new URI(url));
                request.setEntity(recordsToEntity(table, recordList));
 
                QHttpResponse response = makeRequest(table, request);
@@ -688,60 +719,22 @@ public class BaseAPIActionUtil
     *******************************************************************************/
    public void setupAuthorizationInRequest(HttpRequestBase request) throws QException
    {
-      ///////////////////////////////////////////////////////////////////////////////////
-      // if backend specifies that it uses variants, look for that data in the session //
-      ///////////////////////////////////////////////////////////////////////////////////
-      if(backendMetaData.getUsesVariants())
-      {
-         QSession session = QContext.getQSession();
-         if(session.getBackendVariants() == null || !session.getBackendVariants().containsKey(backendMetaData.getVariantOptionsTableTypeValue()))
-         {
-            throw (new QException("Could not find Backend Variant information for Backend '" + backendMetaData.getName() + "'"));
-         }
-
-         Serializable variantId = session.getBackendVariants().get(backendMetaData.getVariantOptionsTableTypeValue());
-         GetInput     getInput  = new GetInput();
-         getInput.setShouldMaskPasswords(false);
-         getInput.setTableName(backendMetaData.getVariantOptionsTableName());
-         getInput.setPrimaryKey(variantId);
-         GetOutput getOutput = new GetAction().execute(getInput);
-
-         QRecord record = getOutput.getRecord();
-         if(record == null)
-         {
-            throw (new QException("Could not find Backend Variant in table " + backendMetaData.getVariantOptionsTableName() + " with id '" + variantId + "'"));
-         }
-
-         if(backendMetaData.getAuthorizationType().equals(AuthorizationType.BASIC_AUTH_USERNAME_PASSWORD))
-         {
-            request.setHeader("Authorization", getBasicAuthenticationHeader(record.getValueString(backendMetaData.getVariantOptionsTableUsernameField()), record.getValueString(backendMetaData.getVariantOptionsTablePasswordField())));
-         }
-         else if(backendMetaData.getAuthorizationType().equals(AuthorizationType.API_KEY_HEADER))
-         {
-            request.setHeader("API-Key", record.getValueString(backendMetaData.getVariantOptionsTableApiKeyField()));
-         }
-         else
-         {
-            throw (new IllegalArgumentException("Unexpected variant authorization type specified: " + backendMetaData.getAuthorizationType()));
-         }
-         return;
-      }
-
-      ///////////////////////////////////////////////////////////////////////////////////////////
-      // if not using variants, the authorization data will be in the backend meta data object //
-      ///////////////////////////////////////////////////////////////////////////////////////////
+      ///////////////////////////////////////////////////////////////////
+      // update the request based on the authorization type being used //
+      ///////////////////////////////////////////////////////////////////
       switch(backendMetaData.getAuthorizationType())
       {
-         case BASIC_AUTH_API_KEY -> request.setHeader("Authorization", getBasicAuthenticationHeader(backendMetaData.getApiKey()));
-         case BASIC_AUTH_USERNAME_PASSWORD -> request.setHeader("Authorization", getBasicAuthenticationHeader(backendMetaData.getUsername(), backendMetaData.getPassword()));
-         case API_KEY_HEADER -> request.setHeader("API-Key", backendMetaData.getApiKey());
-         case API_TOKEN -> request.setHeader("Authorization", "Token " + backendMetaData.getApiKey());
+         case BASIC_AUTH_API_KEY -> request.setHeader("Authorization", getBasicAuthenticationHeader(getApiKey()));
+         case BASIC_AUTH_USERNAME_PASSWORD ->
+         {
+            Pair<String, String> usernameAndPassword = getUsernameAndPassword();
+            request.setHeader("Authorization", getBasicAuthenticationHeader(usernameAndPassword.getA(), usernameAndPassword.getB()));
+         }
+         case API_KEY_HEADER -> request.setHeader("API-Key", getApiKey());
+         case API_TOKEN -> request.setHeader("Authorization", "Token " + getApiKey());
          case OAUTH2 -> request.setHeader("Authorization", "Bearer " + getOAuth2Token());
          case API_KEY_QUERY_PARAM -> addApiKeyQueryParamToRequest(request);
-         case CUSTOM ->
-         {
-            handleCustomAuthorization(request);
-         }
+         case CUSTOM -> handleCustomAuthorization(request);
          default -> throw new IllegalArgumentException("Unexpected authorization type: " + backendMetaData.getAuthorizationType());
       }
    }
@@ -755,8 +748,8 @@ public class BaseAPIActionUtil
    {
       try
       {
-         String uri = request.getURI().toString();
-         String pair = backendMetaData.getApiKeyQueryParamName() + "=" + backendMetaData.getApiKey();
+         String uri  = request.getURI().toString();
+         String pair = backendMetaData.getApiKeyQueryParamName() + "=" + getApiKey();
 
          ///////////////////////////////////////////////////////////////////////////////////
          // avoid re-adding the name=value pair if it's already there (e.g., for a retry) //
@@ -777,39 +770,106 @@ public class BaseAPIActionUtil
 
 
 
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   protected String getApiKey() throws QException
+   {
+      if(backendMetaData.getUsesVariants())
+      {
+         QRecord record = getVariantRecord();
+         return (record.getValueString(backendMetaData.getVariantOptionsTableApiKeyField()));
+      }
+
+      return (backendMetaData.getApiKey());
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   protected Pair<String, String> getUsernameAndPassword() throws QException
+   {
+      if(backendMetaData.getUsesVariants())
+      {
+         QRecord record = getVariantRecord();
+         return (Pair.of(record.getValueString(backendMetaData.getVariantOptionsTableUsernameField()), record.getValueString(backendMetaData.getVariantOptionsTablePasswordField())));
+      }
+
+      return (Pair.of(backendMetaData.getUsername(), backendMetaData.getPassword()));
+   }
+
+
+
+   /*******************************************************************************
+    ** For backends that use variants, look up the variant record (in theory, based
+    ** on an id in the session's backend variants map, then fetched from the backend's
+    ** variant options table.
+    *******************************************************************************/
+   protected QRecord getVariantRecord() throws QException
+   {
+      Serializable variantId = getVariantId();
+      GetInput     getInput  = new GetInput();
+      getInput.setShouldMaskPasswords(false);
+      getInput.setTableName(backendMetaData.getVariantOptionsTableName());
+      getInput.setPrimaryKey(variantId);
+      GetOutput getOutput = new GetAction().execute(getInput);
+
+      QRecord record = getOutput.getRecord();
+      if(record == null)
+      {
+         throw (new QException("Could not find Backend Variant in table " + backendMetaData.getVariantOptionsTableName() + " with id '" + variantId + "'"));
+      }
+      return record;
+   }
+
+
+
+   /*******************************************************************************
+    ** Get the variant id from the session for the backend.
+    *******************************************************************************/
+   protected Serializable getVariantId() throws QException
+   {
+      QSession session = QContext.getQSession();
+      if(session.getBackendVariants() == null || !session.getBackendVariants().containsKey(backendMetaData.getVariantOptionsTableTypeValue()))
+      {
+         throw (new QException("Could not find Backend Variant information for Backend '" + backendMetaData.getName() + "'"));
+      }
+      Serializable variantId = session.getBackendVariants().get(backendMetaData.getVariantOptionsTableTypeValue());
+      return variantId;
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/
-   public String getOAuth2Token() throws OAuthCredentialsException
+   public String getOAuth2Token() throws OAuthCredentialsException, QException
    {
+      //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      // define the key that will be used in the backend's customValues map, to stash the access token.                   //
+      // for non-variant backends, this is just a constant string.  But for variant-backends, append the variantId to it. //
+      //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      String accessTokenKey = "accessToken";
+      if(backendMetaData.getUsesVariants())
+      {
+         Serializable variantId = getVariantId();
+         accessTokenKey = accessTokenKey + ":" + variantId;
+      }
+
       ////////////////////////////////////////////////////////////////////////////////////////////////////////////////
       // check for the access token in the backend meta data.  if it's not there, then issue a request for a token. //
       // this is not generally meant to be put in the meta data by the app programmer - rather, we're just using    //
       // it as a "cheap & easy" way to "cache" the token within our process's memory...                             //
       ////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-      String  accessToken            = ValueUtils.getValueAsString(backendMetaData.getCustomValue("accessToken"));
-      Boolean setCredentialsInHeader = BooleanUtils.isTrue(ValueUtils.getValueAsBoolean(backendMetaData.getCustomValue("setCredentialsInHeader")));
+      String accessToken = ValueUtils.getValueAsString(backendMetaData.getCustomValue(accessTokenKey));
 
       if(!StringUtils.hasContent(accessToken))
       {
-         String fullURL  = backendMetaData.getBaseUrl() + "oauth/token";
-         String postBody = "grant_type=client_credentials";
-
-         if(!setCredentialsInHeader)
-         {
-            postBody += "&client_id=" + backendMetaData.getClientId() + "&client_secret=" + backendMetaData.getClientSecret();
-         }
-
          try(CloseableHttpClient client = HttpClients.custom().setConnectionManager(new PoolingHttpClientConnectionManager()).build())
          {
-            HttpPost request = new HttpPost(fullURL);
-            request.setEntity(new StringEntity(postBody, getCharsetForEntity()));
-
-            if(setCredentialsInHeader)
-            {
-               request.setHeader("Authorization", getBasicAuthenticationHeader(backendMetaData.getClientId(), backendMetaData.getClientSecret()));
-            }
-            request.setHeader("Content-Type", "application/x-www-form-urlencoded; charset=utf-8");
+            HttpRequestBase request = createOAuth2TokenRequest();
 
             HttpResponse response     = executeOAuthTokenRequest(client, request);
             int          statusCode   = response.getStatusLine().getStatusCode();
@@ -827,7 +887,7 @@ public class BaseAPIActionUtil
             ///////////////////////////////////////////////////////////////////////////////////////////////////
             // stash the access token in the backendMetaData, from which it will be used for future requests //
             ///////////////////////////////////////////////////////////////////////////////////////////////////
-            backendMetaData.withCustomValue("accessToken", accessToken);
+            backendMetaData.withCustomValue(accessTokenKey, accessToken);
          }
          catch(OAuthCredentialsException oce)
          {
@@ -846,6 +906,53 @@ public class BaseAPIActionUtil
 
 
 
+   /***************************************************************************
+    ** For doing OAuth2 authentication, create a request for a token.
+    ***************************************************************************/
+   protected HttpRequestBase createOAuth2TokenRequest() throws QException
+   {
+      String fullURL  = backendMetaData.getBaseUrl() + "oauth/token";
+      String postBody = "grant_type=client_credentials";
+
+      Pair<String, String> clientIdAndSecret = getClientIdAndSecret();
+      String               clientId          = clientIdAndSecret.getA();
+      String               clientSecret      = clientIdAndSecret.getB();
+
+      Boolean setCredentialsInHeader = BooleanUtils.isTrue(ValueUtils.getValueAsBoolean(backendMetaData.getCustomValue("setCredentialsInHeader")));
+      if(!setCredentialsInHeader)
+      {
+         postBody += "&client_id=" + clientId + "&client_secret=" + clientSecret;
+      }
+
+      HttpPost request = new HttpPost(fullURL);
+      request.setEntity(new StringEntity(postBody, getCharsetForEntity()));
+
+      if(setCredentialsInHeader)
+      {
+         request.setHeader("Authorization", getBasicAuthenticationHeader(clientId, clientSecret));
+      }
+      request.setHeader("Content-Type", "application/x-www-form-urlencoded; charset=utf-8");
+      return request;
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   protected Pair<String, String> getClientIdAndSecret() throws QException
+   {
+      if(backendMetaData.getUsesVariants())
+      {
+         QRecord record = getVariantRecord();
+         return (Pair.of(record.getValueString(backendMetaData.getVariantOptionsTableClientIdField()), record.getValueString(backendMetaData.getVariantOptionsTableClientSecretField())));
+      }
+
+      return (Pair.of(backendMetaData.getClientId(), backendMetaData.getClientSecret()));
+   }
+
+
+
    /*******************************************************************************
     ** Let a subclass change what charset to use for entities (bodies) being posted/put/etc.
     *******************************************************************************/
@@ -859,6 +966,18 @@ public class BaseAPIActionUtil
    /*******************************************************************************
     ** one-line method, factored out so mock/tests can override
     *******************************************************************************/
+   protected CloseableHttpResponse executeOAuthTokenRequest(CloseableHttpClient client, HttpRequestBase request) throws IOException
+   {
+      return client.execute(request);
+   }
+
+
+
+   /*******************************************************************************
+    ** one-line method, factored out so mock/tests can override
+    ** Deprecated, in favor of more generic overload that takes HttpRequestBase
+    *******************************************************************************/
+   @Deprecated
    protected CloseableHttpResponse executeOAuthTokenRequest(CloseableHttpClient client, HttpPost request) throws IOException
    {
       return client.execute(request);
@@ -1048,6 +1167,16 @@ public class BaseAPIActionUtil
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   protected void logRequestDetails(QTableMetaData table, HttpRequestBase request) throws QException
+   {
+      LOG.info("Making [" + request.getMethod() + "] request to URL [" + request.getURI() + "] on table [" + table.getName() + "].");
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/
@@ -1073,7 +1202,7 @@ public class BaseAPIActionUtil
             setupContentTypeInRequest(request);
             setupAdditionalHeaders(request);
 
-            LOG.info("Making [" + request.getMethod() + "] request to URL [" + request.getURI() + "] on table [" + table.getName() + "].");
+            logRequestDetails(table, request);
             if("POST".equals(request.getMethod()))
             {
                LOG.info("POST contents [" + ((HttpPost) request).getEntity().toString() + "]");

qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/model/OutboundAPILogHeader.java

@@ -1,316 +0,0 @@
-/*
- * QQQ - Low-code Application Framework for Engineers.
- * Copyright (C) 2021-2023.  Kingsrook, LLC
- * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
- * contact@kingsrook.com
- * https://github.com/Kingsrook/
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program.  If not, see <https://www.gnu.org/licenses/>.
- */
-
-package com.kingsrook.qqq.backend.module.api.model;
-
-
-import java.time.Instant;
-import java.util.List;
-import com.kingsrook.qqq.backend.core.exceptions.QException;
-import com.kingsrook.qqq.backend.core.model.data.QAssociation;
-import com.kingsrook.qqq.backend.core.model.data.QField;
-import com.kingsrook.qqq.backend.core.model.data.QRecord;
-import com.kingsrook.qqq.backend.core.model.data.QRecordEntity;
-
-
-/*******************************************************************************
- ** Entity bean for OutboundApiLog table
- *******************************************************************************/
-public class OutboundAPILogHeader extends QRecordEntity
-{
-   public static final String TABLE_NAME = "outboundApiLogHeader";
-
-   @QField(isEditable = false)
-   private Integer id;
-
-   @QField()
-   private Instant timestamp;
-
-   @QField(possibleValueSourceName = "outboundApiMethod")
-   private String method;
-
-   @QField(possibleValueSourceName = "outboundApiStatusCode")
-   private Integer statusCode;
-
-   @QField(label = "URL")
-   private String url;
-
-   @QAssociation(name = OutboundAPILogRequest.TABLE_NAME)
-   private List<OutboundAPILogRequest> outboundAPILogRequestList;
-
-   @QAssociation(name = OutboundAPILogResponse.TABLE_NAME)
-   private List<OutboundAPILogResponse> outboundAPILogResponseList;
-
-
-
-   /*******************************************************************************
-    ** Constructor
-    **
-    *******************************************************************************/
-   public OutboundAPILogHeader()
-   {
-   }
-
-
-
-   /*******************************************************************************
-    ** Constructor
-    **
-    *******************************************************************************/
-   public OutboundAPILogHeader(QRecord qRecord) throws QException
-   {
-      populateFromQRecord(qRecord);
-   }
-
-
-
-   /*******************************************************************************
-    ** Getter for id
-    **
-    *******************************************************************************/
-   public Integer getId()
-   {
-      return id;
-   }
-
-
-
-   /*******************************************************************************
-    ** Setter for id
-    **
-    *******************************************************************************/
-   public void setId(Integer id)
-   {
-      this.id = id;
-   }
-
-
-
-   /*******************************************************************************
-    ** Fluent setter for id
-    **
-    *******************************************************************************/
-   public OutboundAPILogHeader withId(Integer id)
-   {
-      this.id = id;
-      return (this);
-   }
-
-
-
-   /*******************************************************************************
-    ** Getter for timestamp
-    **
-    *******************************************************************************/
-   public Instant getTimestamp()
-   {
-      return timestamp;
-   }
-
-
-
-   /*******************************************************************************
-    ** Setter for timestamp
-    **
-    *******************************************************************************/
-   public void setTimestamp(Instant timestamp)
-   {
-      this.timestamp = timestamp;
-   }
-
-
-
-   /*******************************************************************************
-    ** Fluent setter for timestamp
-    **
-    *******************************************************************************/
-   public OutboundAPILogHeader withTimestamp(Instant timestamp)
-   {
-      this.timestamp = timestamp;
-      return (this);
-   }
-
-
-
-   /*******************************************************************************
-    ** Getter for method
-    **
-    *******************************************************************************/
-   public String getMethod()
-   {
-      return method;
-   }
-
-
-
-   /*******************************************************************************
-    ** Setter for method
-    **
-    *******************************************************************************/
-   public void setMethod(String method)
-   {
-      this.method = method;
-   }
-
-
-
-   /*******************************************************************************
-    ** Fluent setter for method
-    **
-    *******************************************************************************/
-   public OutboundAPILogHeader withMethod(String method)
-   {
-      this.method = method;
-      return (this);
-   }
-
-
-
-   /*******************************************************************************
-    ** Getter for statusCode
-    **
-    *******************************************************************************/
-   public Integer getStatusCode()
-   {
-      return statusCode;
-   }
-
-
-
-   /*******************************************************************************
-    ** Setter for statusCode
-    **
-    *******************************************************************************/
-   public void setStatusCode(Integer statusCode)
-   {
-      this.statusCode = statusCode;
-   }
-
-
-
-   /*******************************************************************************
-    ** Fluent setter for statusCode
-    **
-    *******************************************************************************/
-   public OutboundAPILogHeader withStatusCode(Integer statusCode)
-   {
-      this.statusCode = statusCode;
-      return (this);
-   }
-
-
-
-   /*******************************************************************************
-    ** Getter for url
-    **
-    *******************************************************************************/
-   public String getUrl()
-   {
-      return url;
-   }
-
-
-
-   /*******************************************************************************
-    ** Setter for url
-    **
-    *******************************************************************************/
-   public void setUrl(String url)
-   {
-      this.url = url;
-   }
-
-
-
-   /*******************************************************************************
-    ** Fluent setter for url
-    **
-    *******************************************************************************/
-   public OutboundAPILogHeader withUrl(String url)
-   {
-      this.url = url;
-      return (this);
-   }
-
-
-
-   /*******************************************************************************
-    ** Getter for outboundAPILogRequestList
-    *******************************************************************************/
-   public List<OutboundAPILogRequest> getOutboundAPILogRequestList()
-   {
-      return (this.outboundAPILogRequestList);
-   }
-
-
-
-   /*******************************************************************************
-    ** Setter for outboundAPILogRequestList
-    *******************************************************************************/
-   public void setOutboundAPILogRequestList(List<OutboundAPILogRequest> outboundAPILogRequestList)
-   {
-      this.outboundAPILogRequestList = outboundAPILogRequestList;
-   }
-
-
-
-   /*******************************************************************************
-    ** Fluent setter for outboundAPILogRequestList
-    *******************************************************************************/
-   public OutboundAPILogHeader withOutboundAPILogRequestList(List<OutboundAPILogRequest> outboundAPILogRequestList)
-   {
-      this.outboundAPILogRequestList = outboundAPILogRequestList;
-      return (this);
-   }
-
-
-
-   /*******************************************************************************
-    ** Getter for outboundAPILogResponseList
-    *******************************************************************************/
-   public List<OutboundAPILogResponse> getOutboundAPILogResponseList()
-   {
-      return (this.outboundAPILogResponseList);
-   }
-
-
-
-   /*******************************************************************************
-    ** Setter for outboundAPILogResponseList
-    *******************************************************************************/
-   public void setOutboundAPILogResponseList(List<OutboundAPILogResponse> outboundAPILogResponseList)
-   {
-      this.outboundAPILogResponseList = outboundAPILogResponseList;
-   }
-
-
-
-   /*******************************************************************************
-    ** Fluent setter for outboundAPILogResponseList
-    *******************************************************************************/
-   public OutboundAPILogHeader withOutboundAPILogResponseList(List<OutboundAPILogResponse> outboundAPILogResponseList)
-   {
-      this.outboundAPILogResponseList = outboundAPILogResponseList;
-      return (this);
-   }
-
-}
-

qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/model/OutboundAPILogMetaDataProvider.java

@@ -22,33 +22,21 @@
 package com.kingsrook.qqq.backend.module.api.model;
 
 
-import java.util.ArrayList;
 import java.util.List;
 import java.util.function.Consumer;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.AdornmentType;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.FieldAdornment;
-import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
-import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.ValueTooLongBehavior;
-import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinOn;
-import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinType;
-import com.kingsrook.qqq.backend.core.model.metadata.joins.QJoinMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon;
 import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValue;
 import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSource;
 import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSourceType;
-import com.kingsrook.qqq.backend.core.model.metadata.tables.Association;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.Capability;
-import com.kingsrook.qqq.backend.core.model.metadata.tables.ExposedJoin;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QFieldSection;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.Tier;
-import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess;
-import com.kingsrook.qqq.backend.module.api.processes.MigrateOutboundAPILogExtractStep;
-import com.kingsrook.qqq.backend.module.api.processes.MigrateOutboundAPILogLoadStep;
-import com.kingsrook.qqq.backend.module.api.processes.MigrateOutboundAPILogTransformStep;
 
 
 /*******************************************************************************
@@ -62,15 +50,8 @@ public class OutboundAPILogMetaDataProvider
     *******************************************************************************/
    public static void defineAll(QInstance qInstance, String backendName, Consumer<QTableMetaData> backendDetailEnricher) throws QException
    {
-      definePossibleValueSources().forEach(pvs ->
-      {
-         if(qInstance.getPossibleValueSource(pvs.getName()) == null)
-         {
-            qInstance.addPossibleValueSource(pvs);
-         }
-      });
-
-      qInstance.addTable(defineOutboundAPILogTable(backendName, backendDetailEnricher));
+      definePossibleValueSources(qInstance);
+      defineOutboundAPILogTable(qInstance, backendName, backendDetailEnricher);
    }
 
 
@@ -78,71 +59,9 @@ public class OutboundAPILogMetaDataProvider
    /*******************************************************************************
     **
     *******************************************************************************/
-   public static void defineNewVersion(QInstance qInstance, String backendName, Consumer<QTableMetaData> backendDetailEnricher) throws QException
+   private static void definePossibleValueSources(QInstance instance)
    {
-      definePossibleValueSources().forEach(pvs ->
-      {
-         if(qInstance.getPossibleValueSource(pvs.getName()) == null)
-         {
-            qInstance.addPossibleValueSource(pvs);
-         }
-      });
-
-      qInstance.addTable(defineOutboundAPILogHeaderTable(backendName, backendDetailEnricher));
-      qInstance.addPossibleValueSource(defineOutboundAPILogHeaderPossibleValueSource());
-      qInstance.addTable(defineOutboundAPILogRequestTable(backendName, backendDetailEnricher));
-      qInstance.addTable(defineOutboundAPILogResponseTable(backendName, backendDetailEnricher));
-      defineJoins().forEach(join -> qInstance.add(join));
-   }
-
-
-
-   /***************************************************************************
-    **
-    ***************************************************************************/
-   public static void defineMigrationProcesses(QInstance qInstance, String sourceTableName)
-   {
-      qInstance.addProcess(StreamedETLWithFrontendProcess.processMetaDataBuilder()
-         .withName("migrateOutboundApiLogToHeaderChildProcess")
-         .withLabel("Migrate Outbound API Log Test to Header/Child")
-         .withIcon(new QIcon().withName("drive_file_move"))
-         .withTableName(sourceTableName)
-         .withSourceTable(sourceTableName)
-         .withDestinationTable(OutboundAPILogHeader.TABLE_NAME)
-         .withExtractStepClass(MigrateOutboundAPILogExtractStep.class)
-         .withTransformStepClass(MigrateOutboundAPILogTransformStep.class)
-         .withLoadStepClass(MigrateOutboundAPILogLoadStep.class)
-         .withReviewStepRecordFields(List.of(
-            new QFieldMetaData("url", QFieldType.INTEGER)
-         ))
-         .getProcessMetaData());
-
-      qInstance.addProcess(StreamedETLWithFrontendProcess.processMetaDataBuilder()
-         .withName("migrateOutboundApiLogToMongoDBProcess")
-         .withLabel("Migrate Outbound API Log Test to MongoDB")
-         .withIcon(new QIcon().withName("drive_file_move"))
-         .withTableName(sourceTableName)
-         .withSourceTable(sourceTableName)
-         .withDestinationTable(OutboundAPILog.TABLE_NAME + "MongoDB")
-         .withExtractStepClass(MigrateOutboundAPILogExtractStep.class)
-         .withTransformStepClass(MigrateOutboundAPILogTransformStep.class)
-         .withLoadStepClass(MigrateOutboundAPILogLoadStep.class)
-         .withReviewStepRecordFields(List.of(
-            new QFieldMetaData("url", QFieldType.INTEGER)
-         ))
-         .getProcessMetaData());
-   }
-
-
-
-   /*******************************************************************************
-    **
-    *******************************************************************************/
-   private static List<QPossibleValueSource> definePossibleValueSources()
-   {
-      List<QPossibleValueSource> rs = new ArrayList<>();
-
-      rs.add(new QPossibleValueSource()
+      instance.addPossibleValueSource(new QPossibleValueSource()
          .withName("outboundApiMethod")
          .withType(QPossibleValueSourceType.ENUM)
          .withEnumValues(List.of(
@@ -153,7 +72,7 @@ public class OutboundAPILogMetaDataProvider
             new QPossibleValue<>("DELETE")
          )));
 
-      rs.add(new QPossibleValueSource()
+      instance.addPossibleValueSource(new QPossibleValueSource()
          .withName("outboundApiStatusCode")
          .withType(QPossibleValueSourceType.ENUM)
          .withEnumValues(List.of(
@@ -172,8 +91,6 @@ public class OutboundAPILogMetaDataProvider
             new QPossibleValue<>(503, "503 (Service Unavailable)"),
             new QPossibleValue<>(504, "500 (Gateway Timeout)")
          )));
-
-      return (rs);
    }
 
 
@@ -181,8 +98,9 @@ public class OutboundAPILogMetaDataProvider
    /*******************************************************************************
     **
     *******************************************************************************/
-   public static QTableMetaData defineOutboundAPILogTable(String backendName, Consumer<QTableMetaData> backendDetailEnricher) throws QException
+   private static void defineOutboundAPILogTable(QInstance qInstance, String backendName, Consumer<QTableMetaData> backendDetailEnricher) throws QException
    {
+
       QTableMetaData tableMetaData = new QTableMetaData()
          .withName(OutboundAPILog.TABLE_NAME)
          .withLabel("Outbound API Log")
@@ -201,8 +119,29 @@ public class OutboundAPILogMetaDataProvider
       tableMetaData.getField("requestBody").withFieldAdornment(new FieldAdornment(AdornmentType.CODE_EDITOR).withValue(AdornmentType.CodeEditorValues.languageMode("json")));
       tableMetaData.getField("responseBody").withFieldAdornment(new FieldAdornment(AdornmentType.CODE_EDITOR).withValue(AdornmentType.CodeEditorValues.languageMode("json")));
 
-      addChipAdornmentToMethodField(tableMetaData);
-      addChipAdornmentToStatusCodeField(tableMetaData);
+      tableMetaData.getField("method").withFieldAdornment(new FieldAdornment(AdornmentType.CHIP)
+         .withValue(AdornmentType.ChipValues.colorValue("GET", AdornmentType.ChipValues.COLOR_INFO))
+         .withValue(AdornmentType.ChipValues.colorValue("POST", AdornmentType.ChipValues.COLOR_SUCCESS))
+         .withValue(AdornmentType.ChipValues.colorValue("DELETE", AdornmentType.ChipValues.COLOR_ERROR))
+         .withValue(AdornmentType.ChipValues.colorValue("PATCH", AdornmentType.ChipValues.COLOR_WARNING))
+         .withValue(AdornmentType.ChipValues.colorValue("PUT", AdornmentType.ChipValues.COLOR_WARNING)));
+
+      tableMetaData.getField("statusCode").withFieldAdornment(new FieldAdornment(AdornmentType.CHIP)
+         .withValue(AdornmentType.ChipValues.colorValue(200, AdornmentType.ChipValues.COLOR_SUCCESS))
+         .withValue(AdornmentType.ChipValues.colorValue(201, AdornmentType.ChipValues.COLOR_SUCCESS))
+         .withValue(AdornmentType.ChipValues.colorValue(204, AdornmentType.ChipValues.COLOR_SUCCESS))
+         .withValue(AdornmentType.ChipValues.colorValue(207, AdornmentType.ChipValues.COLOR_INFO))
+         .withValue(AdornmentType.ChipValues.colorValue(400, AdornmentType.ChipValues.COLOR_ERROR))
+         .withValue(AdornmentType.ChipValues.colorValue(401, AdornmentType.ChipValues.COLOR_ERROR))
+         .withValue(AdornmentType.ChipValues.colorValue(403, AdornmentType.ChipValues.COLOR_ERROR))
+         .withValue(AdornmentType.ChipValues.colorValue(404, AdornmentType.ChipValues.COLOR_ERROR))
+         .withValue(AdornmentType.ChipValues.colorValue(422, AdornmentType.ChipValues.COLOR_ERROR))
+         .withValue(AdornmentType.ChipValues.colorValue(429, AdornmentType.ChipValues.COLOR_ERROR))
+         .withValue(AdornmentType.ChipValues.colorValue(500, AdornmentType.ChipValues.COLOR_ERROR))
+         .withValue(AdornmentType.ChipValues.colorValue(502, AdornmentType.ChipValues.COLOR_ERROR))
+         .withValue(AdornmentType.ChipValues.colorValue(503, AdornmentType.ChipValues.COLOR_ERROR))
+         .withValue(AdornmentType.ChipValues.colorValue(504, AdornmentType.ChipValues.COLOR_ERROR))
+      );
 
       ///////////////////////////////////////////
       // these are the lengths of a MySQL TEXT //
@@ -221,210 +160,6 @@ public class OutboundAPILogMetaDataProvider
          backendDetailEnricher.accept(tableMetaData);
       }
 
-      return (tableMetaData);
+      qInstance.addTable(tableMetaData);
    }
-
-
-
-   /***************************************************************************
-    **
-    ***************************************************************************/
-   private static void addChipAdornmentToStatusCodeField(QTableMetaData tableMetaData)
-   {
-      tableMetaData.getField("statusCode").withFieldAdornment(new FieldAdornment(AdornmentType.CHIP)
-         .withValue(AdornmentType.ChipValues.colorValue(200, AdornmentType.ChipValues.COLOR_SUCCESS))
-         .withValue(AdornmentType.ChipValues.colorValue(201, AdornmentType.ChipValues.COLOR_SUCCESS))
-         .withValue(AdornmentType.ChipValues.colorValue(204, AdornmentType.ChipValues.COLOR_SUCCESS))
-         .withValue(AdornmentType.ChipValues.colorValue(207, AdornmentType.ChipValues.COLOR_INFO))
-         .withValue(AdornmentType.ChipValues.colorValue(400, AdornmentType.ChipValues.COLOR_ERROR))
-         .withValue(AdornmentType.ChipValues.colorValue(401, AdornmentType.ChipValues.COLOR_ERROR))
-         .withValue(AdornmentType.ChipValues.colorValue(403, AdornmentType.ChipValues.COLOR_ERROR))
-         .withValue(AdornmentType.ChipValues.colorValue(404, AdornmentType.ChipValues.COLOR_ERROR))
-         .withValue(AdornmentType.ChipValues.colorValue(422, AdornmentType.ChipValues.COLOR_ERROR))
-         .withValue(AdornmentType.ChipValues.colorValue(429, AdornmentType.ChipValues.COLOR_ERROR))
-         .withValue(AdornmentType.ChipValues.colorValue(500, AdornmentType.ChipValues.COLOR_ERROR))
-         .withValue(AdornmentType.ChipValues.colorValue(502, AdornmentType.ChipValues.COLOR_ERROR))
-         .withValue(AdornmentType.ChipValues.colorValue(503, AdornmentType.ChipValues.COLOR_ERROR))
-         .withValue(AdornmentType.ChipValues.colorValue(504, AdornmentType.ChipValues.COLOR_ERROR))
-      );
-   }
-
-
-
-   /***************************************************************************
-    **
-    ***************************************************************************/
-   private static void addChipAdornmentToMethodField(QTableMetaData tableMetaData)
-   {
-      tableMetaData.getField("method").withFieldAdornment(new FieldAdornment(AdornmentType.CHIP)
-         .withValue(AdornmentType.ChipValues.colorValue("GET", AdornmentType.ChipValues.COLOR_INFO))
-         .withValue(AdornmentType.ChipValues.colorValue("POST", AdornmentType.ChipValues.COLOR_SUCCESS))
-         .withValue(AdornmentType.ChipValues.colorValue("DELETE", AdornmentType.ChipValues.COLOR_ERROR))
-         .withValue(AdornmentType.ChipValues.colorValue("PATCH", AdornmentType.ChipValues.COLOR_WARNING))
-         .withValue(AdornmentType.ChipValues.colorValue("PUT", AdornmentType.ChipValues.COLOR_WARNING)));
-   }
-
-
-
-   /*******************************************************************************
-    **
-    *******************************************************************************/
-   private static QTableMetaData defineOutboundAPILogHeaderTable(String backendName, Consumer<QTableMetaData> backendDetailEnricher) throws QException
-   {
-      QTableMetaData tableMetaData = new QTableMetaData()
-         .withName(OutboundAPILogHeader.TABLE_NAME)
-         .withLabel("Outbound API Log Header/Child")
-         .withIcon(new QIcon().withName("data_object"))
-         .withBackendName(backendName)
-         .withRecordLabelFormat("%s")
-         .withRecordLabelFields("id")
-         .withPrimaryKeyField("id")
-         .withFieldsFromEntity(OutboundAPILogHeader.class)
-         .withSection(new QFieldSection("identity", new QIcon().withName("badge"), Tier.T1, List.of("id")))
-         .withSection(new QFieldSection("request", new QIcon().withName("arrow_upward"), Tier.T2, List.of("method", "url", OutboundAPILogRequest.TABLE_NAME + ".requestBody")))
-         .withSection(new QFieldSection("response", new QIcon().withName("arrow_downward"), Tier.T2, List.of("statusCode", OutboundAPILogResponse.TABLE_NAME + ".responseBody")))
-         .withSection(new QFieldSection("dates", new QIcon().withName("calendar_month"), Tier.T3, List.of("timestamp")))
-         .withoutCapabilities(Capability.TABLE_INSERT, Capability.TABLE_UPDATE, Capability.TABLE_DELETE);
-
-      // tableMetaData.getField(OutboundAPILogRequest.TABLE_NAME + ".requestBody").withFieldAdornment(new FieldAdornment(AdornmentType.CODE_EDITOR).withValue(AdornmentType.CodeEditorValues.languageMode("json")));
-      // tableMetaData.getField(OutboundAPILogResponse.TABLE_NAME + ".responseBody").withFieldAdornment(new FieldAdornment(AdornmentType.CODE_EDITOR).withValue(AdornmentType.CodeEditorValues.languageMode("json")));
-
-      addChipAdornmentToMethodField(tableMetaData);
-      addChipAdornmentToStatusCodeField(tableMetaData);
-
-      tableMetaData.withAssociation(new Association()
-         .withName(OutboundAPILogRequest.TABLE_NAME)
-         .withAssociatedTableName(OutboundAPILogRequest.TABLE_NAME)
-         .withJoinName(QJoinMetaData.makeInferredJoinName(OutboundAPILogHeader.TABLE_NAME, OutboundAPILogRequest.TABLE_NAME)));
-
-      tableMetaData.withAssociation(new Association()
-         .withName(OutboundAPILogResponse.TABLE_NAME)
-         .withAssociatedTableName(OutboundAPILogResponse.TABLE_NAME)
-         .withJoinName(QJoinMetaData.makeInferredJoinName(OutboundAPILogHeader.TABLE_NAME, OutboundAPILogResponse.TABLE_NAME)));
-
-      tableMetaData.withExposedJoin(new ExposedJoin()
-         .withJoinTable(OutboundAPILogRequest.TABLE_NAME)
-         .withJoinPath(List.of(QJoinMetaData.makeInferredJoinName(OutboundAPILogHeader.TABLE_NAME, OutboundAPILogRequest.TABLE_NAME))));
-
-      tableMetaData.withExposedJoin(new ExposedJoin()
-         .withJoinTable(OutboundAPILogResponse.TABLE_NAME)
-         .withJoinPath(List.of(QJoinMetaData.makeInferredJoinName(OutboundAPILogHeader.TABLE_NAME, OutboundAPILogResponse.TABLE_NAME))));
-
-      tableMetaData.getField("url").withMaxLength(4096).withBehavior(ValueTooLongBehavior.TRUNCATE_ELLIPSIS);
-      tableMetaData.getField("url").withFieldAdornment(AdornmentType.Size.XLARGE.toAdornment());
-
-      if(backendDetailEnricher != null)
-      {
-         backendDetailEnricher.accept(tableMetaData);
-      }
-
-      return (tableMetaData);
-   }
-
-
-
-   /*******************************************************************************
-    **
-    *******************************************************************************/
-   private static QTableMetaData defineOutboundAPILogRequestTable(String backendName, Consumer<QTableMetaData> backendDetailEnricher) throws QException
-   {
-      QTableMetaData tableMetaData = new QTableMetaData()
-         .withName(OutboundAPILogRequest.TABLE_NAME)
-         .withLabel("Outbound API Log Request")
-         .withIcon(new QIcon().withName("arrow_upward"))
-         .withBackendName(backendName)
-         .withRecordLabelFormat("%s")
-         .withRecordLabelFields("id")
-         .withPrimaryKeyField("id")
-         .withFieldsFromEntity(OutboundAPILogRequest.class)
-         .withSection(new QFieldSection("identity", new QIcon().withName("badge"), Tier.T1, List.of("id", "outboundApiLogHeaderId")))
-         .withSection(new QFieldSection("request", new QIcon().withName("arrow_upward"), Tier.T2, List.of("requestBody")))
-         .withoutCapabilities(Capability.TABLE_INSERT, Capability.TABLE_UPDATE, Capability.TABLE_DELETE);
-
-      tableMetaData.getField("requestBody").withFieldAdornment(new FieldAdornment(AdornmentType.CODE_EDITOR).withValue(AdornmentType.CodeEditorValues.languageMode("json")));
-
-      //////////////////////////////////////////////
-      // this is the length of a MySQL MEDIUMTEXT //
-      //////////////////////////////////////////////
-      tableMetaData.getField("requestBody").withMaxLength(16_777_215).withBehavior(ValueTooLongBehavior.TRUNCATE_ELLIPSIS);
-
-      if(backendDetailEnricher != null)
-      {
-         backendDetailEnricher.accept(tableMetaData);
-      }
-
-      return (tableMetaData);
-   }
-
-
-
-   /*******************************************************************************
-    **
-    *******************************************************************************/
-   private static QTableMetaData defineOutboundAPILogResponseTable(String backendName, Consumer<QTableMetaData> backendDetailEnricher) throws QException
-   {
-      QTableMetaData tableMetaData = new QTableMetaData()
-         .withName(OutboundAPILogResponse.TABLE_NAME)
-         .withLabel("Outbound API Log Response")
-         .withIcon(new QIcon().withName("arrow_upward"))
-         .withBackendName(backendName)
-         .withRecordLabelFormat("%s")
-         .withRecordLabelFields("id")
-         .withPrimaryKeyField("id")
-         .withFieldsFromEntity(OutboundAPILogResponse.class)
-         .withSection(new QFieldSection("identity", new QIcon().withName("badge"), Tier.T1, List.of("id", "outboundApiLogHeaderId")))
-         .withSection(new QFieldSection("response", new QIcon().withName("arrow_upward"), Tier.T2, List.of("responseBody")))
-         .withoutCapabilities(Capability.TABLE_INSERT, Capability.TABLE_UPDATE, Capability.TABLE_DELETE);
-
-      tableMetaData.getField("responseBody").withFieldAdornment(new FieldAdornment(AdornmentType.CODE_EDITOR).withValue(AdornmentType.CodeEditorValues.languageMode("json")));
-
-      //////////////////////////////////////////////
-      // this is the length of a MySQL MEDIUMTEXT //
-      //////////////////////////////////////////////
-      tableMetaData.getField("responseBody").withMaxLength(16_777_215).withBehavior(ValueTooLongBehavior.TRUNCATE_ELLIPSIS);
-
-      if(backendDetailEnricher != null)
-      {
-         backendDetailEnricher.accept(tableMetaData);
-      }
-
-      return (tableMetaData);
-   }
-
-
-
-   /*******************************************************************************
-    **
-    *******************************************************************************/
-   private static List<QJoinMetaData> defineJoins()
-   {
-      List<QJoinMetaData> rs = new ArrayList<>();
-
-      rs.add(new QJoinMetaData()
-         .withLeftTable(OutboundAPILogHeader.TABLE_NAME)
-         .withRightTable(OutboundAPILogRequest.TABLE_NAME)
-         .withInferredName()
-         .withType(JoinType.ONE_TO_ONE)
-         .withJoinOn(new JoinOn("id", "outboundApiLogHeaderId")));
-
-      rs.add(new QJoinMetaData()
-         .withLeftTable(OutboundAPILogHeader.TABLE_NAME)
-         .withRightTable(OutboundAPILogResponse.TABLE_NAME)
-         .withInferredName()
-         .withType(JoinType.ONE_TO_ONE)
-         .withJoinOn(new JoinOn("id", "outboundApiLogHeaderId")));
-
-      return (rs);
-   }
-
-
-
-   /***************************************************************************
-    **
-    ***************************************************************************/
-   private static QPossibleValueSource defineOutboundAPILogHeaderPossibleValueSource()
-   {
-      return QPossibleValueSource.newForTable(OutboundAPILogHeader.TABLE_NAME);
-   }
-
 }

qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/model/OutboundAPILogRequest.java

@@ -1,165 +0,0 @@
-/*
- * QQQ - Low-code Application Framework for Engineers.
- * Copyright (C) 2021-2023.  Kingsrook, LLC
- * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
- * contact@kingsrook.com
- * https://github.com/Kingsrook/
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program.  If not, see <https://www.gnu.org/licenses/>.
- */
-
-package com.kingsrook.qqq.backend.module.api.model;
-
-
-import com.kingsrook.qqq.backend.core.exceptions.QException;
-import com.kingsrook.qqq.backend.core.model.data.QField;
-import com.kingsrook.qqq.backend.core.model.data.QRecord;
-import com.kingsrook.qqq.backend.core.model.data.QRecordEntity;
-
-
-/*******************************************************************************
- ** Entity bean for OutboundApiLogRequest table
- *******************************************************************************/
-public class OutboundAPILogRequest extends QRecordEntity
-{
-   public static final String TABLE_NAME = "outboundApiLogRequest";
-
-   @QField(isEditable = false)
-   private Integer id;
-
-   @QField(possibleValueSourceName = OutboundAPILogHeader.TABLE_NAME)
-   private Integer outboundApiLogHeaderId;
-
-   @QField()
-   private String requestBody;
-
-
-
-   /*******************************************************************************
-    ** Constructor
-    **
-    *******************************************************************************/
-   public OutboundAPILogRequest()
-   {
-   }
-
-
-
-   /*******************************************************************************
-    ** Constructor
-    **
-    *******************************************************************************/
-   public OutboundAPILogRequest(QRecord qRecord) throws QException
-   {
-      populateFromQRecord(qRecord);
-   }
-
-
-
-   /*******************************************************************************
-    ** Getter for id
-    **
-    *******************************************************************************/
-   public Integer getId()
-   {
-      return id;
-   }
-
-
-
-   /*******************************************************************************
-    ** Setter for id
-    **
-    *******************************************************************************/
-   public void setId(Integer id)
-   {
-      this.id = id;
-   }
-
-
-
-   /*******************************************************************************
-    ** Fluent setter for id
-    **
-    *******************************************************************************/
-   public OutboundAPILogRequest withId(Integer id)
-   {
-      this.id = id;
-      return (this);
-   }
-
-
-
-   /*******************************************************************************
-    ** Getter for requestBody
-    *******************************************************************************/
-   public String getRequestBody()
-   {
-      return (this.requestBody);
-   }
-
-
-
-   /*******************************************************************************
-    ** Setter for requestBody
-    *******************************************************************************/
-   public void setRequestBody(String requestBody)
-   {
-      this.requestBody = requestBody;
-   }
-
-
-
-   /*******************************************************************************
-    ** Fluent setter for requestBody
-    *******************************************************************************/
-   public OutboundAPILogRequest withRequestBody(String requestBody)
-   {
-      this.requestBody = requestBody;
-      return (this);
-   }
-
-
-
-   /*******************************************************************************
-    ** Getter for outboundApiLogHeaderId
-    *******************************************************************************/
-   public Integer getOutboundApiLogHeaderId()
-   {
-      return (this.outboundApiLogHeaderId);
-   }
-
-
-
-   /*******************************************************************************
-    ** Setter for outboundApiLogHeaderId
-    *******************************************************************************/
-   public void setOutboundApiLogHeaderId(Integer outboundApiLogHeaderId)
-   {
-      this.outboundApiLogHeaderId = outboundApiLogHeaderId;
-   }
-
-
-
-   /*******************************************************************************
-    ** Fluent setter for outboundApiLogHeaderId
-    *******************************************************************************/
-   public OutboundAPILogRequest withOutboundApiLogHeaderId(Integer outboundApiLogHeaderId)
-   {
-      this.outboundApiLogHeaderId = outboundApiLogHeaderId;
-      return (this);
-   }
-
-}
-

qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/model/OutboundAPILogResponse.java

@@ -1,165 +0,0 @@
-/*
- * QQQ - Low-code Application Framework for Engineers.
- * Copyright (C) 2021-2023.  Kingsrook, LLC
- * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
- * contact@kingsrook.com
- * https://github.com/Kingsrook/
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program.  If not, see <https://www.gnu.org/licenses/>.
- */
-
-package com.kingsrook.qqq.backend.module.api.model;
-
-
-import com.kingsrook.qqq.backend.core.exceptions.QException;
-import com.kingsrook.qqq.backend.core.model.data.QField;
-import com.kingsrook.qqq.backend.core.model.data.QRecord;
-import com.kingsrook.qqq.backend.core.model.data.QRecordEntity;
-
-
-/*******************************************************************************
- ** Entity bean for OutboundApiLogResponse table
- *******************************************************************************/
-public class OutboundAPILogResponse extends QRecordEntity
-{
-   public static final String TABLE_NAME = "outboundApiLogResponse";
-
-   @QField(isEditable = false)
-   private Integer id;
-
-   @QField(possibleValueSourceName = OutboundAPILogHeader.TABLE_NAME)
-   private Integer outboundApiLogHeaderId;
-
-   @QField()
-   private String responseBody;
-
-
-
-   /*******************************************************************************
-    ** Constructor
-    **
-    *******************************************************************************/
-   public OutboundAPILogResponse()
-   {
-   }
-
-
-
-   /*******************************************************************************
-    ** Constructor
-    **
-    *******************************************************************************/
-   public OutboundAPILogResponse(QRecord qRecord) throws QException
-   {
-      populateFromQRecord(qRecord);
-   }
-
-
-
-   /*******************************************************************************
-    ** Getter for id
-    **
-    *******************************************************************************/
-   public Integer getId()
-   {
-      return id;
-   }
-
-
-
-   /*******************************************************************************
-    ** Setter for id
-    **
-    *******************************************************************************/
-   public void setId(Integer id)
-   {
-      this.id = id;
-   }
-
-
-
-   /*******************************************************************************
-    ** Fluent setter for id
-    **
-    *******************************************************************************/
-   public OutboundAPILogResponse withId(Integer id)
-   {
-      this.id = id;
-      return (this);
-   }
-
-
-
-   /*******************************************************************************
-    ** Getter for responseBody
-    *******************************************************************************/
-   public String getResponseBody()
-   {
-      return (this.responseBody);
-   }
-
-
-
-   /*******************************************************************************
-    ** Setter for responseBody
-    *******************************************************************************/
-   public void setResponseBody(String responseBody)
-   {
-      this.responseBody = responseBody;
-   }
-
-
-
-   /*******************************************************************************
-    ** Fluent setter for responseBody
-    *******************************************************************************/
-   public OutboundAPILogResponse withResponseBody(String responseBody)
-   {
-      this.responseBody = responseBody;
-      return (this);
-   }
-
-
-
-   /*******************************************************************************
-    ** Getter for outboundApiLogHeaderId
-    *******************************************************************************/
-   public Integer getOutboundApiLogHeaderId()
-   {
-      return (this.outboundApiLogHeaderId);
-   }
-
-
-
-   /*******************************************************************************
-    ** Setter for outboundApiLogHeaderId
-    *******************************************************************************/
-   public void setOutboundApiLogHeaderId(Integer outboundApiLogHeaderId)
-   {
-      this.outboundApiLogHeaderId = outboundApiLogHeaderId;
-   }
-
-
-
-   /*******************************************************************************
-    ** Fluent setter for outboundApiLogHeaderId
-    *******************************************************************************/
-   public OutboundAPILogResponse withOutboundApiLogHeaderId(Integer outboundApiLogHeaderId)
-   {
-      this.outboundApiLogHeaderId = outboundApiLogHeaderId;
-      return (this);
-   }
-
-}
-

qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/processes/MigrateOutboundAPILogTransformStep.java

@@ -1,127 +0,0 @@
-/*
- * QQQ - Low-code Application Framework for Engineers.
- * Copyright (C) 2021-2024.  Kingsrook, LLC
- * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
- * contact@kingsrook.com
- * https://github.com/Kingsrook/
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program.  If not, see <https://www.gnu.org/licenses/>.
- */
-
-package com.kingsrook.qqq.backend.module.api.processes;
-
-
-import java.util.ArrayList;
-import java.util.List;
-import com.kingsrook.qqq.backend.core.exceptions.QException;
-import com.kingsrook.qqq.backend.core.logging.QLogger;
-import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLine;
-import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLineInterface;
-import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
-import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
-import com.kingsrook.qqq.backend.core.model.data.QRecord;
-import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.AbstractTransformStep;
-import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess;
-import com.kingsrook.qqq.backend.core.processes.implementations.general.StandardProcessSummaryLineProducer;
-import com.kingsrook.qqq.backend.module.api.model.OutboundAPILogHeader;
-import com.kingsrook.qqq.backend.module.api.model.OutboundAPILogRequest;
-import com.kingsrook.qqq.backend.module.api.model.OutboundAPILogResponse;
-
-
-/*******************************************************************************
- ** migrate records from original (singular) outboundApiLog table to new split-up
- ** version (outboundApiLogHeader)
- *******************************************************************************/
-public class MigrateOutboundAPILogTransformStep extends AbstractTransformStep
-{
-   private static final QLogger LOG = QLogger.getLogger(MigrateOutboundAPILogTransformStep.class);
-
-   private ProcessSummaryLine okToInsertLine = StandardProcessSummaryLineProducer.getOkToInsertLine();
-   private ProcessSummaryLine errorLine      = StandardProcessSummaryLineProducer.getErrorLine();
-
-
-
-   /***************************************************************************
-    **
-    ***************************************************************************/
-   /*
-   @Override
-   public Integer getOverrideRecordPipeCapacity(RunBackendStepInput runBackendStepInput)
-   {
-      return (100);
-   }
-   */
-
-
-
-   /*******************************************************************************
-    **
-    *******************************************************************************/
-   @Override
-   public ArrayList<ProcessSummaryLineInterface> getProcessSummary(RunBackendStepOutput runBackendStepOutput, boolean isForResultScreen)
-   {
-      ArrayList<ProcessSummaryLineInterface> rs = new ArrayList<>();
-      okToInsertLine.addSelfToListIfAnyCount(rs);
-      errorLine.addSelfToListIfAnyCount(rs);
-      return (rs);
-   }
-
-
-
-   /***************************************************************************
-    **
-    ***************************************************************************/
-   @Override
-   public void preRun(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException
-   {
-      runBackendStepOutput.addValue("counter", 0);
-   }
-
-
-
-   /*******************************************************************************
-    **
-    *******************************************************************************/
-   @Override
-   public void runOnePage(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException
-   {
-      int counter = runBackendStepOutput.getValueInteger("counter") + runBackendStepInput.getRecords().size();
-      runBackendStepOutput.addValue("counter", counter);
-      runBackendStepInput.getAsyncJobCallback().updateStatus("Migrating records (at #" + String.format("%,d", counter) + ")");
-
-      String destinationTable = runBackendStepInput.getValueString(StreamedETLWithFrontendProcess.FIELD_DESTINATION_TABLE);
-
-      for(QRecord record : runBackendStepInput.getRecords())
-      {
-         okToInsertLine.incrementCountAndAddPrimaryKey(record.getValue("id"));
-
-         if(destinationTable.equals(OutboundAPILogHeader.TABLE_NAME))
-         {
-            OutboundAPILogHeader outboundAPILogHeader = new OutboundAPILogHeader(record);
-            outboundAPILogHeader.withOutboundAPILogRequestList(List.of(new OutboundAPILogRequest().withRequestBody(record.getValueString("requestBody"))));
-            outboundAPILogHeader.withOutboundAPILogResponseList(List.of(new OutboundAPILogResponse().withResponseBody(record.getValueString("responseBody"))));
-            runBackendStepOutput.addRecord(outboundAPILogHeader.toQRecord());
-         }
-         else
-         {
-            ///////////////////////////////////////////////////////////////////
-            // for the mongodb migration, just pass records straight through //
-            ///////////////////////////////////////////////////////////////////
-            record.setValue("id", null);
-            runBackendStepOutput.addRecord(record);
-         }
-      }
-   }
-
-}

qqq-backend-module-api/src/test/java/com/kingsrook/qqq/backend/module/api/BaseTest.java

@@ -26,7 +26,6 @@ import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.logging.QLogger;
 import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
 import com.kingsrook.qqq.backend.core.model.session.QSession;
-import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryRecordStore;
 import org.junit.jupiter.api.AfterEach;
 import org.junit.jupiter.api.BeforeEach;
 
@@ -47,7 +46,6 @@ public class BaseTest
    void baseBeforeEach()
    {
       QContext.init(TestUtils.defineInstance(), new QSession());
-      MemoryRecordStore.getInstance().reset();
    }
 
 
@@ -59,7 +57,6 @@ public class BaseTest
    void baseAfterEach()
    {
       QContext.clear();
-      MemoryRecordStore.getInstance().reset();
    }
 
 

qqq-backend-module-api/src/test/java/com/kingsrook/qqq/backend/module/api/mocks/MockApiActionUtils.java

@@ -28,7 +28,6 @@ import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 import com.kingsrook.qqq.backend.module.api.actions.BaseAPIActionUtil;
 import com.kingsrook.qqq.backend.module.api.actions.QHttpResponse;
 import org.apache.http.client.methods.CloseableHttpResponse;
-import org.apache.http.client.methods.HttpPost;
 import org.apache.http.client.methods.HttpRequestBase;
 import org.apache.http.impl.client.CloseableHttpClient;
 
@@ -89,7 +88,7 @@ public class MockApiActionUtils extends BaseAPIActionUtil
     **
     *******************************************************************************/
    @Override
-   protected CloseableHttpResponse executeOAuthTokenRequest(CloseableHttpClient client, HttpPost request) throws IOException
+   protected CloseableHttpResponse executeOAuthTokenRequest(CloseableHttpClient client, HttpRequestBase request) throws IOException
    {
       runMockAsserter(request);
       return new MockHttpResponse(mockApiUtilsHelper);

qqq-backend-module-api/src/test/java/com/kingsrook/qqq/backend/module/api/processes/MigrateOutboundAPILogProcessTest.java

@@ -1,98 +0,0 @@
-/*
- * QQQ - Low-code Application Framework for Engineers.
- * Copyright (C) 2021-2024.  Kingsrook, LLC
- * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
- * contact@kingsrook.com
- * https://github.com/Kingsrook/
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program.  If not, see <https://www.gnu.org/licenses/>.
- */
-
-package com.kingsrook.qqq.backend.module.api.processes;
-
-
-import java.util.List;
-import com.kingsrook.qqq.backend.core.actions.processes.QProcessCallbackFactory;
-import com.kingsrook.qqq.backend.core.actions.processes.RunProcessAction;
-import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
-import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
-import com.kingsrook.qqq.backend.core.context.QContext;
-import com.kingsrook.qqq.backend.core.exceptions.QException;
-import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput;
-import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessOutput;
-import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
-import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
-import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
-import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryRecordStore;
-import com.kingsrook.qqq.backend.module.api.BaseTest;
-import com.kingsrook.qqq.backend.module.api.TestUtils;
-import com.kingsrook.qqq.backend.module.api.model.OutboundAPILog;
-import com.kingsrook.qqq.backend.module.api.model.OutboundAPILogHeader;
-import com.kingsrook.qqq.backend.module.api.model.OutboundAPILogMetaDataProvider;
-import org.junit.jupiter.api.BeforeEach;
-import org.junit.jupiter.api.Test;
-import static org.junit.jupiter.api.Assertions.assertEquals;
-
-
-/*******************************************************************************
- ** Unit test for MigrateOutboundAPILog process
- *******************************************************************************/
-class MigrateOutboundAPILogProcessTest extends BaseTest
-{
-
-   /*******************************************************************************
-    **
-    *******************************************************************************/
-   @BeforeEach
-   void beforeEach() throws QException
-   {
-      MemoryRecordStore.getInstance().reset();
-      OutboundAPILogMetaDataProvider.defineAll(QContext.getQInstance(), TestUtils.MEMORY_BACKEND_NAME, null);
-      OutboundAPILogMetaDataProvider.defineNewVersion(QContext.getQInstance(), TestUtils.MEMORY_BACKEND_NAME, null);
-      OutboundAPILogMetaDataProvider.defineMigrationProcesses(QContext.getQInstance(), OutboundAPILog.TABLE_NAME);
-   }
-
-
-
-   /*******************************************************************************
-    **
-    *******************************************************************************/
-   @Test
-   void test() throws QException
-   {
-      new InsertAction().execute(new InsertInput(OutboundAPILog.TABLE_NAME).withRecordEntity(new OutboundAPILog()
-         .withMethod("POST")
-         .withUrl("www.google.com")
-         .withRequestBody("please")
-         .withResponseBody("you're welcome")
-         .withStatusCode(201)
-      ));
-
-      RunProcessInput input = new RunProcessInput();
-      input.setProcessName("migrateOutboundApiLogToHeaderChildProcess");
-      input.setCallback(QProcessCallbackFactory.forFilter(new QQueryFilter()));
-      input.setFrontendStepBehavior(RunProcessInput.FrontendStepBehavior.SKIP);
-      RunProcessOutput runProcessOutput = new RunProcessAction().execute(input);
-
-      List<OutboundAPILogHeader> outboundApiLogHeaderList = new QueryAction().execute(new QueryInput(OutboundAPILogHeader.TABLE_NAME).withIncludeAssociations(true)).getRecordEntities(OutboundAPILogHeader.class);
-      assertEquals(1, outboundApiLogHeaderList.size());
-      assertEquals("POST", outboundApiLogHeaderList.get(0).getMethod());
-      assertEquals(201, outboundApiLogHeaderList.get(0).getStatusCode());
-      assertEquals(1, outboundApiLogHeaderList.get(0).getOutboundAPILogRequestList().size());
-      assertEquals("please", outboundApiLogHeaderList.get(0).getOutboundAPILogRequestList().get(0).getRequestBody());
-      assertEquals(1, outboundApiLogHeaderList.get(0).getOutboundAPILogResponseList().size());
-      assertEquals("you're welcome", outboundApiLogHeaderList.get(0).getOutboundAPILogResponseList().get(0).getResponseBody());
-   }
-
-}

qqq-backend-module-mongodb/src/main/java/com/kingsrook/qqq/backend/module/mongodb/actions/AbstractMongoDBAction.java

@@ -41,6 +41,7 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.query.JoinsContext;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.expressions.AbstractFilterExpression;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
 import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
@@ -176,18 +177,28 @@ public class AbstractMongoDBAction
    /*******************************************************************************
     ** Convert a mongodb document to a QRecord.
     *******************************************************************************/
-   protected QRecord documentToRecord(QTableMetaData table, Document document)
+   protected QRecord documentToRecord(QueryInput queryInput, Document document)
    {
-      QRecord record = new QRecord();
+      QTableMetaData table  = queryInput.getTable();
+      QRecord        record = new QRecord();
+
       record.setTableName(table.getName());
 
+      /////////////////////////////////////////////
+      // build the set of field names to include //
+      /////////////////////////////////////////////
+      Set<String> fieldNamesToInclude = queryInput.getFieldNamesToInclude();
+      List<QFieldMetaData> selectedFields = table.getFields().values()
+         .stream().filter(field -> fieldNamesToInclude == null || fieldNamesToInclude.contains(field.getName()))
+         .toList();
+
       //////////////////////////////////////////////////////////////////////////////////////////////
       // first iterate over the table's fields, looking for them (at their backend name (path,    //
       // if it has dots) inside the document note that we'll remove values from the document      //
       // as we go - then after this loop, will handle all remaining values as unstructured fields //
       //////////////////////////////////////////////////////////////////////////////////////////////
       Map<String, Serializable> values = record.getValues();
-      for(QFieldMetaData field : table.getFields().values())
+      for(QFieldMetaData field : selectedFields)
       {
          String fieldName        = field.getName();
          String fieldBackendName = getFieldBackendName(field);
@@ -558,7 +569,7 @@ public class AbstractMongoDBAction
             {
                try
                {
-                  valueListIterator.set(expression.evaluate());
+                  valueListIterator.set(expression.evaluate(field));
                }
                catch(QException qe)
                {

qqq-backend-module-mongodb/src/main/java/com/kingsrook/qqq/backend/module/mongodb/actions/MongoDBQueryAction.java

@@ -41,6 +41,7 @@ import com.kingsrook.qqq.backend.module.mongodb.model.metadata.MongoDBBackendMet
 import com.mongodb.client.FindIterable;
 import com.mongodb.client.MongoCollection;
 import com.mongodb.client.MongoDatabase;
+import com.mongodb.client.model.Projections;
 import org.bson.Document;
 import org.bson.conversions.Bson;
 
@@ -96,6 +97,15 @@ public class MongoDBQueryAction extends AbstractMongoDBAction implements QueryIn
          ////////////////////////////////////////////////////////////
          FindIterable<Document> cursor = collection.find(mongoClientContainer.getMongoSession(), searchQuery);
 
+         ///////////////////////////////////////////////////////////////////////////////////////////////
+         // if input specifies a set of field names to include, then add a 'projection' to the cursor //
+         ///////////////////////////////////////////////////////////////////////////////////////////////
+         if(queryInput.getFieldNamesToInclude() != null)
+         {
+            List<String> backendFieldNames = queryInput.getFieldNamesToInclude().stream().map(f -> getFieldBackendName(table.getField(f))).toList();
+            cursor.projection(Projections.include(backendFieldNames));
+         }
+
          ///////////////////////////////////
          // add a sort operator if needed //
          ///////////////////////////////////
@@ -138,7 +148,7 @@ public class MongoDBQueryAction extends AbstractMongoDBAction implements QueryIn
             actionTimeoutHelper.cancel();
             setQueryStatFirstResultTime();
 
-            QRecord record = documentToRecord(table, document);
+            QRecord record = documentToRecord(queryInput, document);
             queryOutput.addRecord(record);
 
             if(queryInput.getAsyncJobCallback().wasCancelRequested())

qqq-backend-module-mongodb/src/test/java/com/kingsrook/qqq/backend/module/mongodb/actions/MongoDBQueryActionTest.java

@@ -29,6 +29,7 @@ import java.util.Collections;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
+import java.util.Set;
 import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
 import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
 import com.kingsrook.qqq.backend.core.context.QContext;
@@ -54,6 +55,8 @@ import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
 import static org.assertj.core.api.Assertions.assertThat;
 import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
 
 
 /*******************************************************************************
@@ -994,4 +997,46 @@ class MongoDBQueryActionTest extends BaseTest
          .allMatch(r -> r.getValueInteger("storeKey").equals(1));
    }
 
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testFieldNamesToInclude() throws QException
+   {
+      QQueryFilter  filter     = new QQueryFilter().withCriteria("firstName", QCriteriaOperator.EQUALS, "Darin");
+      QueryInput    queryInput = new QueryInput(TestUtils.TABLE_NAME_PERSON).withFilter(filter);
+
+      QRecord record = new QueryAction().execute(queryInput.withFieldNamesToInclude(null)).getRecords().get(0);
+      assertTrue(record.getValues().containsKey("id"));
+      assertTrue(record.getValues().containsKey("firstName"));
+      assertTrue(record.getValues().containsKey("createDate"));
+      //////////////////////////////////////////////////////////////////////////////
+      // note, we get an extra "metaData" field (??), which, i guess is expected? //
+      //////////////////////////////////////////////////////////////////////////////
+      assertEquals(QContext.getQInstance().getTable(TestUtils.TABLE_NAME_PERSON).getFields().size() + 1, record.getValues().size());
+
+      record = new QueryAction().execute(queryInput.withFieldNamesToInclude(Set.of("id", "firstName"))).getRecords().get(0);
+      assertTrue(record.getValues().containsKey("id"));
+      assertTrue(record.getValues().containsKey("firstName"));
+      assertFalse(record.getValues().containsKey("createDate"));
+      assertEquals(2, record.getValues().size());
+      //////////////////////////////////////////////////////////////////////////////////////////////
+      // here, we'd have put _id (which mongo always returns) as "id", since caller requested it. //
+      //////////////////////////////////////////////////////////////////////////////////////////////
+      assertFalse(record.getValues().containsKey("_id"));
+
+      record = new QueryAction().execute(queryInput.withFieldNamesToInclude(Set.of("homeTown"))).getRecords().get(0);
+      assertFalse(record.getValues().containsKey("id"));
+      assertFalse(record.getValues().containsKey("firstName"));
+      assertFalse(record.getValues().containsKey("createDate"));
+      assertTrue(record.getValues().containsKey("homeTown"));
+      //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      // here, mongo always gives back _id (but, we won't have re-mapped it to "id", since caller didn't request it), so, do expect 2 fields here //
+      //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      assertEquals(2, record.getValues().size());
+      assertTrue(record.getValues().containsKey("_id"));
+   }
+
 }

qqq-backend-module-rdbms/pom.xml

@@ -81,6 +81,11 @@
          <artifactId>junit-jupiter-engine</artifactId>
          <scope>test</scope>
       </dependency>
+      <dependency>
+         <groupId>org.junit.jupiter</groupId>
+         <artifactId>junit-jupiter-params</artifactId>
+         <scope>test</scope>
+      </dependency>
       <dependency>
          <groupId>org.assertj</groupId>
          <artifactId>assertj-core</artifactId>

qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/AbstractRDBMSAction.java

@@ -676,7 +676,7 @@ public abstract class AbstractRDBMSAction
                {
                   try
                   {
-                     valueListIterator.set(expression.evaluate());
+                     valueListIterator.set(expression.evaluate(field));
                   }
                   catch(QException qe)
                   {

qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSInsertAction.java

@@ -146,8 +146,8 @@ public class RDBMSInsertAction extends AbstractRDBMSAction implements InsertInte
             // todo sql customization - can edit sql and/or param list
             // todo - non-serial-id style tables
             // todo - other generated values, e.g., createDate...  maybe need to re-select?
-            List<Integer> idList = QueryManager.executeInsertForGeneratedIds(connection, sql.toString(), params);
-            int           index  = 0;
+            List<Serializable> idList = QueryManager.executeInsertForGeneratedIds(connection, sql.toString(), params, table.getField(table.getPrimaryKeyField()).getType());
+            int                index  = 0;
             for(QRecord record : page)
             {
                QRecord outputRecord = new QRecord(record);
@@ -155,7 +155,7 @@ public class RDBMSInsertAction extends AbstractRDBMSAction implements InsertInte
 
                if(CollectionUtils.nullSafeIsEmpty(record.getErrors()))
                {
-                  Integer id = idList.get(index++);
+                  Serializable id = idList.get(index++);
                   outputRecord.setValue(table.getPrimaryKeyField(), id);
                }
             }

qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSQueryAction.java

@@ -34,6 +34,7 @@ import java.util.HashMap;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 import java.util.concurrent.TimeUnit;
 import java.util.stream.Collectors;
 import com.kingsrook.qqq.backend.core.actions.interfaces.QueryInterface;
@@ -94,7 +95,8 @@ public class RDBMSQueryAction extends AbstractRDBMSAction implements QueryInterf
          QTableMetaData table     = queryInput.getTable();
          String         tableName = queryInput.getTableName();
 
-         StringBuilder sql = new StringBuilder(makeSelectClause(queryInput));
+         Selection     selection = makeSelection(queryInput);
+         StringBuilder sql = new StringBuilder(selection.selectClause());
 
          QQueryFilter filter       = clonedOrNewFilter(queryInput.getFilter());
          JoinsContext joinsContext = new JoinsContext(QContext.getQInstance(), tableName, queryInput.getQueryJoins(), filter);
@@ -135,23 +137,6 @@ public class RDBMSQueryAction extends AbstractRDBMSAction implements QueryInterf
             needToCloseConnection = true;
          }
 
-         ////////////////////////////////////////////////////////////////////////////
-         // build the list of fields that will be processed in the result-set loop //
-         ////////////////////////////////////////////////////////////////////////////
-         List<QFieldMetaData> fieldList = new ArrayList<>(table.getFields().values().stream().toList());
-         for(QueryJoin queryJoin : CollectionUtils.nonNullList(queryInput.getQueryJoins()))
-         {
-            if(queryJoin.getSelect())
-            {
-               QTableMetaData joinTable        = QContext.getQInstance().getTable(queryJoin.getJoinTable());
-               String         tableNameOrAlias = queryJoin.getJoinTableOrItsAlias();
-               for(QFieldMetaData joinField : joinTable.getFields().values())
-               {
-                  fieldList.add(joinField.clone().withName(tableNameOrAlias + "." + joinField.getName()));
-               }
-            }
-         }
-
          Long mark = System.currentTimeMillis();
 
          try
@@ -199,7 +184,7 @@ public class RDBMSQueryAction extends AbstractRDBMSAction implements QueryInterf
 
                   for(int i = 1; i <= metaData.getColumnCount(); i++)
                   {
-                     QFieldMetaData field = fieldList.get(i - 1);
+                     QFieldMetaData field = selection.fields().get(i - 1);
 
                      if(!queryInput.getShouldFetchHeavyFields() && field.getIsHeavy())
                      {
@@ -294,26 +279,62 @@ public class RDBMSQueryAction extends AbstractRDBMSAction implements QueryInterf
 
 
 
+   /***************************************************************************
+    ** output wrapper for makeSelection method.
+    ** - selectClause is everything from SELECT up to (but not including) FROM
+    ** - fields are those being selected, in the same order, and with mutated
+    ** names for join fields.
+    ***************************************************************************/
+   private record Selection(String selectClause, List<QFieldMetaData> fields)
+   {
+
+   }
+
+
+
    /*******************************************************************************
-    **
+    ** For a given queryInput, determine what fields are being selected - returning
+    ** a record containing the SELECT clause, as well as a List of QFieldMetaData
+    ** representing those fields - where - note - the names for fields from join
+    ** tables will be prefixed by the join table nameOrAlias.
     *******************************************************************************/
-   private String makeSelectClause(QueryInput queryInput) throws QException
+   private Selection makeSelection(QueryInput queryInput) throws QException
    {
       QInstance       instance   = QContext.getQInstance();
       String          tableName  = queryInput.getTableName();
       List<QueryJoin> queryJoins = queryInput.getQueryJoins();
       QTableMetaData  table      = instance.getTable(tableName);
 
-      boolean requiresDistinct = queryInput.getSelectDistinct() || doesSelectClauseRequireDistinct(table);
-      String  clausePrefix     = (requiresDistinct) ? "SELECT DISTINCT " : "SELECT ";
+      Set<String> fieldNamesToInclude = queryInput.getFieldNamesToInclude();
 
-      List<QFieldMetaData> fieldList = new ArrayList<>(table.getFields().values());
+      ///////////////////////////////////////////////////////////////////////////////////////////////
+      // start with the main table's fields, optionally filtered by the set of fieldNamesToInclude //
+      ///////////////////////////////////////////////////////////////////////////////////////////////
+      List<QFieldMetaData> fieldList = table.getFields().values()
+         .stream().filter(field -> fieldNamesToInclude == null || fieldNamesToInclude.contains(field.getName()))
+         .toList();
+
+      //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      // map those field names to columns, joined with ", ".                                                              //
+      // if a field is heavy, and heavy fields aren't being selected, then replace that field name with a LENGTH function //
+      //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
       String columns = fieldList.stream()
          .map(field -> Pair.of(field, escapeIdentifier(tableName) + "." + escapeIdentifier(getColumnName(field))))
          .map(pair -> wrapHeavyFieldsWithLengthFunctionIfNeeded(pair, queryInput.getShouldFetchHeavyFields()))
          .collect(Collectors.joining(", "));
-      StringBuilder rs = new StringBuilder(clausePrefix).append(columns);
 
+      ///////////////////////////////////////////////////////////////////////////////////////////////////////////
+      // figure out if distinct is being used.  then start building the select clause with the table's columns //
+      ///////////////////////////////////////////////////////////////////////////////////////////////////////////
+      boolean              requiresDistinct   = queryInput.getSelectDistinct() || doesSelectClauseRequireDistinct(table);
+      StringBuilder        selectClause       = new StringBuilder((requiresDistinct) ? "SELECT DISTINCT " : "SELECT ").append(columns);
+      List<QFieldMetaData> selectionFieldList = new ArrayList<>(fieldList);
+
+      boolean needCommaBeforeJoinFields = !columns.isEmpty();
+
+      ///////////////////////////////////
+      // add any 'selected' queryJoins //
+      ///////////////////////////////////
       for(QueryJoin queryJoin : CollectionUtils.nonNullList(queryJoins))
       {
          if(queryJoin.getSelect())
@@ -325,16 +346,41 @@ public class RDBMSQueryAction extends AbstractRDBMSAction implements QueryInterf
                throw new QException("Requested join table [" + queryJoin.getJoinTable() + "] is not a defined table.");
             }
 
-            List<QFieldMetaData> joinFieldList = new ArrayList<>(joinTable.getFields().values());
+            ///////////////////////////////////
+            // filter by fieldNamesToInclude //
+            ///////////////////////////////////
+            List<QFieldMetaData> joinFieldList = joinTable.getFields().values()
+               .stream().filter(field -> fieldNamesToInclude == null || fieldNamesToInclude.contains(tableNameOrAlias + "." + field.getName()))
+               .toList();
+            if(joinFieldList.isEmpty())
+            {
+               continue;
+            }
+
+            /////////////////////////////////////////////////////
+            // map to columns, wrapping heavy fields as needed //
+            /////////////////////////////////////////////////////
             String joinColumns = joinFieldList.stream()
                .map(field -> Pair.of(field, escapeIdentifier(tableNameOrAlias) + "." + escapeIdentifier(getColumnName(field))))
                .map(pair -> wrapHeavyFieldsWithLengthFunctionIfNeeded(pair, queryInput.getShouldFetchHeavyFields()))
                .collect(Collectors.joining(", "));
-            rs.append(", ").append(joinColumns);
+
+            ////////////////////////////////////////////////////////////////////////////////////////////////
+            // append to output objects.                                                                  //
+            // note that fields are cloned, since we are changing their names to have table/alias prefix. //
+            ////////////////////////////////////////////////////////////////////////////////////////////////
+            if(needCommaBeforeJoinFields)
+            {
+               selectClause.append(", ");
+            }
+            selectClause.append(joinColumns);
+            needCommaBeforeJoinFields = true;
+
+            selectionFieldList.addAll(joinFieldList.stream().map(field -> field.clone().withName(tableNameOrAlias + "." + field.getName())).toList());
          }
       }
 
-      return (rs.toString());
+      return (new Selection(selectClause.toString(), selectionFieldList));
    }
 
 

qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/jdbc/QueryManager.java

@@ -53,6 +53,7 @@ import java.util.List;
 import java.util.Map;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.logging.QLogger;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
 import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.PossibleValueEnum;
 import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
 import com.kingsrook.qqq.backend.core.utils.StringUtils;
@@ -526,18 +527,45 @@ public class QueryManager
    /*******************************************************************************
     ** todo - needs (specific) unit test
     *******************************************************************************/
-   public static List<Integer> executeInsertForGeneratedIds(Connection connection, String sql, List<Object> params) throws SQLException
+   public static List<Serializable> executeInsertForGeneratedIds(Connection connection, String sql, List<Object> params, QFieldType idType) throws SQLException
    {
       try(PreparedStatement statement = connection.prepareStatement(sql, Statement.RETURN_GENERATED_KEYS))
       {
          bindParams(params.toArray(), statement);
          incrementStatistic(STAT_QUERIES_RAN);
          statement.executeUpdate();
-         ResultSet     generatedKeys = statement.getGeneratedKeys();
-         List<Integer> rs            = new ArrayList<>();
+
+         /////////////////////////////////////////////////////////////
+         // We default to idType of INTEGER if it was not passed in //
+         /////////////////////////////////////////////////////////////
+         if(idType == null)
+         {
+            idType = QFieldType.INTEGER;
+         }
+
+         ResultSet          generatedKeys = statement.getGeneratedKeys();
+         List<Serializable> rs            = new ArrayList<>();
          while(generatedKeys.next())
          {
-            rs.add(getInteger(generatedKeys, 1));
+            switch(idType)
+            {
+               case INTEGER:
+               {
+                  rs.add(getInteger(generatedKeys, 1));
+                  break;
+               }
+               case LONG:
+               {
+                  rs.add(getLong(generatedKeys, 1));
+                  break;
+               }
+               default:
+               {
+                  LOG.warn("Unknown id data type, attempting to getInteger.", logPair("sql", sql));
+                  rs.add(getInteger(generatedKeys, 1));
+                  break;
+               }
+            }
          }
          return (rs);
       }

qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSCountActionTest.java

@@ -212,7 +212,7 @@ public class RDBMSCountActionTest extends RDBMSActionTest
       CountInput countInput = new CountInput();
       countInput.setTableName(TestUtils.TABLE_NAME_WAREHOUSE);
 
-      assertThat(new CountAction().execute(countInput).getCount()).isEqualTo(1);
+      assertThat(new CountAction().execute(countInput).getCount()).isEqualTo(4);
    }
 
 }

qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSQueryActionJoinsTest.java

@@ -25,6 +25,7 @@ package com.kingsrook.qqq.backend.module.rdbms.actions;
 import java.util.Collections;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 import java.util.concurrent.atomic.AtomicInteger;
 import com.kingsrook.qqq.backend.core.actions.tables.CountAction;
 import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
@@ -770,6 +771,61 @@ public class RDBMSQueryActionJoinsTest extends RDBMSActionTest
 
 
 
+   /*******************************************************************************
+    ** Error seen in CTLive - query for a record in a sub-table, but whose security
+    ** key comes from a main table, but the main-table record doesn't exist.
+    **
+    ** In this QInstance, our warehouse table's security key comes from
+    ** storeWarehouseInt.storeId - so if we insert a warehouse, but no stores, we
+    ** might not be able to find it (if this bug exists!)
+    *******************************************************************************/
+   @Test
+   void testRequestedJoinWithTableWhoseSecurityFieldIsInMainTableAndNoRowIsInMainTable() throws Exception
+   {
+      runTestSql("INSERT INTO warehouse (name) VALUES ('Springfield')", null);
+
+      QueryInput queryInput = new QueryInput();
+      queryInput.setTableName(TestUtils.TABLE_NAME_WAREHOUSE);
+      queryInput.setFilter(new QQueryFilter(new QFilterCriteria("name", QCriteriaOperator.EQUALS, "Springfield")));
+
+      /////////////////////////////////////////
+      // with all access key, should find it //
+      /////////////////////////////////////////
+      QContext.setQSession(new QSession().withSecurityKeyValue(TestUtils.SECURITY_KEY_STORE_ALL_ACCESS, true));
+      assertThat(new QueryAction().execute(queryInput).getRecords()).hasSize(1);
+
+      ////////////////////////////////////////////
+      // with a regular key, should not find it //
+      ////////////////////////////////////////////
+      QContext.setQSession(new QSession().withSecurityKeyValue(TestUtils.TABLE_NAME_STORE, 1));
+      assertThat(new QueryAction().execute(queryInput).getRecords()).hasSize(0);
+
+      /////////////////////////////////////////
+      // now assign the warehouse to a store //
+      /////////////////////////////////////////
+      runTestSql("INSERT INTO warehouse_store_int (store_id, warehouse_id) SELECT 1, id FROM warehouse WHERE name='Springfield'", null);
+
+      /////////////////////////////////////////
+      // with all access key, should find it //
+      /////////////////////////////////////////
+      QContext.setQSession(new QSession().withSecurityKeyValue(TestUtils.SECURITY_KEY_STORE_ALL_ACCESS, true));
+      assertThat(new QueryAction().execute(queryInput).getRecords()).hasSize(1);
+
+      ///////////////////////////////////////////////////////
+      // with a regular key, should find it if key matches //
+      ///////////////////////////////////////////////////////
+      QContext.setQSession(new QSession().withSecurityKeyValue(TestUtils.TABLE_NAME_STORE, 1));
+      assertThat(new QueryAction().execute(queryInput).getRecords()).hasSize(1);
+
+      //////////////////////////////////////////////////////////////////
+      // with a regular key, should not find it if key does not match //
+      //////////////////////////////////////////////////////////////////
+      QContext.setQSession(new QSession().withSecurityKeyValue(TestUtils.TABLE_NAME_STORE, 2));
+      assertThat(new QueryAction().execute(queryInput).getRecords()).hasSize(0);
+
+   }
+
+
    /*******************************************************************************
     **
     *******************************************************************************/
@@ -887,7 +943,10 @@ public class RDBMSQueryActionJoinsTest extends RDBMSActionTest
 
 
    /*******************************************************************************
-    **
+    ** Note, this test was originally written asserting size=1... but reading
+    ** the data, for an all-access key, that seems wrong - as the user should see
+    ** all the records in this table, not just ones associated with a store...
+    ** so, switching to 4 (same issue in CountActionTest too).
     *******************************************************************************/
    @Test
    void testRecordSecurityWithLockFromJoinTableWhereTheKeyIsOnTheManySide() throws QException
@@ -896,8 +955,9 @@ public class RDBMSQueryActionJoinsTest extends RDBMSActionTest
       QueryInput queryInput = new QueryInput();
       queryInput.setTableName(TestUtils.TABLE_NAME_WAREHOUSE);
 
-      assertThat(new QueryAction().execute(queryInput).getRecords())
-         .hasSize(1);
+      List<QRecord> records = new QueryAction().execute(queryInput).getRecords();
+      assertThat(records)
+         .hasSize(4);
    }
 
 
@@ -1052,4 +1112,48 @@ public class RDBMSQueryActionJoinsTest extends RDBMSActionTest
       assertEquals(5, new QueryAction().execute(new QueryInput(TestUtils.TABLE_NAME_PERSON)).getRecords().size());
    }
 
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testFieldNamesToInclude() throws QException
+   {
+      QueryInput queryInput = initQueryRequest();
+      queryInput.withQueryJoin(new QueryJoin(TestUtils.TABLE_NAME_PERSONAL_ID_CARD).withSelect(true));
+      queryInput.withFieldNamesToInclude(Set.of("firstName", TestUtils.TABLE_NAME_PERSONAL_ID_CARD + ".idNumber"));
+      QueryOutput queryOutput = new QueryAction().execute(queryInput);
+      assertThat(queryOutput.getRecords()).allMatch(r -> r.getValues().containsKey("firstName"));
+      assertThat(queryOutput.getRecords()).allMatch(r -> r.getValues().containsKey(TestUtils.TABLE_NAME_PERSONAL_ID_CARD + ".idNumber"));
+      assertThat(queryOutput.getRecords()).allMatch(r -> r.getValues().size() == 2);
+
+      ////////////////////////////////////////////////////////////////////////////////////////////////////
+      // re-run w/ null fieldNamesToInclude -- and should still see those 2, and more (values size > 2) //
+      ////////////////////////////////////////////////////////////////////////////////////////////////////
+      queryInput.withFieldNamesToInclude(null);
+      queryOutput = new QueryAction().execute(queryInput);
+      assertThat(queryOutput.getRecords()).allMatch(r -> r.getValues().containsKey("firstName"));
+      assertThat(queryOutput.getRecords()).allMatch(r -> r.getValues().containsKey(TestUtils.TABLE_NAME_PERSONAL_ID_CARD + ".idNumber"));
+      assertThat(queryOutput.getRecords()).allMatch(r -> r.getValues().size() > 2);
+
+      ////////////////////////////////////////////////////////////////////////////
+      // regression from original build - where only join fields made sql error //
+      ////////////////////////////////////////////////////////////////////////////
+      queryInput.withFieldNamesToInclude(Set.of(TestUtils.TABLE_NAME_PERSONAL_ID_CARD + ".idNumber"));
+      queryOutput = new QueryAction().execute(queryInput);
+      assertThat(queryOutput.getRecords()).allMatch(r -> r.getValues().containsKey(TestUtils.TABLE_NAME_PERSONAL_ID_CARD + ".idNumber"));
+      assertThat(queryOutput.getRecords()).allMatch(r -> r.getValues().size() == 1);
+
+      //////////////////////////////////////////////////////////////////////////////////////////
+      // similar regression to above, if one of the join tables didn't have anything selected //
+      //////////////////////////////////////////////////////////////////////////////////////////
+      queryInput.withQueryJoin(new QueryJoin(TestUtils.TABLE_NAME_PERSONAL_ID_CARD).withAlias("id2").withSelect(true));
+      queryInput.withFieldNamesToInclude(Set.of("firstName", "id2.idNumber"));
+      queryOutput = new QueryAction().execute(queryInput);
+      assertThat(queryOutput.getRecords()).allMatch(r -> r.getValues().containsKey("firstName"));
+      assertThat(queryOutput.getRecords()).allMatch(r -> r.getValues().containsKey("id2.idNumber"));
+      assertThat(queryOutput.getRecords()).allMatch(r -> r.getValues().size() == 2);
+   }
+
 }

qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSQueryActionTest.java

@@ -24,10 +24,14 @@ package com.kingsrook.qqq.backend.module.rdbms.actions;
 
 import java.io.Serializable;
 import java.time.Instant;
+import java.time.LocalDate;
+import java.time.ZoneId;
 import java.time.temporal.ChronoUnit;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
+import java.util.Set;
+import java.util.function.Consumer;
 import java.util.function.Predicate;
 import com.kingsrook.qqq.backend.core.actions.QBackendTransaction;
 import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
@@ -37,22 +41,29 @@ import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.expressions.Now;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.expressions.NowWithOffset;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.expressions.ThisOrLastPeriod;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
 import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
 import com.kingsrook.qqq.backend.core.model.metadata.security.RecordSecurityLock;
 import com.kingsrook.qqq.backend.core.model.session.QSession;
+import com.kingsrook.qqq.backend.core.utils.lambdas.UnsafeFunction;
 import com.kingsrook.qqq.backend.module.rdbms.TestUtils;
 import org.junit.jupiter.api.AfterEach;
-import org.junit.jupiter.api.Assertions;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
 import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
 import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
 
 
 /*******************************************************************************
@@ -82,6 +93,7 @@ public class RDBMSQueryActionTest extends RDBMSActionTest
    void afterEach()
    {
       AbstractRDBMSAction.setLogSQL(false);
+      QContext.getQSession().removeValue(QSession.VALUE_KEY_USER_TIMEZONE);
    }
 
 
@@ -166,7 +178,7 @@ public class RDBMSQueryActionTest extends RDBMSActionTest
       );
       QueryOutput queryOutput = new RDBMSQueryAction().execute(queryInput);
       assertEquals(4, queryOutput.getRecords().size(), "Expected # of rows");
-      Assertions.assertTrue(queryOutput.getRecords().stream().noneMatch(r -> r.getValueString("email").equals(email)), "Should NOT find expected email address");
+      assertTrue(queryOutput.getRecords().stream().noneMatch(r -> r.getValueString("email").equals(email)), "Should NOT find expected email address");
    }
 
 
@@ -199,7 +211,7 @@ public class RDBMSQueryActionTest extends RDBMSActionTest
             .withValues(List.of(1_000_000))));
       queryOutput = new RDBMSQueryAction().execute(queryInput);
       assertEquals(4, queryOutput.getRecords().size(), "Expected # of rows");
-      Assertions.assertTrue(queryOutput.getRecords().stream().noneMatch(r -> Objects.equals(1_000_000, r.getValueInteger("annualSalary"))), "Should NOT find expected salary");
+      assertTrue(queryOutput.getRecords().stream().noneMatch(r -> Objects.equals(1_000_000, r.getValueInteger("annualSalary"))), "Should NOT find expected salary");
    }
 
 
@@ -219,7 +231,7 @@ public class RDBMSQueryActionTest extends RDBMSActionTest
       );
       QueryOutput queryOutput = new RDBMSQueryAction().execute(queryInput);
       assertEquals(2, queryOutput.getRecords().size(), "Expected # of rows");
-      Assertions.assertTrue(queryOutput.getRecords().stream().allMatch(r -> r.getValueInteger("id").equals(2) || r.getValueInteger("id").equals(4)), "Should find expected ids");
+      assertTrue(queryOutput.getRecords().stream().allMatch(r -> r.getValueInteger("id").equals(2) || r.getValueInteger("id").equals(4)), "Should find expected ids");
    }
 
 
@@ -239,7 +251,7 @@ public class RDBMSQueryActionTest extends RDBMSActionTest
       );
       QueryOutput queryOutput = new RDBMSQueryAction().execute(queryInput);
       assertEquals(2, queryOutput.getRecords().size(), "Expected # of rows");
-      Assertions.assertTrue(queryOutput.getRecords().stream().allMatch(r -> r.getValueInteger("id").equals(1) || r.getValueInteger("id").equals(5)), "Should find expected ids");
+      assertTrue(queryOutput.getRecords().stream().allMatch(r -> r.getValueInteger("id").equals(1) || r.getValueInteger("id").equals(5)), "Should find expected ids");
    }
 
 
@@ -259,7 +271,7 @@ public class RDBMSQueryActionTest extends RDBMSActionTest
       );
       QueryOutput queryOutput = new RDBMSQueryAction().execute(queryInput);
       assertEquals(1, queryOutput.getRecords().size(), "Expected # of rows");
-      Assertions.assertTrue(queryOutput.getRecords().stream().allMatch(r -> r.getValueString("email").matches("darin.*")), "Should find matching email address");
+      assertTrue(queryOutput.getRecords().stream().allMatch(r -> r.getValueString("email").matches("darin.*")), "Should find matching email address");
    }
 
 
@@ -279,7 +291,7 @@ public class RDBMSQueryActionTest extends RDBMSActionTest
       );
       QueryOutput queryOutput = new RDBMSQueryAction().execute(queryInput);
       assertEquals(1, queryOutput.getRecords().size(), "Expected # of rows");
-      Assertions.assertTrue(queryOutput.getRecords().stream().allMatch(r -> r.getValueString("email").matches(".*kelkhoff.*")), "Should find matching email address");
+      assertTrue(queryOutput.getRecords().stream().allMatch(r -> r.getValueString("email").matches(".*kelkhoff.*")), "Should find matching email address");
    }
 
 
@@ -299,7 +311,7 @@ public class RDBMSQueryActionTest extends RDBMSActionTest
       );
       QueryOutput queryOutput = new RDBMSQueryAction().execute(queryInput);
       assertEquals(1, queryOutput.getRecords().size(), "Expected # of rows");
-      Assertions.assertTrue(queryOutput.getRecords().stream().allMatch(r -> r.getValueString("email").matches(".*kelkhoff.*")), "Should find matching email address");
+      assertTrue(queryOutput.getRecords().stream().allMatch(r -> r.getValueString("email").matches(".*kelkhoff.*")), "Should find matching email address");
    }
 
 
@@ -319,7 +331,7 @@ public class RDBMSQueryActionTest extends RDBMSActionTest
       );
       QueryOutput queryOutput = new RDBMSQueryAction().execute(queryInput);
       assertEquals(4, queryOutput.getRecords().size(), "Expected # of rows");
-      Assertions.assertTrue(queryOutput.getRecords().stream().noneMatch(r -> r.getValueString("email").matches(".*kelkhoff.*")), "Should find matching email address");
+      assertTrue(queryOutput.getRecords().stream().noneMatch(r -> r.getValueString("email").matches(".*kelkhoff.*")), "Should find matching email address");
    }
 
 
@@ -339,7 +351,7 @@ public class RDBMSQueryActionTest extends RDBMSActionTest
       );
       QueryOutput queryOutput = new RDBMSQueryAction().execute(queryInput);
       assertEquals(1, queryOutput.getRecords().size(), "Expected # of rows");
-      Assertions.assertTrue(queryOutput.getRecords().stream().allMatch(r -> r.getValueString("email").matches(".*gmail.com")), "Should find matching email address");
+      assertTrue(queryOutput.getRecords().stream().allMatch(r -> r.getValueString("email").matches(".*gmail.com")), "Should find matching email address");
    }
 
 
@@ -359,7 +371,7 @@ public class RDBMSQueryActionTest extends RDBMSActionTest
       );
       QueryOutput queryOutput = new RDBMSQueryAction().execute(queryInput);
       assertEquals(4, queryOutput.getRecords().size(), "Expected # of rows");
-      Assertions.assertTrue(queryOutput.getRecords().stream().noneMatch(r -> r.getValueString("email").matches("darin.*")), "Should find matching email address");
+      assertTrue(queryOutput.getRecords().stream().noneMatch(r -> r.getValueString("email").matches("darin.*")), "Should find matching email address");
    }
 
 
@@ -379,7 +391,7 @@ public class RDBMSQueryActionTest extends RDBMSActionTest
       );
       QueryOutput queryOutput = new RDBMSQueryAction().execute(queryInput);
       assertEquals(4, queryOutput.getRecords().size(), "Expected # of rows");
-      Assertions.assertTrue(queryOutput.getRecords().stream().noneMatch(r -> r.getValueString("email").matches(".*kelkhoff.*")), "Should find matching email address");
+      assertTrue(queryOutput.getRecords().stream().noneMatch(r -> r.getValueString("email").matches(".*kelkhoff.*")), "Should find matching email address");
    }
 
 
@@ -399,7 +411,7 @@ public class RDBMSQueryActionTest extends RDBMSActionTest
       );
       QueryOutput queryOutput = new RDBMSQueryAction().execute(queryInput);
       assertEquals(4, queryOutput.getRecords().size(), "Expected # of rows");
-      Assertions.assertTrue(queryOutput.getRecords().stream().noneMatch(r -> r.getValueString("email").matches(".*gmail.com")), "Should find matching email address");
+      assertTrue(queryOutput.getRecords().stream().noneMatch(r -> r.getValueString("email").matches(".*gmail.com")), "Should find matching email address");
    }
 
 
@@ -419,7 +431,7 @@ public class RDBMSQueryActionTest extends RDBMSActionTest
       );
       QueryOutput queryOutput = new RDBMSQueryAction().execute(queryInput);
       assertEquals(2, queryOutput.getRecords().size(), "Expected # of rows");
-      Assertions.assertTrue(queryOutput.getRecords().stream().allMatch(r -> r.getValueInteger("id").equals(1) || r.getValueInteger("id").equals(2)), "Should find expected ids");
+      assertTrue(queryOutput.getRecords().stream().allMatch(r -> r.getValueInteger("id").equals(1) || r.getValueInteger("id").equals(2)), "Should find expected ids");
    }
 
 
@@ -439,7 +451,7 @@ public class RDBMSQueryActionTest extends RDBMSActionTest
       );
       QueryOutput queryOutput = new RDBMSQueryAction().execute(queryInput);
       assertEquals(2, queryOutput.getRecords().size(), "Expected # of rows");
-      Assertions.assertTrue(queryOutput.getRecords().stream().allMatch(r -> r.getValueInteger("id").equals(1) || r.getValueInteger("id").equals(2)), "Should find expected ids");
+      assertTrue(queryOutput.getRecords().stream().allMatch(r -> r.getValueInteger("id").equals(1) || r.getValueInteger("id").equals(2)), "Should find expected ids");
    }
 
 
@@ -459,7 +471,7 @@ public class RDBMSQueryActionTest extends RDBMSActionTest
       );
       QueryOutput queryOutput = new RDBMSQueryAction().execute(queryInput);
       assertEquals(2, queryOutput.getRecords().size(), "Expected # of rows");
-      Assertions.assertTrue(queryOutput.getRecords().stream().allMatch(r -> r.getValueInteger("id").equals(4) || r.getValueInteger("id").equals(5)), "Should find expected ids");
+      assertTrue(queryOutput.getRecords().stream().allMatch(r -> r.getValueInteger("id").equals(4) || r.getValueInteger("id").equals(5)), "Should find expected ids");
    }
 
 
@@ -479,7 +491,7 @@ public class RDBMSQueryActionTest extends RDBMSActionTest
       );
       QueryOutput queryOutput = new RDBMSQueryAction().execute(queryInput);
       assertEquals(2, queryOutput.getRecords().size(), "Expected # of rows");
-      Assertions.assertTrue(queryOutput.getRecords().stream().allMatch(r -> r.getValueInteger("id").equals(4) || r.getValueInteger("id").equals(5)), "Should find expected ids");
+      assertTrue(queryOutput.getRecords().stream().allMatch(r -> r.getValueInteger("id").equals(4) || r.getValueInteger("id").equals(5)), "Should find expected ids");
    }
 
 
@@ -498,7 +510,7 @@ public class RDBMSQueryActionTest extends RDBMSActionTest
          ));
       QueryOutput queryOutput = new RDBMSQueryAction().execute(queryInput);
       assertEquals(1, queryOutput.getRecords().size(), "Expected # of rows");
-      Assertions.assertTrue(queryOutput.getRecords().stream().allMatch(r -> r.getValue("birthDate") == null), "Should find expected row");
+      assertTrue(queryOutput.getRecords().stream().allMatch(r -> r.getValue("birthDate") == null), "Should find expected row");
    }
 
 
@@ -517,7 +529,7 @@ public class RDBMSQueryActionTest extends RDBMSActionTest
          ));
       QueryOutput queryOutput = new RDBMSQueryAction().execute(queryInput);
       assertEquals(5, queryOutput.getRecords().size(), "Expected # of rows");
-      Assertions.assertTrue(queryOutput.getRecords().stream().allMatch(r -> r.getValue("firstName") != null), "Should find expected rows");
+      assertTrue(queryOutput.getRecords().stream().allMatch(r -> r.getValue("firstName") != null), "Should find expected rows");
    }
 
 
@@ -537,7 +549,7 @@ public class RDBMSQueryActionTest extends RDBMSActionTest
          ));
       QueryOutput queryOutput = new RDBMSQueryAction().execute(queryInput);
       assertEquals(3, queryOutput.getRecords().size(), "Expected # of rows");
-      Assertions.assertTrue(queryOutput.getRecords().stream().allMatch(r -> r.getValueInteger("id").equals(2) || r.getValueInteger("id").equals(3) || r.getValueInteger("id").equals(4)), "Should find expected ids");
+      assertTrue(queryOutput.getRecords().stream().allMatch(r -> r.getValueInteger("id").equals(2) || r.getValueInteger("id").equals(3) || r.getValueInteger("id").equals(4)), "Should find expected ids");
    }
 
 
@@ -557,7 +569,7 @@ public class RDBMSQueryActionTest extends RDBMSActionTest
          ));
       QueryOutput queryOutput = new RDBMSQueryAction().execute(queryInput);
       assertEquals(2, queryOutput.getRecords().size(), "Expected # of rows");
-      Assertions.assertTrue(queryOutput.getRecords().stream().allMatch(r -> r.getValueInteger("id").equals(1) || r.getValueInteger("id").equals(5)), "Should find expected ids");
+      assertTrue(queryOutput.getRecords().stream().allMatch(r -> r.getValueInteger("id").equals(1) || r.getValueInteger("id").equals(5)), "Should find expected ids");
    }
 
 
@@ -583,7 +595,7 @@ public class RDBMSQueryActionTest extends RDBMSActionTest
             .withCriteria(new QFilterCriteria().withFieldName("birthDate").withOperator(QCriteriaOperator.LESS_THAN).withValues(List.of(new Now()))));
          QueryOutput queryOutput = new RDBMSQueryAction().execute(queryInput);
          assertEquals(1, queryOutput.getRecords().size(), "Expected # of rows");
-         Assertions.assertTrue(queryOutput.getRecords().stream().anyMatch(r -> r.getValue("firstName").equals("past")), "Should find expected row");
+         assertTrue(queryOutput.getRecords().stream().anyMatch(r -> r.getValue("firstName").equals("past")), "Should find expected row");
       }
 
       {
@@ -593,7 +605,7 @@ public class RDBMSQueryActionTest extends RDBMSActionTest
             .withCriteria(new QFilterCriteria().withFieldName("birthDate").withOperator(QCriteriaOperator.LESS_THAN).withValues(List.of(NowWithOffset.plus(2, ChronoUnit.DAYS)))));
          QueryOutput queryOutput = new RDBMSQueryAction().execute(queryInput);
          assertEquals(1, queryOutput.getRecords().size(), "Expected # of rows");
-         Assertions.assertTrue(queryOutput.getRecords().stream().anyMatch(r -> r.getValue("firstName").equals("past")), "Should find expected row");
+         assertTrue(queryOutput.getRecords().stream().anyMatch(r -> r.getValue("firstName").equals("past")), "Should find expected row");
       }
 
       {
@@ -603,13 +615,108 @@ public class RDBMSQueryActionTest extends RDBMSActionTest
             .withCriteria(new QFilterCriteria().withFieldName("birthDate").withOperator(QCriteriaOperator.GREATER_THAN).withValues(List.of(NowWithOffset.minus(5, ChronoUnit.DAYS)))));
          QueryOutput queryOutput = new RDBMSQueryAction().execute(queryInput);
          assertEquals(2, queryOutput.getRecords().size(), "Expected # of rows");
-         Assertions.assertTrue(queryOutput.getRecords().stream().anyMatch(r -> r.getValue("firstName").equals("past")), "Should find expected row");
-         Assertions.assertTrue(queryOutput.getRecords().stream().anyMatch(r -> r.getValue("firstName").equals("future")), "Should find expected row");
+         assertTrue(queryOutput.getRecords().stream().anyMatch(r -> r.getValue("firstName").equals("past")), "Should find expected row");
+         assertTrue(queryOutput.getRecords().stream().anyMatch(r -> r.getValue("firstName").equals("future")), "Should find expected row");
       }
    }
 
 
 
+   /*******************************************************************************
+    ** Adding additional test conditions, specifically for DATE-precision
+    *******************************************************************************/
+   @ParameterizedTest()
+   @ValueSource(strings = { "UTC", "US/Eastern", "UTC+12" })
+   void testMoreFilterExpressions(String userTimezone) throws QException
+   {
+      QContext.getQSession().setValue(QSession.VALUE_KEY_USER_TIMEZONE, userTimezone);
+
+      LocalDate today     = Instant.now().atZone(ZoneId.of(userTimezone)).toLocalDate();
+      LocalDate yesterday = today.minusDays(1);
+      LocalDate tomorrow  = today.plusDays(1);
+
+      new InsertAction().execute(new InsertInput(TestUtils.TABLE_NAME_PERSON).withRecords(List.of(
+         new QRecord().withValue("email", "-").withValue("firstName", "yesterday").withValue("lastName", "ExpressionTest").withValue("birthDate", yesterday),
+         new QRecord().withValue("email", "-").withValue("firstName", "today").withValue("lastName", "ExpressionTest").withValue("birthDate", today),
+         new QRecord().withValue("email", "-").withValue("firstName", "tomorrow").withValue("lastName", "ExpressionTest").withValue("birthDate", tomorrow))
+      ));
+
+      UnsafeFunction<Consumer<QQueryFilter>, List<QRecord>, QException> testFunction = (filterConsumer) ->
+      {
+         QQueryFilter filter = new QQueryFilter().withCriteria("lastName", QCriteriaOperator.EQUALS, "ExpressionTest");
+         filter.withOrderBy(new QFilterOrderBy("birthDate"));
+         filterConsumer.accept(filter);
+
+         return QueryAction.execute(TestUtils.TABLE_NAME_PERSON, filter);
+      };
+
+      assertOneRecordWithFirstName("today", testFunction.apply(filter -> filter.withCriteria(new QFilterCriteria("birthDate", QCriteriaOperator.EQUALS, new Now()))));
+      assertOneRecordWithFirstName("tomorrow", testFunction.apply(filter -> filter.withCriteria(new QFilterCriteria("birthDate", QCriteriaOperator.GREATER_THAN, new Now()))));
+      assertOneRecordWithFirstName("yesterday", testFunction.apply(filter -> filter.withCriteria(new QFilterCriteria("birthDate", QCriteriaOperator.LESS_THAN, new Now()))));
+      assertTwoRecordsWithFirstNames("yesterday", "today", testFunction.apply(filter -> filter.withCriteria(new QFilterCriteria("birthDate", QCriteriaOperator.LESS_THAN_OR_EQUALS, new Now()))));
+      assertTwoRecordsWithFirstNames("today", "tomorrow", testFunction.apply(filter -> filter.withCriteria(new QFilterCriteria("birthDate", QCriteriaOperator.GREATER_THAN_OR_EQUALS, new Now()))));
+
+      assertNoOfRecords(0, testFunction.apply(filter -> filter.withCriteria(new QFilterCriteria("birthDate", QCriteriaOperator.LESS_THAN, NowWithOffset.minus(1, ChronoUnit.DAYS)))));
+      assertNoOfRecords(3, testFunction.apply(filter -> filter.withCriteria(new QFilterCriteria("birthDate", QCriteriaOperator.LESS_THAN_OR_EQUALS, NowWithOffset.plus(1, ChronoUnit.DAYS)))));
+      assertOneRecordWithFirstName("yesterday", testFunction.apply(filter -> filter.withCriteria(new QFilterCriteria("birthDate", QCriteriaOperator.EQUALS, NowWithOffset.minus(1, ChronoUnit.DAYS)))));
+      assertOneRecordWithFirstName("tomorrow", testFunction.apply(filter -> filter.withCriteria(new QFilterCriteria("birthDate", QCriteriaOperator.EQUALS, NowWithOffset.plus(1, ChronoUnit.DAYS)))));
+      assertNoOfRecords(3, testFunction.apply(filter -> filter.withCriteria(new QFilterCriteria("birthDate", QCriteriaOperator.LESS_THAN, NowWithOffset.plus(1, ChronoUnit.WEEKS)))));
+      assertNoOfRecords(3, testFunction.apply(filter -> filter.withCriteria(new QFilterCriteria("birthDate", QCriteriaOperator.LESS_THAN, NowWithOffset.plus(1, ChronoUnit.MONTHS)))));
+      assertNoOfRecords(3, testFunction.apply(filter -> filter.withCriteria(new QFilterCriteria("birthDate", QCriteriaOperator.LESS_THAN, NowWithOffset.plus(1, ChronoUnit.YEARS)))));
+
+      assertThatThrownBy(() -> testFunction.apply(filter -> filter.withCriteria(new QFilterCriteria("birthDate", QCriteriaOperator.LESS_THAN, NowWithOffset.plus(1, ChronoUnit.HOURS)))))
+         .hasRootCauseMessage("Unsupported unit: Hours");
+
+      assertOneRecordWithFirstName("today", testFunction.apply(filter -> filter.withCriteria(new QFilterCriteria("birthDate", QCriteriaOperator.EQUALS, ThisOrLastPeriod.this_(ChronoUnit.DAYS)))));
+      assertOneRecordWithFirstName("yesterday", testFunction.apply(filter -> filter.withCriteria(new QFilterCriteria("birthDate", QCriteriaOperator.EQUALS, ThisOrLastPeriod.last(ChronoUnit.DAYS)))));
+      assertNoOfRecords(3, testFunction.apply(filter -> filter.withCriteria(new QFilterCriteria("birthDate", QCriteriaOperator.GREATER_THAN, ThisOrLastPeriod.last(ChronoUnit.WEEKS)))));
+      assertNoOfRecords(3, testFunction.apply(filter -> filter.withCriteria(new QFilterCriteria("birthDate", QCriteriaOperator.GREATER_THAN, ThisOrLastPeriod.last(ChronoUnit.MONTHS)))));
+      assertNoOfRecords(3, testFunction.apply(filter -> filter.withCriteria(new QFilterCriteria("birthDate", QCriteriaOperator.GREATER_THAN, ThisOrLastPeriod.last(ChronoUnit.YEARS)))));
+      assertNoOfRecords(0, testFunction.apply(filter -> filter.withCriteria(new QFilterCriteria("birthDate", QCriteriaOperator.LESS_THAN, ThisOrLastPeriod.last(ChronoUnit.WEEKS)))));
+      assertNoOfRecords(0, testFunction.apply(filter -> filter.withCriteria(new QFilterCriteria("birthDate", QCriteriaOperator.LESS_THAN, ThisOrLastPeriod.last(ChronoUnit.MONTHS)))));
+      assertNoOfRecords(0, testFunction.apply(filter -> filter.withCriteria(new QFilterCriteria("birthDate", QCriteriaOperator.LESS_THAN, ThisOrLastPeriod.last(ChronoUnit.YEARS)))));
+
+      assertThatThrownBy(() -> testFunction.apply(filter -> filter.withCriteria(new QFilterCriteria("birthDate", QCriteriaOperator.LESS_THAN, ThisOrLastPeriod.this_(ChronoUnit.HOURS)))))
+         .hasRootCauseMessage("Unsupported unit: Hours");
+      assertThatThrownBy(() -> testFunction.apply(filter -> filter.withCriteria(new QFilterCriteria("birthDate", QCriteriaOperator.LESS_THAN, ThisOrLastPeriod.last(ChronoUnit.MINUTES)))))
+         .hasRootCauseMessage("Unsupported unit: Minutes");
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   private void assertNoOfRecords(Integer expectedSize, List<QRecord> actualRecords)
+   {
+      assertEquals(expectedSize, actualRecords.size());
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   private void assertOneRecordWithFirstName(String expectedFirstName, List<QRecord> actualRecords)
+   {
+      assertEquals(1, actualRecords.size());
+      assertEquals(expectedFirstName, actualRecords.get(0).getValueString("firstName"));
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   private void assertTwoRecordsWithFirstNames(String expectedFirstName0, String expectedFirstName1, List<QRecord> actualRecords)
+   {
+      assertEquals(2, actualRecords.size());
+      assertEquals(expectedFirstName0, actualRecords.get(0).getValueString("firstName"));
+      assertEquals(expectedFirstName1, actualRecords.get(1).getValueString("firstName"));
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/
@@ -1005,7 +1112,36 @@ public class RDBMSQueryActionTest extends RDBMSActionTest
       queryInput.setShouldFetchHeavyFields(true);
       records = new QueryAction().execute(queryInput).getRecords();
       assertThat(records).describedAs("Some records should have the heavy homeTown field set when heavies are requested").anyMatch(r -> r.getValue("homeTown") != null);
+   }
 
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testFieldNamesToInclude() throws QException
+   {
+      QQueryFilter filter     = new QQueryFilter().withCriteria("id", QCriteriaOperator.EQUALS, 1);
+      QueryInput   queryInput = new QueryInput(TestUtils.TABLE_NAME_PERSON).withFilter(filter);
+
+      QRecord record = new QueryAction().execute(queryInput.withFieldNamesToInclude(null)).getRecords().get(0);
+      assertTrue(record.getValues().containsKey("id"));
+      assertTrue(record.getValues().containsKey("firstName"));
+      assertTrue(record.getValues().containsKey("createDate"));
+      assertEquals(QContext.getQInstance().getTable(TestUtils.TABLE_NAME_PERSON).getFields().size(), record.getValues().size());
+
+      record = new QueryAction().execute(queryInput.withFieldNamesToInclude(Set.of("id", "firstName"))).getRecords().get(0);
+      assertTrue(record.getValues().containsKey("id"));
+      assertTrue(record.getValues().containsKey("firstName"));
+      assertFalse(record.getValues().containsKey("createDate"));
+      assertEquals(2, record.getValues().size());
+
+      record = new QueryAction().execute(queryInput.withFieldNamesToInclude(Set.of("homeTown"))).getRecords().get(0);
+      assertFalse(record.getValues().containsKey("id"));
+      assertFalse(record.getValues().containsKey("firstName"));
+      assertFalse(record.getValues().containsKey("createDate"));
+      assertEquals(1, record.getValues().size());
    }
 
 }

qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/jdbc/C3P0PooledConnectionProviderTest.java

@@ -102,6 +102,14 @@ class C3P0PooledConnectionProviderTest extends BaseTest
       backend.setConnectionProvider(new QCodeReference(C3P0PooledConnectionProvider.class));
       QContext.init(qInstance, new QSession());
 
+      /////////////////////////////////////////////////////////////////////////////////
+      // sometimes we're seeing this test fail w/ only 2 connections in the pool...  //
+      // theory is, maybe, the pool doesn't quite have enough time to open them all? //
+      // so, try adding a little sleep here.                                         //
+      /////////////////////////////////////////////////////////////////////////////////
+      new QueryAction().execute(new QueryInput(TestUtils.TABLE_NAME_PERSON));
+      SleepUtils.sleep(500, TimeUnit.MILLISECONDS);
+
       for(int i = 0; i < 5; i++)
       {
          new QueryAction().execute(new QueryInput(TestUtils.TABLE_NAME_PERSON));
@@ -110,17 +118,18 @@ class C3P0PooledConnectionProviderTest extends BaseTest
       JSONObject debugValues = getDebugStateValues(true);
       assertThat(debugValues.getInt("numConnections")).isBetween(3, 6); // due to potential timing issues, sometimes pool will acquire another 3 conns, so 3 or 6 seems ok.
 
-      ////////////////////////////////////////////////////////////////////
-      // open up 4 transactions - confirm the pool opens some new conns //
-      ////////////////////////////////////////////////////////////////////
+      //////////////////////////////////////////////////////////////////////////
+      // open up several transactions - confirm the pool opens some new conns //
+      //////////////////////////////////////////////////////////////////////////
+      int noTransactions = 7;
       List<QBackendTransaction> transactions = new ArrayList<>();
-      for(int i = 0; i < 5; i++)
+      for(int i = 0; i < noTransactions; i++)
       {
          transactions.add(QBackendTransaction.openFor(new InsertInput(TestUtils.TABLE_NAME_PERSON)));
       }
 
       debugValues = getDebugStateValues(true);
-      assertThat(debugValues.getInt("numConnections")).isGreaterThan(3);
+      assertThat(debugValues.getInt("numConnections")).isGreaterThanOrEqualTo(noTransactions);
 
       transactions.forEach(transaction -> transaction.close());
 
@@ -128,7 +137,7 @@ class C3P0PooledConnectionProviderTest extends BaseTest
       // might take a second for the pool to re-claim the closed connections //
       /////////////////////////////////////////////////////////////////////////
       boolean foundMatch = false;
-      for(int i = 0; i < 5; i++)
+      for(int i = 0; i < noTransactions; i++)
       {
          debugValues = getDebugStateValues(true);
          if(debugValues.getInt("numConnections") == debugValues.getInt("numIdleConnections"))

qqq-dev-tools/CURRENT-SNAPSHOT-VERSION

@@ -1 +1 @@
-0.21.0
+0.23.0

qqq-middleware-api/src/main/java/com/kingsrook/qqq/api/model/metadata/processes/ApiProcessInputFieldsContainer.java

@@ -43,32 +43,6 @@ public class ApiProcessInputFieldsContainer
 
 
 
-   /***************************************************************************
-    ** factory method to build one of these containers using all of the input fields
-    ** in a process
-    ***************************************************************************/
-   public static ApiProcessInputFieldsContainer forAllInputFields(QProcessMetaData process)
-   {
-      return forFields(process.getInputFields());
-   }
-
-
-
-   /***************************************************************************
-    ** factory method to build one of these containers using a list of fields.
-    ***************************************************************************/
-   public static ApiProcessInputFieldsContainer forFields(List<QFieldMetaData> fields)
-   {
-      ApiProcessInputFieldsContainer container = new ApiProcessInputFieldsContainer();
-      for(QFieldMetaData inputField : CollectionUtils.nonNullList(fields))
-      {
-         container.withField(inputField);
-      }
-      return (container);
-   }
-
-
-
    /*******************************************************************************
     ** find all input fields in frontend steps of the process, and add them as fields
     ** in this container.

qqq-middleware-api/src/main/java/com/kingsrook/qqq/api/model/metadata/processes/ApiProcessSummaryListOutput.java

@@ -48,9 +48,7 @@ import org.eclipse.jetty.http.HttpStatus;
 
 
 /*******************************************************************************
- ** For a process that puts "processResults" in its output (as a list of
- ** ProcessSummaryLineInterface objects) - this class converts such an object
- ** to a suitable ApiProcessOutput.
+ **
  *******************************************************************************/
 public class ApiProcessSummaryListOutput implements ApiProcessOutputInterface
 {
@@ -145,31 +143,22 @@ public class ApiProcessSummaryListOutput implements ApiProcessOutputInterface
          {
             if(processSummaryLineInterface instanceof ProcessSummaryLine processSummaryLine)
             {
+               processSummaryLine.setCount(1);
+               processSummaryLine.pickMessage(true);
+
                List<Serializable> primaryKeys = processSummaryLine.getPrimaryKeys();
                if(CollectionUtils.nullSafeHasContents(primaryKeys))
                {
-                  ////////////////////////////////////////////////////////////////////////////
-                  // if there are primary keys in the line, then we'll loop over those, and //
-                  // output an object in the API output for each one - and we'll make the   //
-                  // line appear to be a singular-past-tense line about that individual key //
-                  ////////////////////////////////////////////////////////////////////////////
-                  processSummaryLine.setCount(1);
-                  processSummaryLine.pickMessage(true);
-
                   for(Serializable primaryKey : primaryKeys)
                   {
-                     HashMap<String, Serializable> map = toMap(processSummaryLine, false);
+                     HashMap<String, Serializable> map = toMap(processSummaryLine);
                      map.put("id", primaryKey);
                      apiOutput.add(map);
                   }
                }
                else
                {
-                  //////////////////////////////////////////////////////////////////////////
-                  // otherwise, handle a line without pkeys as a single output map/object //
-                  //////////////////////////////////////////////////////////////////////////
-                  HashMap<String, Serializable> map = toMap(processSummaryLine, true);
-                  apiOutput.add(map);
+                  apiOutput.add(toMap(processSummaryLine));
                }
             }
             else if(processSummaryLineInterface instanceof ProcessSummaryRecordLink processSummaryRecordLink)
@@ -230,19 +219,12 @@ public class ApiProcessSummaryListOutput implements ApiProcessOutputInterface
    /*******************************************************************************
     **
     *******************************************************************************/
-   private static HashMap<String, Serializable> toMap(ProcessSummaryLine processSummaryLine, boolean tryToIncludeCount)
+   private static HashMap<String, Serializable> toMap(ProcessSummaryLine processSummaryLine)
    {
       HashMap<String, Serializable> map = initResultMapForProcessSummaryLine(processSummaryLine);
 
       String messagePrefix = getResultMapMessagePrefix(processSummaryLine);
-
-      String messageSuffix = processSummaryLine.getMessage();
-      if(tryToIncludeCount && processSummaryLine.getCount() != null)
-      {
-         messageSuffix = processSummaryLine.getCount() + " " + messageSuffix;
-      }
-
-      map.put("message", messagePrefix + messageSuffix);
+      map.put("message", messagePrefix + processSummaryLine.getMessage());
 
       return (map);
    }

qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinImplementation.java

@@ -114,6 +114,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.fields.FieldAdornment;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
 import com.kingsrook.qqq.backend.core.model.metadata.frontend.QFrontendVariant;
+import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.PossibleValueSearchFilterUseCase;
 import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSource;
 import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
@@ -125,6 +126,7 @@ import com.kingsrook.qqq.backend.core.modules.authentication.implementations.Aut
 import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
 import com.kingsrook.qqq.backend.core.utils.ExceptionUtils;
 import com.kingsrook.qqq.backend.core.utils.JsonUtils;
+import com.kingsrook.qqq.backend.core.utils.ObjectUtils;
 import com.kingsrook.qqq.backend.core.utils.StringUtils;
 import com.kingsrook.qqq.backend.core.utils.ValueUtils;
 import com.kingsrook.qqq.backend.core.utils.collections.MapBuilder;
@@ -1277,6 +1279,11 @@ public class QJavalinImplementation
             queryInput.getFilter().setLimit(limit);
          }
 
+         if(queryInput.getFilter() == null || queryInput.getFilter().getLimit() == null)
+         {
+            handleQueryNullLimit(context, queryInput);
+         }
+
          List<QueryJoin> queryJoins = processQueryJoinsParam(context);
          queryInput.setQueryJoins(queryJoins);
 
@@ -1297,6 +1304,28 @@ public class QJavalinImplementation
 
 
 
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   private static void handleQueryNullLimit(Context context, QueryInput queryInput)
+   {
+      boolean allowed = javalinMetaData.getQueryWithoutLimitAllowed();
+      if(!allowed)
+      {
+         if(queryInput.getFilter() == null)
+         {
+            queryInput.setFilter(new QQueryFilter());
+         }
+
+         queryInput.getFilter().setLimit(javalinMetaData.getQueryWithoutLimitDefault());
+         LOG.log(javalinMetaData.getQueryWithoutLimitLogLevel(), "Query request did not specify a limit, which is not allowed.  Using default instead", null,
+            logPair("defaultLimit", javalinMetaData.getQueryWithoutLimitDefault()),
+            logPair("path", context.path()));
+      }
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/
@@ -1792,7 +1821,11 @@ public class QJavalinImplementation
          }
 
          defaultQueryFilter = field.getPossibleValueSourceFilter().clone();
-         defaultQueryFilter.interpretValues(values);
+
+         String                           useCaseParam = QJavalinUtils.getQueryParamOrFormParam(context, "useCase");
+         PossibleValueSearchFilterUseCase useCase      = ObjectUtils.tryElse(() -> PossibleValueSearchFilterUseCase.valueOf(useCaseParam.toUpperCase()), PossibleValueSearchFilterUseCase.FORM);
+
+         defaultQueryFilter.interpretValues(values, useCase);
       }
 
       finishPossibleValuesRequest(context, field.getPossibleValueSourceName(), defaultQueryFilter);

qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinMetaData.java

@@ -23,6 +23,7 @@ package com.kingsrook.qqq.backend.javalin;
 
 
 import java.util.function.Function;
+import org.apache.logging.log4j.Level;
 
 
 /*******************************************************************************
@@ -36,6 +37,10 @@ public class QJavalinMetaData
 
    private Function<QJavalinAccessLogger.LogEntry, Boolean> logFilter;
 
+   private boolean queryWithoutLimitAllowed  = false;
+   private Integer queryWithoutLimitDefault  = 1000;
+   private Level   queryWithoutLimitLogLevel = Level.INFO;
+
 
 
    /*******************************************************************************
@@ -143,4 +148,97 @@ public class QJavalinMetaData
       return (this);
    }
 
+
+
+   /*******************************************************************************
+    ** Getter for queryWithoutLimitAllowed
+    *******************************************************************************/
+   public boolean getQueryWithoutLimitAllowed()
+   {
+      return (this.queryWithoutLimitAllowed);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for queryWithoutLimitAllowed
+    *******************************************************************************/
+   public void setQueryWithoutLimitAllowed(boolean queryWithoutLimitAllowed)
+   {
+      this.queryWithoutLimitAllowed = queryWithoutLimitAllowed;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for queryWithoutLimitAllowed
+    *******************************************************************************/
+   public QJavalinMetaData withQueryWithoutLimitAllowed(boolean queryWithoutLimitAllowed)
+   {
+      this.queryWithoutLimitAllowed = queryWithoutLimitAllowed;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for queryWithoutLimitDefault
+    *******************************************************************************/
+   public Integer getQueryWithoutLimitDefault()
+   {
+      return (this.queryWithoutLimitDefault);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for queryWithoutLimitDefault
+    *******************************************************************************/
+   public void setQueryWithoutLimitDefault(Integer queryWithoutLimitDefault)
+   {
+      this.queryWithoutLimitDefault = queryWithoutLimitDefault;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for queryWithoutLimitDefault
+    *******************************************************************************/
+   public QJavalinMetaData withQueryWithoutLimitDefault(Integer queryWithoutLimitDefault)
+   {
+      this.queryWithoutLimitDefault = queryWithoutLimitDefault;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for queryWithoutLimitLogLevel
+    *******************************************************************************/
+   public Level getQueryWithoutLimitLogLevel()
+   {
+      return (this.queryWithoutLimitLogLevel);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for queryWithoutLimitLogLevel
+    *******************************************************************************/
+   public void setQueryWithoutLimitLogLevel(Level queryWithoutLimitLogLevel)
+   {
+      this.queryWithoutLimitLogLevel = queryWithoutLimitLogLevel;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for queryWithoutLimitLogLevel
+    *******************************************************************************/
+   public QJavalinMetaData withQueryWithoutLimitLogLevel(Level queryWithoutLimitLogLevel)
+   {
+      this.queryWithoutLimitLogLevel = queryWithoutLimitLogLevel;
+      return (this);
+   }
+
 }

qqq-middleware-javalin/src/test/java/com/kingsrook/qqq/backend/javalin/QJavalinImplementationTest.java

@@ -33,6 +33,8 @@ import java.util.concurrent.TimeUnit;
 import java.util.function.Function;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.exceptions.QInstanceValidationException;
+import com.kingsrook.qqq.backend.core.logging.QCollectingLogger;
+import com.kingsrook.qqq.backend.core.logging.QLogger;
 import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportFormat;
 import com.kingsrook.qqq.backend.core.model.dashboard.widgets.WidgetType;
 import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
@@ -45,6 +47,7 @@ import com.kingsrook.qqq.backend.core.utils.JsonUtils;
 import com.kingsrook.qqq.backend.core.utils.SleepUtils;
 import kong.unirest.HttpResponse;
 import kong.unirest.Unirest;
+import org.apache.logging.log4j.Level;
 import org.eclipse.jetty.http.HttpStatus;
 import org.json.JSONArray;
 import org.json.JSONObject;
@@ -469,6 +472,101 @@ class QJavalinImplementationTest extends QJavalinTestBase
 
 
 
+   /*******************************************************************************
+    ** test a table query using an actual filter via POST, with no limit specified,
+    ** and with that not being allowed.
+    **
+    *******************************************************************************/
+   @Test
+   public void test_dataQueryWithFilterPOSTWithoutLimitNotAllowed() throws QInstanceValidationException
+   {
+      try
+      {
+         qJavalinImplementation.getJavalinMetaData()
+            .withQueryWithoutLimitAllowed(false)
+            .withQueryWithoutLimitDefault(3)
+            .withQueryWithoutLimitLogLevel(Level.WARN);
+
+         QCollectingLogger collectingLogger = QLogger.activateCollectingLoggerForClass(QJavalinImplementation.class);
+
+         String filterJson = """
+            {"criteria":[]}""";
+
+         HttpResponse<String> response = Unirest.post(BASE_URL + "/data/person/query")
+            .field("filter", filterJson)
+            .asString();
+
+         assertEquals(200, response.getStatus());
+         JSONObject jsonObject = JsonUtils.toJSONObject(response.getBody());
+         assertTrue(jsonObject.has("records"));
+         JSONArray records = jsonObject.getJSONArray("records");
+         assertEquals(3, records.length());
+
+         assertThat(collectingLogger.getCollectedMessages())
+            .anyMatch(m -> m.getLevel().equals(Level.WARN) && m.getMessage().contains("Query request did not specify a limit"));
+      }
+      finally
+      {
+         QLogger.activateCollectingLoggerForClass(QJavalinImplementation.class);
+         resetMetaDataQueryWithoutLimitSettings();
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** test a table query using an actual filter via POST, with no limit specified,
+    ** but with that being allowed.
+    **
+    *******************************************************************************/
+   @Test
+   public void test_dataQueryWithFilterPOSTWithoutLimitAllowed() throws QInstanceValidationException
+   {
+      try
+      {
+         qJavalinImplementation.getJavalinMetaData()
+            .withQueryWithoutLimitAllowed(true);
+
+         QCollectingLogger collectingLogger = QLogger.activateCollectingLoggerForClass(QJavalinImplementation.class);
+
+         String filterJson = """
+            {"criteria":[]}""";
+
+         HttpResponse<String> response = Unirest.post(BASE_URL + "/data/person/query")
+            .field("filter", filterJson)
+            .asString();
+
+         assertEquals(200, response.getStatus());
+         JSONObject jsonObject = JsonUtils.toJSONObject(response.getBody());
+         assertTrue(jsonObject.has("records"));
+         JSONArray records = jsonObject.getJSONArray("records");
+         assertEquals(6, records.length());
+
+         assertThat(collectingLogger.getCollectedMessages())
+            .noneMatch(m -> m.getMessage().contains("Query request did not specify a limit"));
+      }
+      finally
+      {
+         QLogger.activateCollectingLoggerForClass(QJavalinImplementation.class);
+         resetMetaDataQueryWithoutLimitSettings();
+      }
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   private void resetMetaDataQueryWithoutLimitSettings()
+   {
+      qJavalinImplementation.getJavalinMetaData()
+         .withQueryWithoutLimitAllowed(false)
+         .withQueryWithoutLimitDefault(1000)
+         .withQueryWithoutLimitLogLevel(Level.INFO);
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/
@@ -950,24 +1048,138 @@ class QJavalinImplementationTest extends QJavalinTestBase
 
 
 
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   private JSONArray assertPossibleValueSuccessfulResponseAndGetOptionsArray(HttpResponse<String> response)
+   {
+      assertEquals(200, response.getStatus());
+      JSONObject jsonObject = JsonUtils.toJSONObject(response.getBody());
+      assertNotNull(jsonObject);
+      return (jsonObject.getJSONArray("options"));
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   private void assertPossibleValueSuccessfulResponseWithNoOptions(HttpResponse<String> response)
+   {
+      assertEquals(200, response.getStatus());
+      JSONObject jsonObject = JsonUtils.toJSONObject(response.getBody());
+      assertNotNull(jsonObject);
+      assertFalse(jsonObject.has("options")); // no results comes back as result w/o options array.
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/
    @Test
    void testPossibleValueWithFilter()
    {
+      /////////////////////////////////////////////////////////////
+      // post with no search term, and values that find a result //
+      /////////////////////////////////////////////////////////////
       HttpResponse<String> response = Unirest.post(BASE_URL + "/data/pet/possibleValues/ownerPersonId?searchTerm=")
          .field("values", """
-         {"email":"tsamples@mmltholdings.com"}
-         """)
+            {"email":"tsamples@mmltholdings.com"}
+            """)
          .asString();
 
-      assertEquals(200, response.getStatus());
-      JSONObject jsonObject = JsonUtils.toJSONObject(response.getBody());
-      assertNotNull(jsonObject);
-      assertNotNull(jsonObject.getJSONArray("options"));
-      assertEquals(1, jsonObject.getJSONArray("options").length());
-      assertEquals("Tyler Samples (4)", jsonObject.getJSONArray("options").getJSONObject(0).getString("label"));
+      JSONArray options = assertPossibleValueSuccessfulResponseAndGetOptionsArray(response);
+      assertNotNull(options);
+      assertEquals(1, options.length());
+      assertEquals("Tyler Samples (4)", options.getJSONObject(0).getString("label"));
+
+      ///////////////////////////////////////////////////////////
+      // post with search term and values that find no results //
+      ///////////////////////////////////////////////////////////
+      response = Unirest.post(BASE_URL + "/data/pet/possibleValues/ownerPersonId?searchTerm=notFound")
+         .field("values", """
+            {"email":"tsamples@mmltholdings.com"}
+            """)
+         .asString();
+      assertPossibleValueSuccessfulResponseWithNoOptions(response);
+
+      ////////////////////////////////////////////////////////////////
+      // post with no search term, but values that cause no matches //
+      ////////////////////////////////////////////////////////////////
+      response = Unirest.post(BASE_URL + "/data/pet/possibleValues/ownerPersonId?searchTerm=")
+         .field("values", """
+            {"email":"noUser@mmltholdings.com"}
+            """)
+         .asString();
+      assertPossibleValueSuccessfulResponseWithNoOptions(response);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testPossibleValueWithFilterMissingValue()
+   {
+      /////////////////////////////////////////////////////////////
+      // filter use-case, with no values, should return options. //
+      /////////////////////////////////////////////////////////////
+      HttpResponse<String> response = Unirest.post(BASE_URL + "/data/pet/possibleValues/ownerPersonId?searchTerm=&useCase=filter").asString();
+      JSONArray            options  = assertPossibleValueSuccessfulResponseAndGetOptionsArray(response);
+      assertNotNull(options);
+      assertThat(options.length()).isGreaterThanOrEqualTo(5);
+
+      ///////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      // similarly, values map, but not the 'email' value, that this PVS field is based on, should return options. //
+      ///////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      response = Unirest.post(BASE_URL + "/data/pet/possibleValues/ownerPersonId?searchTerm=&useCase=filter")
+         .field("values", """
+            {"userId":"123"}
+            """)
+         .asString();
+      options = assertPossibleValueSuccessfulResponseAndGetOptionsArray(response);
+      assertNotNull(options);
+      assertThat(options.length()).isGreaterThanOrEqualTo(5);
+
+      ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      // similarly, values map, with the email value, but an empty string in there - should act the same as if it's missing, and not filter the values. //
+      ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      response = Unirest.post(BASE_URL + "/data/pet/possibleValues/ownerPersonId?searchTerm=&useCase=filter")
+         .field("values", """
+            {"email":""}
+            """)
+         .asString();
+      options = assertPossibleValueSuccessfulResponseAndGetOptionsArray(response);
+      assertNotNull(options);
+      assertThat(options.length()).isGreaterThanOrEqualTo(5);
+
+      /////////////////////////////////////////////////////////////////////////
+      // versus form use-case with no values, which should return 0 options. //
+      /////////////////////////////////////////////////////////////////////////
+      response = Unirest.post(BASE_URL + "/data/pet/possibleValues/ownerPersonId?searchTerm=&useCase=form").asString();
+      assertPossibleValueSuccessfulResponseWithNoOptions(response);
+
+      /////////////////////////////////////////////////////////////////////////////////
+      // versus form use-case with expected value, which should return some options. //
+      /////////////////////////////////////////////////////////////////////////////////
+      response = Unirest.post(BASE_URL + "/data/pet/possibleValues/ownerPersonId?searchTerm=&useCase=form")
+         .field("values", """
+            {"email":"tsamples@mmltholdings.com"}
+            """)
+         .asString();
+      options = assertPossibleValueSuccessfulResponseAndGetOptionsArray(response);
+      assertNotNull(options);
+      assertEquals(1, options.length());
+      assertEquals("Tyler Samples (4)", options.getJSONObject(0).getString("label"));
+
+      ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      // finally an unrecognized useCase (or missing or empty), should behave the same as a form, and return 0 options if the filter-value is missing. //
+      ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      assertPossibleValueSuccessfulResponseWithNoOptions(Unirest.post(BASE_URL + "/data/pet/possibleValues/ownerPersonId?searchTerm=&useCase=notAUseCase").asString());
+      assertPossibleValueSuccessfulResponseWithNoOptions(Unirest.post(BASE_URL + "/data/pet/possibleValues/ownerPersonId?searchTerm=").asString());
+      assertPossibleValueSuccessfulResponseWithNoOptions(Unirest.post(BASE_URL + "/data/pet/possibleValues/ownerPersonId?searchTerm=&useCase=").asString());
    }
 
 

qqq-middleware-javalin/src/test/java/com/kingsrook/qqq/backend/javalin/QJavalinTestBase.java

@@ -102,7 +102,7 @@ public class QJavalinTestBase
       {
          qJavalinImplementation.stopJavalinServer();
       }
-      qJavalinImplementation = new QJavalinImplementation(qInstance);
+      qJavalinImplementation = new QJavalinImplementation(qInstance, new QJavalinMetaData());
       QJavalinProcessHandler.setAsyncStepTimeoutMillis(250);
       qJavalinImplementation.startJavalinServer(PORT);
    }