Merge branch 'rel/0.22.1'

Tag release

.circleci/adjust-pom-version.sh

@@ -7,8 +7,6 @@ fi
 
 if [ "$CIRCLE_BRANCH" == "dev" ] || [ "$CIRCLE_BRANCH" == "staging" ] || [ "$CIRCLE_BRANCH" == "main" ] || [ \! -z $(echo "$CIRCLE_TAG" | grep "^version-") ]; then
    echo "On a primary branch or tag [${CIRCLE_BRANCH}${CIRCLE_TAG}] - will not edit the pom version.";
-   echo "(or do we need to do this for dev...?)"
-   echo "(maybe we do this if the current version is a SNAPSHOT?)"
    exit 0;
 fi
 
@@ -19,9 +17,7 @@ else
 fi
 
 POM=$(dirname $0)/../pom.xml
-UNIQ=$(date +%Y%m%d-%H%M%S)-$(git rev-parse --short HEAD)
-SLUG="${SLUG}-${UNIQ}"
 
-echo "Updating $POM <revision> to: $SLUG"
-sed -i "s/<revision>.*/<revision>$SLUG<\/revision>/" $POM
+echo "Updating $POM <revision> to: $SLUG-SNAPSHOT"
+sed -i "s/<revision>.*/<revision>$SLUG-SNAPSHOT<\/revision>/" $POM
 git diff $POM

README.md

@@ -48,5 +48,3 @@ GNU Affero General Public License for more details.
 
 You should have received a copy of the GNU Affero General Public License
 along with this program.  If not, see <https://www.gnu.org/licenses/>.
-
-

docs/actions/QueryAction.adoc

@@ -33,9 +33,6 @@ If the {link-table} has a `POST_QUERY_CUSTOMIZER` defined, then after records ar
 * `table` - *String, Required* - Name of the table being queried against.
 * `filter` - *<<QQueryFilter>> object* - Specification for what records should be returned, based on *<<QFilterCriteria>>* objects, and how they should be sorted, based on *<<QFilterOrderBy>>* objects.
 If a `filter` is not given, then all rows in the table will be returned by the query.
-* `skip` - *Integer* - Optional number of records to be skipped at the beginning of the result set.
-e.g., for implementing pagination.
-* `limit` - *Integer* - Optional maximum number of records to be returned by the query.
 * `transaction` - *QBackendTransaction object* - Optional transaction object.
 ** Behavior for this object is backend-dependant.
 In an RDBMS backend, this object is generally needed if you want your query to see data that may have been modified within the same transaction.
@@ -55,6 +52,14 @@ But if running a query to provide data as part of a process, then this can gener
 * `shouldMaskPassword` - *boolean, default: true* - Controls whether or not fields with `type` = `PASSWORD` should be masked, or if their actual values should be returned.
 * `queryJoins` - *List of <<QueryJoin>> objects* - Optional list of tables to be joined with the main table being queried.
 See QueryJoin below for further details.
+* `fieldNamesToInclude` - *Set of String* - Optional set of field names to be included in the records.
+** Fields from a queryJoin must be prefixed by the join table's name or alias, and a period.
+Field names from the table being queried should not have any sort of prefix.
+** A `null` set here (default) means to include all fields from the table and any queryJoins set as select=true.
+** An empty set will cause an error, as well any unrecognized field names.
+** `QueryAction` will validate the set of field names, and throw an exception if any unrecognized names are given.
+** _Note that this is an optional feature, which some backend modules may not implement.
+Meaning, they would always return all fields._
 
 ==== QQueryFilter
 A key component of *<<QueryInput>>*, a *QQueryFilter* defines both what records should be included in a query's results (e.g., an SQL `WHERE`), as well as how those results should be sorted (SQL `ORDER BY`).
@@ -68,6 +73,9 @@ In general, multiple *orderBys* can be given (depending on backend implementatio
 ** Each *subFilter* can include its own additional *subFilters*.
 ** Each *subFilter* can specify a different *booleanOperator*.
 ** For example, consider the following *QQueryFilter*, that uses two *subFilters*, and a mix of *booleanOperators*
+* `skip` - *Integer* - Optional number of records to be skipped at the beginning of the result set.
+e.g., for implementing pagination.
+* `limit` - *Integer* - Optional maximum number of records to be returned by the query.
 
 [source,java]
 ----

pom.xml

@@ -46,7 +46,7 @@
    </modules>
 
    <properties>
-      <revision>0.21.0-SNAPSHOT</revision>
+      <revision>0.22.1</revision>
 
       <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
       <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/audits/AuditAction.java

@@ -272,7 +272,7 @@ public class AuditAction extends AbstractQActionFunction<AuditInput, AuditOutput
             List<QRecord> auditDetailRecords = new ArrayList<>();
             for(AuditSingleInput auditSingleInput : CollectionUtils.nonNullList(input.getAuditSingleInputList()))
             {
-               Integer auditId = insertOutput.getRecords().get(i++).getValueInteger("id");
+               Long auditId = insertOutput.getRecords().get(i++).getValueLong("id");
                if(auditId == null)
                {
                   LOG.warn("Missing an id for inserted audit - so won't be able to store its child details...");

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/RecordCustomizerUtilityInterface.java

@@ -24,10 +24,12 @@ package com.kingsrook.qqq.backend.core.actions.customizers;
 
 import java.io.Serializable;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 import java.util.Objects;
 import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.logging.QLogger;
+import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
@@ -143,4 +145,19 @@ public interface RecordCustomizerUtilityInterface
       }
    }
 
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   default Map<Serializable, QRecord> getOldRecordMap(List<QRecord> oldRecordList, UpdateInput updateInput)
+   {
+      Map<Serializable, QRecord> oldRecordMap = new HashMap<>();
+      for(QRecord qRecord : oldRecordList)
+      {
+         oldRecordMap.put(qRecord.getValue(updateInput.getTable().getPrimaryKeyField()), qRecord);
+      }
+
+      return (oldRecordMap);
+   }
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/GenerateReportAction.java

@@ -42,6 +42,7 @@ import com.kingsrook.qqq.backend.core.actions.AbstractQActionFunction;
 import com.kingsrook.qqq.backend.core.actions.async.AsyncRecordPipeLoop;
 import com.kingsrook.qqq.backend.core.actions.customizers.QCodeLoader;
 import com.kingsrook.qqq.backend.core.actions.reporting.customizers.DataSourceQueryInputCustomizer;
+import com.kingsrook.qqq.backend.core.actions.reporting.customizers.ReportCustomRecordSourceInterface;
 import com.kingsrook.qqq.backend.core.actions.reporting.customizers.ReportViewCustomizer;
 import com.kingsrook.qqq.backend.core.actions.tables.CountAction;
 import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
@@ -62,6 +63,8 @@ import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportOutput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.QueryHint;
 import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountOutput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.CriteriaMissingInputValueBehavior;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.FilterUseCase;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.JoinsContext;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
@@ -302,10 +305,19 @@ public class GenerateReportAction extends AbstractQActionFunction<ReportInput, R
       JoinsContext joinsContext = null;
       if(dataSource != null)
       {
+         ///////////////////////////////////////////////////////////////////////////////////////
+         // count records, if applicable, from the data source - for populating into the      //
+         // countByDataSource map, as well as for checking if too many rows (e.g., for excel) //
+         ///////////////////////////////////////////////////////////////////////////////////////
+         countDataSourceRecords(reportInput, dataSource, reportFormat);
+
+         ///////////////////////////////////////////////////////////////////////////////////////////
+         // if there's a source table, set up a joins context, to use below for looking up fields //
+         ///////////////////////////////////////////////////////////////////////////////////////////
          if(StringUtils.hasContent(dataSource.getSourceTable()))
          {
-            joinsContext = new JoinsContext(QContext.getQInstance(), dataSource.getSourceTable(), cloneDataSourceQueryJoins(dataSource), dataSource.getQueryFilter() == null ? null : dataSource.getQueryFilter().clone());
-            countDataSourceRecords(reportInput, dataSource, reportFormat);
+            QQueryFilter queryFilter = dataSource.getQueryFilter() == null ? new QQueryFilter() : dataSource.getQueryFilter().clone();
+            joinsContext = new JoinsContext(QContext.getQInstance(), dataSource.getSourceTable(), dataSource.getQueryJoins(), queryFilter);
          }
       }
 
@@ -329,6 +341,7 @@ public class GenerateReportAction extends AbstractQActionFunction<ReportInput, R
             field.setName(column.getName());
             if(StringUtils.hasContent(column.getLabel()))
             {
+
                field.setLabel(column.getLabel());
             }
             fields.add(field);
@@ -346,23 +359,33 @@ public class GenerateReportAction extends AbstractQActionFunction<ReportInput, R
     *******************************************************************************/
    private void countDataSourceRecords(ReportInput reportInput, QReportDataSource dataSource, ReportFormat reportFormat) throws QException
    {
-      QQueryFilter queryFilter = dataSource.getQueryFilter() == null ? new QQueryFilter() : dataSource.getQueryFilter().clone();
-      setInputValuesInQueryFilter(reportInput, queryFilter);
-
-      CountInput countInput = new CountInput();
-      countInput.setTableName(dataSource.getSourceTable());
-      countInput.setFilter(queryFilter);
-      countInput.setQueryJoins(cloneDataSourceQueryJoins(dataSource));
-      CountOutput countOutput = new CountAction().execute(countInput);
-
-      if(countOutput.getCount() != null)
+      Integer count = null;
+      if(dataSource.getCustomRecordSource() != null)
       {
-         countByDataSource.put(dataSource.getName(), countOutput.getCount());
+         // todo - add `count` method to interface?
+      }
+      else if(StringUtils.hasContent(dataSource.getSourceTable()))
+      {
+         QQueryFilter queryFilter = dataSource.getQueryFilter() == null ? new QQueryFilter() : dataSource.getQueryFilter().clone();
+         setInputValuesInQueryFilter(reportInput, queryFilter);
 
-         if(reportFormat.getMaxRows() != null && countOutput.getCount() > reportFormat.getMaxRows())
+         CountInput countInput = new CountInput();
+         countInput.setTableName(dataSource.getSourceTable());
+         countInput.setFilter(queryFilter);
+         countInput.setQueryJoins(cloneDataSourceQueryJoins(dataSource));
+         CountOutput countOutput = new CountAction().execute(countInput);
+
+         count = countOutput.getCount();
+      }
+
+      if(count != null)
+      {
+         countByDataSource.put(dataSource.getName(), count);
+
+         if(reportFormat.getMaxRows() != null && count > reportFormat.getMaxRows())
          {
             throw (new QUserFacingException("The requested report would include more rows ("
-               + String.format("%,d", countOutput.getCount()) + ") than the maximum allowed ("
+               + String.format("%,d", count) + ") than the maximum allowed ("
                + String.format("%,d", reportFormat.getMaxRows()) + ") for the selected file format (" + reportFormat + ")."));
          }
       }
@@ -423,13 +446,19 @@ public class GenerateReportAction extends AbstractQActionFunction<ReportInput, R
       String        tableLabel    = ObjectUtils.tryElse(() -> QContext.getQInstance().getTable(dataSource.getSourceTable()).getLabel(), Objects.requireNonNullElse(dataSource.getSourceTable(), ""));
       AtomicInteger consumedCount = new AtomicInteger(0);
 
-      /////////////////////////////////////////////////////////////////
-      // run a record pipe loop, over the query for this data source //
-      /////////////////////////////////////////////////////////////////
+      /////////////////////////////////////////////////////////////////////////////////////////////////
+      // run a record pipe loop, over the query (or other data-supplier/source) for this data source //
+      /////////////////////////////////////////////////////////////////////////////////////////////////
       RecordPipe recordPipe = new BufferedRecordPipe(1000);
       new AsyncRecordPipeLoop().run("Report[" + reportInput.getReportName() + "]", null, recordPipe, (callback) ->
       {
-         if(dataSource.getSourceTable() != null)
+         if(dataSource.getCustomRecordSource() != null)
+         {
+            ReportCustomRecordSourceInterface recordSource = QCodeLoader.getAdHoc(ReportCustomRecordSourceInterface.class, dataSource.getCustomRecordSource());
+            recordSource.execute(reportInput, dataSource, recordPipe);
+            return (true);
+         }
+         else if(dataSource.getSourceTable() != null)
          {
             QQueryFilter queryFilter = dataSource.getQueryFilter() == null ? new QQueryFilter() : dataSource.getQueryFilter().clone();
             setInputValuesInQueryFilter(reportInput, queryFilter);
@@ -587,7 +616,56 @@ public class GenerateReportAction extends AbstractQActionFunction<ReportInput, R
          return;
       }
 
-      queryFilter.interpretValues(reportInput.getInputValues());
+      //////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      // for reports defined in meta-data, the established rule is, that missing input variable values are discarded. //
+      // but for non-meta-data reports (e.g., user-saved), we expect an exception for missing values.                 //
+      // so, set those use-cases up.                                                                                  //
+      //////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      FilterUseCase filterUseCase;
+      if(StringUtils.hasContent(reportInput.getReportName()) && QContext.getQInstance().getReport(reportInput.getReportName()) != null)
+      {
+         filterUseCase = new ReportFromMetaDataFilterUseCase();
+      }
+      else
+      {
+         filterUseCase = new ReportNotFromMetaDataFilterUseCase();
+      }
+
+      queryFilter.interpretValues(reportInput.getInputValues(), filterUseCase);
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   private static class ReportFromMetaDataFilterUseCase implements FilterUseCase
+   {
+      /***************************************************************************
+       **
+       ***************************************************************************/
+      @Override
+      public CriteriaMissingInputValueBehavior getDefaultCriteriaMissingInputValueBehavior()
+      {
+         return CriteriaMissingInputValueBehavior.REMOVE_FROM_FILTER;
+      }
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   private static class ReportNotFromMetaDataFilterUseCase implements FilterUseCase
+   {
+      /***************************************************************************
+       **
+       ***************************************************************************/
+      @Override
+      public CriteriaMissingInputValueBehavior getDefaultCriteriaMissingInputValueBehavior()
+      {
+         return CriteriaMissingInputValueBehavior.THROW_EXCEPTION;
+      }
    }
 
 

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/customizers/ReportCustomRecordSourceInterface.java

@@ -0,0 +1,43 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.actions.reporting.customizers;
+
+
+import com.kingsrook.qqq.backend.core.actions.reporting.RecordPipe;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportInput;
+import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportDataSource;
+
+
+/*******************************************************************************
+ ** Interface to be implemented to do a custom source of data for a report
+ ** (instead of just a query against a table).
+ *******************************************************************************/
+public interface ReportCustomRecordSourceInterface
+{
+
+   /***************************************************************************
+    ** Given the report input, put records into the pipe, for the report.
+    ***************************************************************************/
+   void execute(ReportInput reportInput, QReportDataSource reportDataSource, RecordPipe recordPipe) throws QException;
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/excel/poi/ExcelPoiBasedStreamingExportStreamer.java

@@ -124,10 +124,11 @@ public class ExcelPoiBasedStreamingExportStreamer implements ExportStreamerInter
    private Writer              activeSheetWriter = null;
    private StreamedSheetWriter sheetWriter       = null;
 
-   private QReportView                       currentView      = null;
-   private Map<String, List<QFieldMetaData>> fieldsPerView    = new HashMap<>();
-   private Map<String, Integer>              rowsPerView      = new HashMap<>();
-   private Map<String, String>               labelViewsByName = new HashMap<>();
+   private QReportView                       currentView              = null;
+   private Map<String, List<QFieldMetaData>> fieldsPerView            = new HashMap<>();
+   private Map<String, Integer>              rowsPerView              = new HashMap<>();
+   private Map<String, String>               labelViewsByName         = new HashMap<>();
+   private Map<String, String>               sheetReferenceByViewName = new HashMap<>();
 
 
 
@@ -180,6 +181,7 @@ public class ExcelPoiBasedStreamingExportStreamer implements ExportStreamerInter
             String    sheetReference = sheet.getPackagePart().getPartName().getName().substring(1);
             sheetMapByExcelReference.put(sheetReference, sheet);
             sheetMapByViewName.put(view.getName(), sheet);
+            sheetReferenceByViewName.put(view.getName(), sheetReference);
             sheetCounter++;
          }
 
@@ -446,7 +448,7 @@ public class ExcelPoiBasedStreamingExportStreamer implements ExportStreamerInter
          // - with a new output stream writer    //
          // - and with a SpreadsheetWriter       //
          //////////////////////////////////////////
-         zipOutputStream.putNextEntry(new ZipEntry("xl/worksheets/sheet" + this.sheetIndex++ + ".xml"));
+         zipOutputStream.putNextEntry(new ZipEntry(sheetReferenceByViewName.get(view.getName())));
          activeSheetWriter = new OutputStreamWriter(zipOutputStream);
          sheetWriter = new StreamedSheetWriter(activeSheetWriter);
 

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/QueryAction.java

@@ -26,6 +26,7 @@ import java.io.Serializable;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
+import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
@@ -50,6 +51,7 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperat
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryJoin;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
 import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
@@ -64,6 +66,7 @@ import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleDispatcher;
 import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleInterface;
 import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
 import com.kingsrook.qqq.backend.core.utils.ListingHash;
+import com.kingsrook.qqq.backend.core.utils.StringUtils;
 import com.kingsrook.qqq.backend.core.utils.ValueUtils;
 
 
@@ -101,6 +104,8 @@ public class QueryAction
          throw (new QException("A table named [" + queryInput.getTableName() + "] was not found in the active QInstance"));
       }
 
+      validateFieldNamesToInclude(queryInput);
+
       QBackendMetaData backend = queryInput.getBackend();
       postQueryRecordCustomizer = QCodeLoader.getTableCustomizer(table, TableCustomizers.POST_QUERY_RECORD.getRole());
       this.queryInput = queryInput;
@@ -158,6 +163,109 @@ public class QueryAction
 
 
 
+   /***************************************************************************
+    ** if QueryInput contains a set of FieldNamesToInclude, then validate that
+    ** those are known field names in the table being queried, or a selected
+    ** queryJoin.
+    ***************************************************************************/
+   static void validateFieldNamesToInclude(QueryInput queryInput) throws QException
+   {
+      Set<String> fieldNamesToInclude = queryInput.getFieldNamesToInclude();
+      if(fieldNamesToInclude == null)
+      {
+         ////////////////////////////////
+         // null set means select all. //
+         ////////////////////////////////
+         return;
+      }
+
+      if(fieldNamesToInclude.isEmpty())
+      {
+         /////////////////////////////////////
+         // empty set, however, is an error //
+         /////////////////////////////////////
+         throw (new QException("An empty set of fieldNamesToInclude was given as queryInput, which is not allowed."));
+      }
+
+      List<String>                unrecognizedFieldNames = new ArrayList<>();
+      Map<String, QTableMetaData> selectedQueryJoins     = null;
+      for(String fieldName : fieldNamesToInclude)
+      {
+         if(fieldName.contains("."))
+         {
+            ////////////////////////////////////////////////
+            // handle names with dots - fields from joins //
+            ////////////////////////////////////////////////
+            String[] parts = fieldName.split("\\.");
+            if(parts.length != 2)
+            {
+               unrecognizedFieldNames.add(fieldName);
+            }
+            else
+            {
+               String tableOrAlias  = parts[0];
+               String fieldNamePart = parts[1];
+
+               ////////////////////////////////////////////
+               // build map of queryJoins being selected //
+               ////////////////////////////////////////////
+               if(selectedQueryJoins == null)
+               {
+                  selectedQueryJoins = new HashMap<>();
+                  for(QueryJoin queryJoin : CollectionUtils.nonNullList(queryInput.getQueryJoins()))
+                  {
+                     if(queryJoin.getSelect())
+                     {
+                        String         joinTableOrAlias = queryJoin.getJoinTableOrItsAlias();
+                        QTableMetaData joinTable        = QContext.getQInstance().getTable(queryJoin.getJoinTable());
+                        if(joinTable != null)
+                        {
+                           selectedQueryJoins.put(joinTableOrAlias, joinTable);
+                        }
+                     }
+                  }
+               }
+
+               if(!selectedQueryJoins.containsKey(tableOrAlias))
+               {
+                  ///////////////////////////////////////////
+                  // unrecognized tableOrAlias is an error //
+                  ///////////////////////////////////////////
+                  unrecognizedFieldNames.add(fieldName);
+               }
+               else
+               {
+                  QTableMetaData joinTable = selectedQueryJoins.get(tableOrAlias);
+                  if(!joinTable.getFields().containsKey(fieldNamePart))
+                  {
+                     //////////////////////////////////////////////////////////
+                     // unrecognized field within the join table is an error //
+                     //////////////////////////////////////////////////////////
+                     unrecognizedFieldNames.add(fieldName);
+                  }
+               }
+            }
+         }
+         else
+         {
+            ///////////////////////////////////////////////////////////////////////
+            // non-join fields - just ensure field name is in table's fields map //
+            ///////////////////////////////////////////////////////////////////////
+            if(!queryInput.getTable().getFields().containsKey(fieldName))
+            {
+               unrecognizedFieldNames.add(fieldName);
+            }
+         }
+      }
+
+      if(!unrecognizedFieldNames.isEmpty())
+      {
+         throw (new QException("QueryInput contained " + unrecognizedFieldNames.size() + " unrecognized field name" + StringUtils.plural(unrecognizedFieldNames) + ": " + StringUtils.join(",", unrecognizedFieldNames)));
+      }
+   }
+
+
+
    /*******************************************************************************
     ** shorthand way to call for the most common use-case, when you just want the
     ** records to be returned, and you just want to pass in a table name and filter.

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidator.java

@@ -44,6 +44,7 @@ import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizers;
 import com.kingsrook.qqq.backend.core.actions.dashboard.widgets.AbstractWidgetRenderer;
 import com.kingsrook.qqq.backend.core.actions.metadata.JoinGraph;
 import com.kingsrook.qqq.backend.core.actions.processes.BackendStep;
+import com.kingsrook.qqq.backend.core.actions.reporting.customizers.ReportCustomRecordSourceInterface;
 import com.kingsrook.qqq.backend.core.actions.scripts.TestScriptActionInterface;
 import com.kingsrook.qqq.backend.core.actions.values.QCustomPossibleValueProvider;
 import com.kingsrook.qqq.backend.core.exceptions.QInstanceValidationException;
@@ -1660,9 +1661,12 @@ public class QInstanceValidator
 
                   String dataSourceErrorPrefix = "Report " + reportName + " data source " + dataSource.getName() + " ";
 
+                  boolean hasASource = false;
+
                   if(StringUtils.hasContent(dataSource.getSourceTable()))
                   {
-                     assertCondition(dataSource.getStaticDataSupplier() == null, dataSourceErrorPrefix + "has both a sourceTable and a staticDataSupplier (exactly 1 is required).");
+                     hasASource = true;
+                     assertCondition(dataSource.getStaticDataSupplier() == null, dataSourceErrorPrefix + "has both a sourceTable and a staticDataSupplier (not compatible together).");
                      if(assertCondition(qInstance.getTable(dataSource.getSourceTable()) != null, dataSourceErrorPrefix + "source table " + dataSource.getSourceTable() + " is not a table in this instance."))
                      {
                         if(dataSource.getQueryFilter() != null)
@@ -1671,14 +1675,21 @@ public class QInstanceValidator
                         }
                      }
                   }
-                  else if(dataSource.getStaticDataSupplier() != null)
+
+                  if(dataSource.getStaticDataSupplier() != null)
                   {
+                     assertCondition(dataSource.getCustomRecordSource() == null, dataSourceErrorPrefix + "has both a staticDataSupplier and a customRecordSource (not compatible together).");
+                     hasASource = true;
                      validateSimpleCodeReference(dataSourceErrorPrefix, dataSource.getStaticDataSupplier(), Supplier.class);
                   }
-                  else
+
+                  if(dataSource.getCustomRecordSource() != null)
                   {
-                     errors.add(dataSourceErrorPrefix + "does not have a sourceTable or a staticDataSupplier (exactly 1 is required).");
+                     hasASource = true;
+                     validateSimpleCodeReference(dataSourceErrorPrefix, dataSource.getCustomRecordSource(), ReportCustomRecordSourceInterface.class);
                   }
+
+                  assertCondition(hasASource, dataSourceErrorPrefix + "does not have a sourceTable, customRecordSource, or a staticDataSupplier.");
                }
             }
 

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/CriteriaMissingInputValueBehavior.java

@@ -0,0 +1,66 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.model.actions.tables.query;
+
+
+/***************************************************************************
+ ** Possible behaviors for doing interpretValues on a filter, and a criteria
+ ** has a variable value (either as a string-that-looks-like-a-variable,
+ ** as in ${input.foreignId} for a PVS filter, or a FilterVariableExpression),
+ ** and a value for that variable isn't available.
+ **
+ ** Used in conjunction with FilterUseCase and its implementations, e.g.,
+ ** PossibleValueSearchFilterUseCase.
+ ***************************************************************************/
+public enum CriteriaMissingInputValueBehavior
+{
+   //////////////////////////////////////////////////////////////////////
+   // this was the original behavior, before we added this enum.  but, //
+   // it doesn't ever seem entirely valid, and isn't currently used.   //
+   //////////////////////////////////////////////////////////////////////
+   INTERPRET_AS_NULL_VALUE,
+
+   //////////////////////////////////////////////////////////////////////////
+   // make the criteria behave as though it's not in the filter at all.    //
+   // effectively by changing its operator to TRUE, so it always matches.  //
+   // original intended use is for possible-values on  query screens,      //
+   // where a foreign-id isn't present, so we want to show all PV options. //
+   //////////////////////////////////////////////////////////////////////////
+   REMOVE_FROM_FILTER,
+
+   //////////////////////////////////////////////////////////////////////////////////////
+   // make the criteria such that it makes no rows ever match.                         //
+   // e.g., changes it to a FALSE.  I suppose, within an OR, that might                //
+   // not be powerful enough...  but, it solves the immediate use-case in              //
+   // front of us, which is forms, where a PV field should show no values              //
+   // until a foreign key field has a value.                                           //
+   // Note that this use-case used to have the same end-effect by such                 //
+   // variables being interpreted as nulls - but this approach feels more intentional. //
+   //////////////////////////////////////////////////////////////////////////////////////
+   MAKE_NO_MATCHES,
+
+   ///////////////////////////////////////////////////////////////////////////////////////////
+   // throw an exception if a value isn't available.  This is the overall default,          //
+   // and originally was what we did for FilterVariableExpressions, e.g., for saved reports //
+   ///////////////////////////////////////////////////////////////////////////////////////////
+   THROW_EXCEPTION
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/FilterUseCase.java

@@ -0,0 +1,58 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.model.actions.tables.query;
+
+
+/*******************************************************************************
+ ** Interface where we can associate behaviors with various use cases for
+ ** QQueryFilters - the original being, how to handle (in the interpretValues
+ ** method) how to handle missing input values.
+ **
+ ** Includes a default implementation, with a default behavior - which is to
+ ** throw an exception upon missing criteria variable values.
+ *******************************************************************************/
+public interface FilterUseCase
+{
+   FilterUseCase DEFAULT = new DefaultFilterUseCase();
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   CriteriaMissingInputValueBehavior getDefaultCriteriaMissingInputValueBehavior();
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   class DefaultFilterUseCase implements FilterUseCase
+   {
+
+      /***************************************************************************
+       **
+       ***************************************************************************/
+      @Override
+      public CriteriaMissingInputValueBehavior getDefaultCriteriaMissingInputValueBehavior()
+      {
+         return CriteriaMissingInputValueBehavior.THROW_EXCEPTION;
+      }
+   }
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QQueryFilter.java

@@ -528,8 +528,27 @@ public class QQueryFilter implements Serializable, Cloneable
     ** Note - it may be very important that you call this method on a clone of a
     ** QQueryFilter - e.g., if it's one that defined in metaData, and that we don't
     ** want to be (permanently) changed!!
-    *******************************************************************************/
+    **
+    ** This overload does not take in a FilterUseCase - it uses FilterUseCase.DEFAULT
+    ******************************************************************************/
    public void interpretValues(Map<String, Serializable> inputValues) throws QException
+   {
+      interpretValues(inputValues, FilterUseCase.DEFAULT);
+   }
+
+
+
+   /*******************************************************************************
+    ** Replace any criteria values that look like ${input.XXX} with the value of XXX
+    ** from the supplied inputValues map - where the handling of missing values
+    ** is specified in the inputted FilterUseCase parameter
+    **
+    ** Note - it may be very important that you call this method on a clone of a
+    ** QQueryFilter - e.g., if it's one that defined in metaData, and that we don't
+    ** want to be (permanently) changed!!
+    **
+    *******************************************************************************/
+   public void interpretValues(Map<String, Serializable> inputValues, FilterUseCase useCase) throws QException
    {
       List<Exception> caughtExceptions = new ArrayList<>();
 
@@ -545,6 +564,9 @@ public class QQueryFilter implements Serializable, Cloneable
             {
                try
                {
+                  Serializable interpretedValue = value;
+                  Exception    caughtException  = null;
+
                   if(value instanceof AbstractFilterExpression<?>)
                   {
                      ///////////////////////////////////////////////////////////////////////
@@ -553,17 +575,54 @@ public class QQueryFilter implements Serializable, Cloneable
                      ///////////////////////////////////////////////////////////////////////
                      if(value instanceof FilterVariableExpression filterVariableExpression)
                      {
-                        newValues.add(filterVariableExpression.evaluateInputValues(inputValues));
-                     }
-                     else
-                     {
-                        newValues.add(value);
+                        try
+                        {
+                           interpretedValue = filterVariableExpression.evaluateInputValues(inputValues);
+                        }
+                        catch(Exception e)
+                        {
+                           caughtException = e;
+                           interpretedValue = InputNotFound.instance;
+                        }
+                     }
+                  }
+                  else
+                  {
+                     /////////////////////////////////////////////////////////////////////////////////////////////////////////
+                     // for non-expressions, cast the value to a string, and see if it can be resolved a variable.          //
+                     // there are 3 possible cases here:                                                                    //
+                     // 1: it doesn't look like a variable, so it just comes back as a string version of whatever went in.  //
+                     // 2: it was resolved from a variable to a value, e.g., ${input.someVar} => someValue                  //
+                     // 3: it looked like a variable, but no value for that variable was present in the interpreter's value //
+                     // map - so we'll get back the InputNotFound.instance.                                                 //
+                     /////////////////////////////////////////////////////////////////////////////////////////////////////////
+                     String valueAsString = ValueUtils.getValueAsString(value);
+                     interpretedValue = variableInterpreter.interpretForObject(valueAsString, InputNotFound.instance);
+                  }
+
+                  //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+                  // if interpreting a value returned the not-found value, or an empty string,                                            //
+                  // then decide how to handle the missing value, based on the use-case input                                             //
+                  // Note: questionable, using "" here, but that's what reality is passing a lot for cases we want to treat as missing... //
+                  //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+                  if(interpretedValue == InputNotFound.instance || "".equals(interpretedValue))
+                  {
+                     CriteriaMissingInputValueBehavior missingInputValueBehavior = getMissingInputValueBehavior(useCase);
+
+                     switch(missingInputValueBehavior)
+                     {
+                        case REMOVE_FROM_FILTER -> criterion.setOperator(QCriteriaOperator.TRUE);
+                        case MAKE_NO_MATCHES -> criterion.setOperator(QCriteriaOperator.FALSE);
+                        case INTERPRET_AS_NULL_VALUE -> newValues.add(null);
+
+                        /////////////////////////////////////////////////
+                        // handle case in the default: THROW_EXCEPTION //
+                        /////////////////////////////////////////////////
+                        default -> throw (Objects.requireNonNullElseGet(caughtException, () -> new QUserFacingException("Missing value for criteria on field: " + criterion.getFieldName())));
                      }
                   }
                   else
                   {
-                     String       valueAsString    = ValueUtils.getValueAsString(value);
-                     Serializable interpretedValue = variableInterpreter.interpretForObject(valueAsString);
                      newValues.add(interpretedValue);
                   }
                }
@@ -586,6 +645,44 @@ public class QQueryFilter implements Serializable, Cloneable
 
 
 
+   /***************************************************************************
+    ** Note:  in the original build of this, it felt like we *might* want to be
+    ** able to specify these behaviors at the individual criteria level, where
+    ** the implementation would be to add to QFilterCriteria:
+    ** - Map<FilterUseCase, CriteriaMissingInputValueBehavior> missingInputValueBehaviors;
+    ** - CriteriaMissingInputValueBehavior getMissingInputValueBehaviorForUseCase(FilterUseCase useCase) {}
+    *
+    ** (and maybe do that in a sub-class of QFilterCriteria, so it isn't always
+    ** there?  idk...) and then here we'd call:
+    ** - CriteriaMissingInputValueBehavior missingInputValueBehavior = criterion.getMissingInputValueBehaviorForUseCase(useCase);
+    *
+    ** But, we don't actually have that use-case at hand now, so - let's keep it
+    ** just at the level we need for now.
+    **
+    ***************************************************************************/
+   private CriteriaMissingInputValueBehavior getMissingInputValueBehavior(FilterUseCase useCase)
+   {
+      if(useCase == null)
+      {
+         useCase = FilterUseCase.DEFAULT;
+      }
+
+      CriteriaMissingInputValueBehavior missingInputValueBehavior = useCase.getDefaultCriteriaMissingInputValueBehavior();
+      if(missingInputValueBehavior == null)
+      {
+         missingInputValueBehavior = useCase.getDefaultCriteriaMissingInputValueBehavior();
+      }
+
+      if(missingInputValueBehavior == null)
+      {
+         missingInputValueBehavior = FilterUseCase.DEFAULT.getDefaultCriteriaMissingInputValueBehavior();
+      }
+
+      return (missingInputValueBehavior);
+   }
+
+
+
    /*******************************************************************************
     ** Getter for skip
     *******************************************************************************/
@@ -678,4 +775,28 @@ public class QQueryFilter implements Serializable, Cloneable
    {
       return Objects.hash(criteria, orderBys, booleanOperator, subFilters, skip, limit);
    }
+
+
+
+   /***************************************************************************
+    ** "Token" object to be used as the defaultIfLooksLikeVariableButNotFound
+    ** parameter to variableInterpreter.interpretForObject, so we can be
+    ** very clear that we got this default back (e.g., instead of a null,
+    ** which could maybe mean something else?)
+    ***************************************************************************/
+   private static final class InputNotFound implements Serializable
+   {
+      private static InputNotFound instance = new InputNotFound();
+
+
+
+      /*******************************************************************************
+       ** private singleton constructor
+       *******************************************************************************/
+      private InputNotFound()
+      {
+
+      }
+   }
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QueryInput.java

@@ -66,6 +66,14 @@ public class QueryInput extends AbstractTableActionInput implements QueryOrGetIn
    private List<QueryJoin> queryJoins     = null;
    private boolean         selectDistinct = false;
 
+   /////////////////////////////////////////////////////////////////////////////
+   // if this set is null, then the default (all fields) should be included   //
+   // if it's an empty set, that should throw an error                        //
+   // or if there are any fields in it that aren't valid fields on the table, //
+   // or in a selected queryJoin.                                             //
+   /////////////////////////////////////////////////////////////////////////////
+   private Set<String> fieldNamesToInclude;
+
    ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
    // if you say you want to includeAssociations, you can limit which ones by passing them in associationNamesToInclude. //
    // if you leave it null, you get all associations defined on the table.  if you pass it as empty, you get none.       //
@@ -686,4 +694,35 @@ public class QueryInput extends AbstractTableActionInput implements QueryOrGetIn
       return (queryHints.contains(queryHint));
    }
 
+
+
+   /*******************************************************************************
+    ** Getter for fieldNamesToInclude
+    *******************************************************************************/
+   public Set<String> getFieldNamesToInclude()
+   {
+      return (this.fieldNamesToInclude);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for fieldNamesToInclude
+    *******************************************************************************/
+   public void setFieldNamesToInclude(Set<String> fieldNamesToInclude)
+   {
+      this.fieldNamesToInclude = fieldNamesToInclude;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for fieldNamesToInclude
+    *******************************************************************************/
+   public QueryInput withFieldNamesToInclude(Set<String> fieldNamesToInclude)
+   {
+      this.fieldNamesToInclude = fieldNamesToInclude;
+      return (this);
+   }
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/expressions/AbstractFilterExpression.java

@@ -25,6 +25,7 @@ package com.kingsrook.qqq.backend.core.model.actions.tables.query.expressions;
 import java.io.Serializable;
 import java.util.Map;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
 
 
 /*******************************************************************************
@@ -35,7 +36,7 @@ public abstract class AbstractFilterExpression<T extends Serializable> implement
    /*******************************************************************************
     **
     *******************************************************************************/
-   public abstract T evaluate() throws QException;
+   public abstract T evaluate(QFieldMetaData field) throws QException;
 
 
 
@@ -47,7 +48,7 @@ public abstract class AbstractFilterExpression<T extends Serializable> implement
     *******************************************************************************/
    public T evaluateInputValues(Map<String, Serializable> inputValues) throws QException
    {
-      return evaluate();
+      return evaluate(null);
    }
 
 

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/expressions/FilterVariableExpression.java

@@ -26,6 +26,7 @@ import java.io.Serializable;
 import java.util.Map;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.exceptions.QUserFacingException;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
 import com.kingsrook.qqq.backend.core.utils.ValueUtils;
 
 
@@ -45,7 +46,7 @@ public class FilterVariableExpression extends AbstractFilterExpression<Serializa
     **
     *******************************************************************************/
    @Override
-   public Serializable evaluate() throws QException
+   public Serializable evaluate(QFieldMetaData field) throws QException
    {
       throw (new QUserFacingException("Missing variable value."));
    }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/expressions/Now.java

@@ -22,23 +22,42 @@
 package com.kingsrook.qqq.backend.core.model.actions.tables.query.expressions;
 
 
+import java.io.Serializable;
 import java.time.Instant;
+import java.time.ZoneId;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
+import com.kingsrook.qqq.backend.core.utils.ValueUtils;
 
 
 /*******************************************************************************
  **
  *******************************************************************************/
-public class Now extends AbstractFilterExpression<Instant>
+public class Now extends AbstractFilterExpression<Serializable>
 {
 
    /*******************************************************************************
     **
     *******************************************************************************/
    @Override
-   public Instant evaluate() throws QException
+   public Serializable evaluate(QFieldMetaData field) throws QException
    {
-      return (Instant.now());
+      QFieldType type = field == null ? QFieldType.DATE_TIME : field.getType();
+
+      if(type.equals(QFieldType.DATE_TIME))
+      {
+         return (Instant.now());
+      }
+      else if(type.equals(QFieldType.DATE))
+      {
+         ZoneId zoneId = ValueUtils.getSessionOrInstanceZoneId();
+         return (Instant.now().atZone(zoneId).toLocalDate());
+      }
+      else
+      {
+         throw (new QException("Unsupported field type [" + type + "]"));
+      }
    }
 
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/expressions/NowWithOffset.java

@@ -22,19 +22,24 @@
 package com.kingsrook.qqq.backend.core.model.actions.tables.query.expressions;
 
 
+import java.io.Serializable;
 import java.time.Instant;
+import java.time.LocalDate;
 import java.time.LocalDateTime;
 import java.time.ZoneId;
 import java.time.ZoneOffset;
 import java.time.temporal.ChronoUnit;
 import java.util.concurrent.TimeUnit;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
+import com.kingsrook.qqq.backend.core.utils.ValueUtils;
 
 
 /*******************************************************************************
  **
  *******************************************************************************/
-public class NowWithOffset extends AbstractFilterExpression<Instant>
+public class NowWithOffset extends AbstractFilterExpression<Serializable>
 {
    private Operator   operator;
    private int        amount;
@@ -123,7 +128,30 @@ public class NowWithOffset extends AbstractFilterExpression<Instant>
     **
     *******************************************************************************/
    @Override
-   public Instant evaluate() throws QException
+   public Serializable evaluate(QFieldMetaData field) throws QException
+   {
+      QFieldType type = field == null ? QFieldType.DATE_TIME : field.getType();
+
+      if(type.equals(QFieldType.DATE_TIME))
+      {
+         return (evaluateForDateTime());
+      }
+      else if(type.equals(QFieldType.DATE))
+      {
+         return (evaluateForDate());
+      }
+      else
+      {
+         throw (new QException("Unsupported field type [" + type + "]"));
+      }
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   private Instant evaluateForDateTime()
    {
       /////////////////////////////////////////////////////////////////////////////
       // Instant doesn't let us plus/minus WEEK, MONTH, or YEAR...               //
@@ -147,6 +175,26 @@ public class NowWithOffset extends AbstractFilterExpression<Instant>
 
 
 
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   private LocalDate evaluateForDate()
+   {
+      ZoneId    zoneId = ValueUtils.getSessionOrInstanceZoneId();
+      LocalDate now    = Instant.now().atZone(zoneId).toLocalDate();
+
+      if(operator.equals(Operator.PLUS))
+      {
+         return (now.plus(amount, timeUnit));
+      }
+      else
+      {
+         return (now.minus(amount, timeUnit));
+      }
+   }
+
+
+
    /*******************************************************************************
     ** Getter for operator
     **

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/expressions/ThisOrLastPeriod.java

@@ -22,27 +22,32 @@
 package com.kingsrook.qqq.backend.core.model.actions.tables.query.expressions;
 
 
+import java.io.Serializable;
 import java.time.DayOfWeek;
 import java.time.Instant;
+import java.time.LocalDate;
 import java.time.LocalDateTime;
 import java.time.ZoneId;
 import java.time.temporal.ChronoField;
 import java.time.temporal.ChronoUnit;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.exceptions.QRuntimeException;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
 import com.kingsrook.qqq.backend.core.utils.ValueUtils;
 
 
 /*******************************************************************************
  **
  *******************************************************************************/
-public class ThisOrLastPeriod extends AbstractFilterExpression<Instant>
+public class ThisOrLastPeriod extends AbstractFilterExpression<Serializable>
 {
    private Operator   operator;
    private ChronoUnit timeUnit;
 
 
 
+
    /***************************************************************************
     **
     ***************************************************************************/
@@ -88,7 +93,7 @@ public class ThisOrLastPeriod extends AbstractFilterExpression<Instant>
     ** Factory
     **
     *******************************************************************************/
-   public static ThisOrLastPeriod last(int amount, ChronoUnit timeUnit)
+   public static ThisOrLastPeriod last(ChronoUnit timeUnit)
    {
       return (new ThisOrLastPeriod(Operator.LAST, timeUnit));
    }
@@ -99,7 +104,31 @@ public class ThisOrLastPeriod extends AbstractFilterExpression<Instant>
     **
     *******************************************************************************/
    @Override
-   public Instant evaluate() throws QException
+   public Serializable evaluate(QFieldMetaData field) throws QException
+   {
+      QFieldType type = field == null ? QFieldType.DATE_TIME : field.getType();
+
+      if(type.equals(QFieldType.DATE_TIME))
+      {
+         return (evaluateForDateTime());
+      }
+      else if(type.equals(QFieldType.DATE))
+      {
+         // return (evaluateForDateTime());
+         return (evaluateForDate());
+      }
+      else
+      {
+         throw (new QException("Unsupported field type [" + type + "]"));
+      }
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   private Instant evaluateForDateTime()
    {
       ZoneId zoneId = ValueUtils.getSessionOrInstanceZoneId();
 
@@ -154,7 +183,57 @@ public class ThisOrLastPeriod extends AbstractFilterExpression<Instant>
 
             return operator.equals(Operator.THIS) ? startOfThisYear : startOfLastYear;
          }
-         default -> throw (new QRuntimeException("Unsupported timeUnit: " + timeUnit));
+         default -> throw (new QRuntimeException("Unsupported unit: " + timeUnit));
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public LocalDate evaluateForDate()
+   {
+      ZoneId    zoneId = ValueUtils.getSessionOrInstanceZoneId();
+      LocalDate today  = Instant.now().atZone(zoneId).toLocalDate();
+
+      switch(timeUnit)
+      {
+         case DAYS ->
+         {
+            return operator.equals(Operator.THIS) ? today : today.minusDays(1);
+         }
+         case WEEKS ->
+         {
+            LocalDate startOfThisWeek = today;
+            while(startOfThisWeek.get(ChronoField.DAY_OF_WEEK) != DayOfWeek.SUNDAY.getValue())
+            {
+               ////////////////////////////////////////
+               // go backwards until sunday is found //
+               ////////////////////////////////////////
+               startOfThisWeek = startOfThisWeek.minusDays(1);
+            }
+            return operator.equals(Operator.THIS) ? startOfThisWeek : startOfThisWeek.minusDays(7);
+         }
+         case MONTHS ->
+         {
+            Instant       startOfThisMonth    = ValueUtils.getStartOfMonthInZoneId(zoneId.getId());
+            LocalDateTime startOfThisMonthLDT = LocalDateTime.ofInstant(startOfThisMonth, ZoneId.of(zoneId.getId()));
+            LocalDateTime startOfLastMonthLDT = startOfThisMonthLDT.minusMonths(1);
+            Instant       startOfLastMonth    = startOfLastMonthLDT.toInstant(ZoneId.of(zoneId.getId()).getRules().getOffset(Instant.now()));
+
+            return (operator.equals(Operator.THIS) ? startOfThisMonth : startOfLastMonth).atZone(zoneId).toLocalDate();
+         }
+         case YEARS ->
+         {
+            Instant       startOfThisYear    = ValueUtils.getStartOfYearInZoneId(zoneId.getId());
+            LocalDateTime startOfThisYearLDT = LocalDateTime.ofInstant(startOfThisYear, zoneId);
+            LocalDateTime startOfLastYearLDT = startOfThisYearLDT.minusYears(1);
+            Instant       startOfLastYear    = startOfLastYearLDT.toInstant(zoneId.getRules().getOffset(Instant.now()));
+
+            return (operator.equals(Operator.THIS) ? startOfThisYear : startOfLastYear).atZone(zoneId).toLocalDate();
+         }
+         default -> throw (new QRuntimeException("Unsupported unit: " + timeUnit));
       }
    }
 

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/audits/AuditsMetaDataProvider.java

@@ -176,7 +176,7 @@ public class AuditsMetaDataProvider
          .withRecordLabelFields("label")
          .withPrimaryKeyField("id")
          .withUniqueKey(new UniqueKey("name"))
-         .withField(new QFieldMetaData("id", QFieldType.INTEGER))
+         .withField(new QFieldMetaData("id", QFieldType.LONG))
          .withField(new QFieldMetaData("name", QFieldType.STRING))
          .withField(new QFieldMetaData("label", QFieldType.STRING))
          .withField(new QFieldMetaData("createDate", QFieldType.DATE_TIME))
@@ -199,7 +199,7 @@ public class AuditsMetaDataProvider
          .withRecordLabelFields("name")
          .withPrimaryKeyField("id")
          .withUniqueKey(new UniqueKey("name"))
-         .withField(new QFieldMetaData("id", QFieldType.INTEGER))
+         .withField(new QFieldMetaData("id", QFieldType.LONG))
          .withField(new QFieldMetaData("name", QFieldType.STRING))
          .withField(new QFieldMetaData("createDate", QFieldType.DATE_TIME))
          .withField(new QFieldMetaData("modifyDate", QFieldType.DATE_TIME))
@@ -220,7 +220,7 @@ public class AuditsMetaDataProvider
          .withRecordLabelFormat("%s %s")
          .withRecordLabelFields("auditTableId", "recordId")
          .withPrimaryKeyField("id")
-         .withField(new QFieldMetaData("id", QFieldType.INTEGER))
+         .withField(new QFieldMetaData("id", QFieldType.LONG))
          .withField(new QFieldMetaData("auditTableId", QFieldType.INTEGER).withPossibleValueSourceName(TABLE_NAME_AUDIT_TABLE))
          .withField(new QFieldMetaData("auditUserId", QFieldType.INTEGER).withPossibleValueSourceName(TABLE_NAME_AUDIT_USER))
          .withField(new QFieldMetaData("recordId", QFieldType.INTEGER))
@@ -243,8 +243,8 @@ public class AuditsMetaDataProvider
          .withRecordLabelFormat("%s")
          .withRecordLabelFields("id")
          .withPrimaryKeyField("id")
-         .withField(new QFieldMetaData("id", QFieldType.INTEGER))
-         .withField(new QFieldMetaData("auditId", QFieldType.INTEGER).withPossibleValueSourceName(TABLE_NAME_AUDIT))
+         .withField(new QFieldMetaData("id", QFieldType.LONG))
+         .withField(new QFieldMetaData("auditId", QFieldType.LONG).withPossibleValueSourceName(TABLE_NAME_AUDIT))
          .withField(new QFieldMetaData("message", QFieldType.STRING).withMaxLength(250).withBehavior(ValueTooLongBehavior.TRUNCATE_ELLIPSIS))
          .withField(new QFieldMetaData("fieldName", QFieldType.STRING).withMaxLength(100).withBehavior(ValueTooLongBehavior.TRUNCATE_ELLIPSIS))
          .withField(new QFieldMetaData("oldValue", QFieldType.STRING).withMaxLength(250).withBehavior(ValueTooLongBehavior.TRUNCATE_ELLIPSIS))

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/dashboard/widgets/CompositeWidgetData.java

@@ -40,9 +40,10 @@ public class CompositeWidgetData extends AbstractBlockWidgetData<CompositeWidget
 {
    private List<AbstractBlockWidgetData<?, ?, ?, ?>> blocks = new ArrayList<>();
 
-   private Map<String, Serializable> styleOverrides = new HashMap<>();
-
-   private Layout layout;
+   private Layout                    layout;
+   private Map<String, Serializable> styleOverrides        = new HashMap<>();
+   private String                    overlayHtml;
+   private Map<String, Serializable> overlayStyleOverrides = new HashMap<>();
 
 
 
@@ -218,4 +219,91 @@ public class CompositeWidgetData extends AbstractBlockWidgetData<CompositeWidget
       return (this);
    }
 
+
+
+   /*******************************************************************************
+    ** Getter for overlayHtml
+    *******************************************************************************/
+   public String getOverlayHtml()
+   {
+      return (this.overlayHtml);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for overlayHtml
+    *******************************************************************************/
+   public void setOverlayHtml(String overlayHtml)
+   {
+      this.overlayHtml = overlayHtml;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for overlayHtml
+    *******************************************************************************/
+   public CompositeWidgetData withOverlayHtml(String overlayHtml)
+   {
+      this.overlayHtml = overlayHtml;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for overlayStyleOverrides
+    *******************************************************************************/
+   public Map<String, Serializable> getOverlayStyleOverrides()
+   {
+      return (this.overlayStyleOverrides);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for overlayStyleOverrides
+    *******************************************************************************/
+   public void setOverlayStyleOverrides(Map<String, Serializable> overlayStyleOverrides)
+   {
+      this.overlayStyleOverrides = overlayStyleOverrides;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for overlayStyleOverrides
+    *******************************************************************************/
+   public CompositeWidgetData withOverlayStyleOverrides(Map<String, Serializable> overlayStyleOverrides)
+   {
+      this.overlayStyleOverrides = overlayStyleOverrides;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public CompositeWidgetData withOverlayStyleOverride(String key, Serializable value)
+   {
+      addOverlayStyleOverride(key, value);
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public void addOverlayStyleOverride(String key, Serializable value)
+   {
+      if(this.overlayStyleOverrides == null)
+      {
+         this.overlayStyleOverrides = new HashMap<>();
+      }
+      this.overlayStyleOverrides.put(key, value);
+   }
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/dashboard/widgets/blocks/AbstractBlockWidgetData.java

@@ -24,6 +24,7 @@ package com.kingsrook.qqq.backend.core.model.dashboard.widgets.blocks;
 
 import java.util.HashMap;
 import java.util.Map;
+import com.kingsrook.qqq.backend.core.model.dashboard.widgets.CompositeWidgetData;
 import com.kingsrook.qqq.backend.core.model.dashboard.widgets.QWidgetData;
 
 
@@ -203,6 +204,19 @@ public abstract class AbstractBlockWidgetData<
 
 
 
+   /*******************************************************************************
+    ** Fluent setter for tooltip
+    **
+    *******************************************************************************/
+   @SuppressWarnings("unchecked")
+   public T withTooltip(CompositeWidgetData data)
+   {
+      this.tooltip = new BlockTooltip(data);
+      return (T) (this);
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/
@@ -398,6 +412,7 @@ public abstract class AbstractBlockWidgetData<
    }
 
 
+
    /*******************************************************************************
     ** Getter for blockId
     *******************************************************************************/
@@ -428,5 +443,4 @@ public abstract class AbstractBlockWidgetData<
       return (T) this;
    }
 
-
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/dashboard/widgets/blocks/BlockTooltip.java

@@ -22,14 +22,18 @@
 package com.kingsrook.qqq.backend.core.model.dashboard.widgets.blocks;
 
 
+import com.kingsrook.qqq.backend.core.model.dashboard.widgets.CompositeWidgetData;
+
+
 /*******************************************************************************
  ** A tooltip used within a (widget) block.
  **
  *******************************************************************************/
 public class BlockTooltip
 {
-   private String    title;
-   private Placement placement = Placement.BOTTOM;
+   private CompositeWidgetData blockData;
+   private String              title;
+   private Placement           placement = Placement.BOTTOM;
 
 
 
@@ -62,6 +66,17 @@ public class BlockTooltip
 
 
 
+   /*******************************************************************************
+    ** Constructor
+    **
+    *******************************************************************************/
+   public BlockTooltip(CompositeWidgetData blockData)
+   {
+      this.blockData = blockData;
+   }
+
+
+
    /*******************************************************************************
     ** Getter for title
     *******************************************************************************/
@@ -122,4 +137,35 @@ public class BlockTooltip
       return (this);
    }
 
+
+
+   /*******************************************************************************
+    ** Getter for blockData
+    *******************************************************************************/
+   public CompositeWidgetData getBlockData()
+   {
+      return (this.blockData);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for blockData
+    *******************************************************************************/
+   public void setBlockData(CompositeWidgetData blockData)
+   {
+      this.blockData = blockData;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for blockData
+    *******************************************************************************/
+   public BlockTooltip withBlockData(CompositeWidgetData blockData)
+   {
+      this.blockData = blockData;
+      return (this);
+   }
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/possiblevalues/PossibleValueSearchFilterUseCase.java

@@ -0,0 +1,72 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.model.metadata.possiblevalues;
+
+
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.CriteriaMissingInputValueBehavior;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.FilterUseCase;
+
+
+/*******************************************************************************
+ ** FilterUseCase implementation for the ways that possible value searches
+ ** are performed, and where we want to have different behaviors for criteria
+ ** that are missing an input value.  That is, either for a:
+ **
+ ** - FORM - e.g., creating a new record, or in a process - where we want a
+ ** missing filter value to basically block you from selecting a value in the
+ ** PVS field - e.g., you must enter some other foreign-key value before choosing
+ ** from this possible value - at least that's the use-case we know of now.
+ **
+ ** - FILTER - e.g., a query screen - where there isn't really quite the same
+ ** scenario of choosing that foreign-key value first - so, such a PVS should
+ ** list all its values (e.g., a criteria missing an input value should be
+ ** removed from the filter).
+ *******************************************************************************/
+public enum PossibleValueSearchFilterUseCase implements FilterUseCase
+{
+   FORM(CriteriaMissingInputValueBehavior.MAKE_NO_MATCHES),
+   FILTER(CriteriaMissingInputValueBehavior.REMOVE_FROM_FILTER);
+
+
+   private final CriteriaMissingInputValueBehavior defaultCriteriaMissingInputValueBehavior;
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   PossibleValueSearchFilterUseCase(CriteriaMissingInputValueBehavior defaultCriteriaMissingInputValueBehavior)
+   {
+      this.defaultCriteriaMissingInputValueBehavior = defaultCriteriaMissingInputValueBehavior;
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for defaultCriteriaMissingInputValueBehavior
+    **
+    *******************************************************************************/
+   public CriteriaMissingInputValueBehavior getDefaultCriteriaMissingInputValueBehavior()
+   {
+      return defaultCriteriaMissingInputValueBehavior;
+   }
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/reporting/QReportDataSource.java

@@ -32,6 +32,13 @@ import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
 /*******************************************************************************
  ** Meta-data definition of a source of data for a report (e.g., a table and query
  ** filter or custom-code reference).
+ **
+ ** Runs in 3 modes:
+ **
+ ** - If a customRecordSource is specified, then that code is executed to get the records.
+ ** - else, if a sourceTable is specified, then the corresponding queryFilter
+ ** (optionally along with queryJoins and queryInputCustomizer) is used.
+ ** - else a staticDataSupplier is used.
  *******************************************************************************/
 public class QReportDataSource
 {
@@ -44,6 +51,7 @@ public class QReportDataSource
 
    private QCodeReference queryInputCustomizer;
    private QCodeReference staticDataSupplier;
+   private QCodeReference customRecordSource;
 
 
 
@@ -265,4 +273,35 @@ public class QReportDataSource
       return (this);
    }
 
+
+   /*******************************************************************************
+    ** Getter for customRecordSource
+    *******************************************************************************/
+   public QCodeReference getCustomRecordSource()
+   {
+      return (this.customRecordSource);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for customRecordSource
+    *******************************************************************************/
+   public void setCustomRecordSource(QCodeReference customRecordSource)
+   {
+      this.customRecordSource = customRecordSource;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for customRecordSource
+    *******************************************************************************/
+   public QReportDataSource withCustomRecordSource(QCodeReference customRecordSource)
+   {
+      this.customRecordSource = customRecordSource;
+      return (this);
+   }
+
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/memory/MemoryRecordStore.java

@@ -58,6 +58,7 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.JoinsContext;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryJoin;
 import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
@@ -170,6 +171,8 @@ public class MemoryRecordStore
       Collection<QRecord> tableData = getTableData(input.getTable()).values();
       List<QRecord>       records   = new ArrayList<>();
 
+      QQueryFilter filter       = clonedOrNewFilter(input.getFilter());
+      JoinsContext joinsContext = new JoinsContext(QContext.getQInstance(), input.getTableName(), input.getQueryJoins(), filter);
       if(CollectionUtils.nullSafeHasContents(input.getQueryJoins()))
       {
          tableData = buildJoinCrossProduct(input);
@@ -185,7 +188,7 @@ public class MemoryRecordStore
             qRecord.setTableName(input.getTableName());
          }
 
-         boolean recordMatches = BackendQueryFilterUtils.doesRecordMatch(input.getFilter(), qRecord);
+         boolean recordMatches = BackendQueryFilterUtils.doesRecordMatch(input.getFilter(), joinsContext, qRecord);
 
          if(recordMatches)
          {
@@ -224,8 +227,7 @@ public class MemoryRecordStore
     *******************************************************************************/
    private Collection<QRecord> buildJoinCrossProduct(QueryInput input) throws QException
    {
-      QInstance    qInstance    = QContext.getQInstance();
-      JoinsContext joinsContext = new JoinsContext(qInstance, input.getTableName(), input.getQueryJoins(), input.getFilter());
+      QInstance qInstance = QContext.getQInstance();
 
       List<QRecord>  crossProduct = new ArrayList<>();
       QTableMetaData leftTable    = input.getTable();
@@ -373,7 +375,14 @@ public class MemoryRecordStore
          /////////////////////////////////////////////////
          if(recordToInsert.getValue(primaryKeyField.getName()) == null && (primaryKeyField.getType().equals(QFieldType.INTEGER) || primaryKeyField.getType().equals(QFieldType.LONG)))
          {
-            recordToInsert.setValue(primaryKeyField.getName(), nextSerial++);
+            if(primaryKeyField.getType().equals(QFieldType.LONG))
+            {
+               recordToInsert.setValue(primaryKeyField.getName(), (nextSerial++).longValue());
+            }
+            else
+            {
+               recordToInsert.setValue(primaryKeyField.getName(), nextSerial++);
+            }
          }
 
          ///////////////////////////////////////////////////////////////////////////////////////////////////
@@ -383,7 +392,7 @@ public class MemoryRecordStore
          {
             nextSerial = recordToInsert.getValueInteger(primaryKeyField.getName()) + 1;
          }
-         else if(primaryKeyField.getType().equals(QFieldType.LONG) && recordToInsert.getValueLong(primaryKeyField.getName()) > nextSerial)
+         else if(primaryKeyField.getType().equals(QFieldType.LONG) && recordToInsert.getValueInteger(primaryKeyField.getName()) > nextSerial)
          {
             //////////////////////////////////////
             // todo - mmm, could overflow here? //
@@ -901,4 +910,21 @@ public class MemoryRecordStore
 
       return ValueUtils.getValueAsFieldType(fieldType, aggregateValue);
    }
+
+
+
+   /*******************************************************************************
+    ** Either clone the input filter (so we can change it safely), or return a new blank filter.
+    *******************************************************************************/
+   protected QQueryFilter clonedOrNewFilter(QQueryFilter filter)
+   {
+      if(filter == null)
+      {
+         return (new QQueryFilter());
+      }
+      else
+      {
+         return (filter.clone());
+      }
+   }
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/utils/BackendQueryFilterUtils.java

@@ -34,14 +34,18 @@ import java.util.Objects;
 import java.util.concurrent.atomic.AtomicBoolean;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.logging.QLogger;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.JoinsContext;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.expressions.AbstractFilterExpression;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
 import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
 import com.kingsrook.qqq.backend.core.utils.ValueUtils;
 import org.apache.commons.lang.NotImplementedException;
+import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
 
 
 /*******************************************************************************
@@ -58,8 +62,22 @@ public class BackendQueryFilterUtils
 
    /*******************************************************************************
     ** Test if record matches filter.
-    *******************************************************************************/
+    ******************************************************************************/
    public static boolean doesRecordMatch(QQueryFilter filter, QRecord qRecord)
+   {
+      return doesRecordMatch(filter, null, qRecord);
+   }
+
+
+
+   /*******************************************************************************
+    ** Test if record matches filter - where we are executing a QueryAction, and
+    ** we have a JoinsContext.  Note, if you don't have one of those, you can call
+    ** the overload of this method that doesn't take one, and everything downstream
+    ** /should/ be tolerant of that being absent...  You just might not have the
+    ** benefit of things like knowing field-meta-data associated with criteria...
+    *******************************************************************************/
+   public static boolean doesRecordMatch(QQueryFilter filter, JoinsContext joinsContext, QRecord qRecord)
    {
       if(filter == null || !filter.hasAnyCriteria())
       {
@@ -97,7 +115,36 @@ public class BackendQueryFilterUtils
             }
          }
 
-         boolean criterionMatches = doesCriteriaMatch(criterion, fieldName, value);
+         ///////////////////////////////////////////////////////////////////////////////////////////////
+         // Test if this criteria(on) matches the record.                                             //
+         // As criteria have become more sophisticated over time, we would like to be able to know    //
+         // what field they are for. In general, we'll try to get that from the query's JoinsContext. //
+         // But, in some scenarios, that isn't available - so - be safe and defer to simpler methods  //
+         // that might not have the full field, when necessary.                                       //
+         ///////////////////////////////////////////////////////////////////////////////////////////////
+         Boolean criterionMatches = null;
+         if(joinsContext != null)
+         {
+            JoinsContext.FieldAndTableNameOrAlias fieldAndTableNameOrAlias = null;
+            try
+            {
+               fieldAndTableNameOrAlias = joinsContext.getFieldAndTableNameOrAlias(criterion.getFieldName());
+            }
+            catch(Exception e)
+            {
+               LOG.debug("Exception getting field from joinsContext", e, logPair("fieldName", criterion.getFieldName()));
+            }
+
+            if(fieldAndTableNameOrAlias != null)
+            {
+               criterionMatches = doesCriteriaMatch(criterion, fieldAndTableNameOrAlias.field(), value);
+            }
+         }
+
+         if(criterionMatches == null)
+         {
+            criterionMatches = doesCriteriaMatch(criterion, criterion.getFieldName(), value);
+         }
 
          ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
          // add this new value to the existing recordMatches value - and if we can short circuit the remaining checks, do so. //
@@ -131,11 +178,24 @@ public class BackendQueryFilterUtils
 
 
 
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   public static boolean doesCriteriaMatch(QFilterCriteria criterion, String fieldName, Serializable value)
+   {
+      QFieldMetaData field = new QFieldMetaData(fieldName, ValueUtils.inferQFieldTypeFromValue(value, QFieldType.STRING));
+      return doesCriteriaMatch(criterion, field, value);
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/
-   public static boolean doesCriteriaMatch(QFilterCriteria criterion, String fieldName, Serializable value)
+   private static boolean doesCriteriaMatch(QFilterCriteria criterion, QFieldMetaData field, Serializable value)
    {
+      String fieldName = field == null ? "__unknownField" : field.getName();
+
       ListIterator<Serializable> valueListIterator = criterion.getValues().listIterator();
       while(valueListIterator.hasNext())
       {
@@ -144,7 +204,7 @@ public class BackendQueryFilterUtils
          {
             try
             {
-               valueListIterator.set(expression.evaluate());
+               valueListIterator.set(expression.evaluate(field));
             }
             catch(QException qe)
             {

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/reports/BasicRunReportProcess.java

@@ -46,7 +46,8 @@ public class BasicRunReportProcess
    public static final String STEP_NAME_EXECUTE = "execute";
    public static final String STEP_NAME_ACCESS  = "accessReport";
 
-   public static final String FIELD_REPORT_NAME = "reportName";
+   public static final String FIELD_REPORT_NAME   = "reportName";
+   public static final String FIELD_REPORT_FORMAT = "reportFormat";
 
 
 

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/reports/ExecuteReportStep.java

@@ -33,6 +33,7 @@ import com.kingsrook.qqq.backend.core.actions.processes.BackendStep;
 import com.kingsrook.qqq.backend.core.actions.reporting.GenerateReportAction;
 import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.exceptions.QUserFacingException;
 import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
 import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
 import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportDestination;
@@ -50,6 +51,7 @@ import com.kingsrook.qqq.backend.core.utils.StringUtils;
 public class ExecuteReportStep implements BackendStep
 {
 
+
    /*******************************************************************************
     **
     *******************************************************************************/
@@ -58,9 +60,10 @@ public class ExecuteReportStep implements BackendStep
    {
       try
       {
-         String          reportName = runBackendStepInput.getValueString("reportName");
-         QReportMetaData report     = QContext.getQInstance().getReport(reportName);
-         File            tmpFile    = File.createTempFile(reportName, ".xlsx", new File("/tmp/"));
+         ReportFormat    reportFormat = getReportFormat(runBackendStepInput);
+         String          reportName   = runBackendStepInput.getValueString("reportName");
+         QReportMetaData report       = QContext.getQInstance().getReport(reportName);
+         File            tmpFile      = File.createTempFile(reportName, "." + reportFormat.getExtension());
 
          runBackendStepInput.getAsyncJobCallback().updateStatus("Generating Report");
 
@@ -69,7 +72,7 @@ public class ExecuteReportStep implements BackendStep
             ReportInput reportInput = new ReportInput();
             reportInput.setReportName(reportName);
             reportInput.setReportDestination(new ReportDestination()
-               .withReportFormat(ReportFormat.XLSX) // todo - variable
+               .withReportFormat(reportFormat)
                .withReportOutputStream(reportOutputStream));
 
             Map<String, Serializable> values = runBackendStepInput.getValues();
@@ -79,7 +82,7 @@ public class ExecuteReportStep implements BackendStep
 
             String downloadFileBaseName = getDownloadFileBaseName(runBackendStepInput, report);
 
-            runBackendStepOutput.addValue("downloadFileName", downloadFileBaseName + ".xlsx");
+            runBackendStepOutput.addValue("downloadFileName", downloadFileBaseName + "." + reportFormat.getExtension());
             runBackendStepOutput.addValue("serverFilePath", tmpFile.getCanonicalPath());
          }
       }
@@ -91,6 +94,22 @@ public class ExecuteReportStep implements BackendStep
 
 
 
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   private ReportFormat getReportFormat(RunBackendStepInput runBackendStepInput) throws QUserFacingException
+   {
+      String reportFormatInput = runBackendStepInput.getValueString(BasicRunReportProcess.FIELD_REPORT_FORMAT);
+      if(StringUtils.hasContent(reportFormatInput))
+      {
+         return (ReportFormat.fromString(reportFormatInput));
+      }
+
+      return (ReportFormat.XLSX);
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/ValueUtils.java

@@ -41,6 +41,7 @@ import java.util.List;
 import java.util.TimeZone;
 import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.exceptions.QValueException;
+import com.kingsrook.qqq.backend.core.logging.QLogger;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
 import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.PossibleValueEnum;
 import com.kingsrook.qqq.backend.core.model.session.QSession;
@@ -51,6 +52,8 @@ import com.kingsrook.qqq.backend.core.model.session.QSession;
  *******************************************************************************/
 public class ValueUtils
 {
+   private static final QLogger LOG = QLogger.getLogger(ValueUtils.class);
+
    private static final DateTimeFormatter dateTimeFormatter_yyyyMMddWithDashes = DateTimeFormatter.ofPattern("yyyy-MM-dd");
    private static final DateTimeFormatter dateTimeFormatter_MdyyyyWithSlashes  = DateTimeFormatter.ofPattern("M/d/yyyy");
    private static final DateTimeFormatter dateTimeFormatter_yyyyMMdd           = DateTimeFormatter.ofPattern("yyyyMMdd");
@@ -931,4 +934,48 @@ public class ValueUtils
       return (ZoneId.of(QContext.getQInstance().getDefaultTimeZoneId()));
    }
 
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   public static QFieldType inferQFieldTypeFromValue(Serializable value, QFieldType defaultIfCannotInfer)
+   {
+      if(value instanceof String)
+      {
+         return QFieldType.STRING;
+      }
+      else if(value instanceof Integer)
+      {
+         return QFieldType.INTEGER;
+      }
+      else if(value instanceof Long)
+      {
+         return QFieldType.LONG;
+      }
+      else if(value instanceof BigDecimal)
+      {
+         return QFieldType.DECIMAL;
+      }
+      else if(value instanceof Boolean)
+      {
+         return QFieldType.BOOLEAN;
+      }
+      else if(value instanceof Instant)
+      {
+         return QFieldType.DATE_TIME;
+      }
+      else if(value instanceof LocalDate)
+      {
+         return QFieldType.DATE;
+      }
+      else if(value instanceof LocalTime)
+      {
+         return QFieldType.TIME;
+      }
+
+      LOG.debug("Could not infer QFieldType from value [" + (value == null ? "null" : value.getClass().getSimpleName()) + "]");
+
+      return defaultIfCannotInfer;
+   }
 }

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/audits/AuditActionTest.java

@@ -180,7 +180,7 @@ class AuditActionTest extends BaseTest
       QRecord auditRecord = GeneralProcessUtils.getRecordByFieldOrElseThrow("audit", "recordId", recordId1);
       assertEquals("Test Audit", auditRecord.getValueString("message"));
 
-      List<QRecord> auditDetails = GeneralProcessUtils.getRecordListByField("auditDetail", "auditId", auditRecord.getValue("id"));
+      List<QRecord> auditDetails = GeneralProcessUtils.getRecordListByField("auditDetail", "auditId", auditRecord.getValueLong("id"));
       assertEquals(2, auditDetails.size());
       assertThat(auditDetails).anyMatch(r -> r.getValueString("message").equals("Detail1"));
       assertThat(auditDetails).anyMatch(r -> r.getValueString("message").equals("Detail2"));
@@ -188,13 +188,13 @@ class AuditActionTest extends BaseTest
       auditRecord = GeneralProcessUtils.getRecordByFieldOrElseThrow("audit", "recordId", recordId2);
       assertEquals("Test Another Audit", auditRecord.getValueString("message"));
       assertEquals(47, auditRecord.getValueInteger(TestUtils.SECURITY_KEY_TYPE_STORE));
-      auditDetails = GeneralProcessUtils.getRecordListByField("auditDetail", "auditId", auditRecord.getValue("id"));
+      auditDetails = GeneralProcessUtils.getRecordListByField("auditDetail", "auditId", auditRecord.getValueLong("id"));
       assertEquals(0, auditDetails.size());
 
       auditRecord = GeneralProcessUtils.getRecordByFieldOrElseThrow("audit", "recordId", recordId3);
       assertEquals("Audit 3", auditRecord.getValueString("message"));
       assertEquals(42, auditRecord.getValueInteger(TestUtils.SECURITY_KEY_TYPE_STORE));
-      auditDetails = GeneralProcessUtils.getRecordListByField("auditDetail", "auditId", auditRecord.getValue("id"));
+      auditDetails = GeneralProcessUtils.getRecordListByField("auditDetail", "auditId", auditRecord.getValueLong("id"));
       assertEquals(1, auditDetails.size());
       assertThat(auditDetails).anyMatch(r -> r.getValueString("message").equals("Detail3"));
    }

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/tables/QQueryFilterTest.java

@@ -29,6 +29,8 @@ import java.util.Map;
 import com.kingsrook.qqq.backend.core.BaseTest;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.exceptions.QUserFacingException;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.CriteriaMissingInputValueBehavior;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.FilterUseCase;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.expressions.AbstractFilterExpression;
@@ -36,9 +38,12 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.query.expressions.Fil
 import org.junit.jupiter.api.Test;
 import static com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator.BETWEEN;
 import static com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator.EQUALS;
+import static com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator.FALSE;
 import static com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator.IS_BLANK;
+import static com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator.TRUE;
 import static org.assertj.core.api.Assertions.assertThatThrownBy;
 import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNull;
 
 
 /*******************************************************************************
@@ -140,4 +145,230 @@ class QQueryFilterTest extends BaseTest
       assertEquals("joinTableSomeFieldIdEquals", fve7.getVariableName());
    }
 
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testInterpretValueVariableExpressionNotFoundUseCases() throws QException
+   {
+      Map<String, Serializable> inputValues = new HashMap<>();
+
+      AbstractFilterExpression<Serializable> expression = new FilterVariableExpression()
+         .withVariableName("clientId");
+
+      ////////////////////////////////////////
+      // Control - where the value IS found //
+      ////////////////////////////////////////
+      inputValues.put("clientId", 47);
+      {
+         QQueryFilter filter = new QQueryFilter(new QFilterCriteria("id", EQUALS, expression));
+         filter.interpretValues(inputValues);
+         assertEquals(47, filter.getCriteria().get(0).getValues().get(0));
+         assertEquals(EQUALS, filter.getCriteria().get(0).getOperator());
+      }
+
+      //////////////////////////////////////////////////////
+      // now - remove the value for the next set of cases //
+      //////////////////////////////////////////////////////
+      inputValues.remove("clientId");
+
+      ////////////////////////////////////////////////////////////////////////////////////////////////
+      // a use-case that says to remove-from-filter, which, means translate to a criteria of "TRUE" //
+      ////////////////////////////////////////////////////////////////////////////////////////////////
+      {
+         QQueryFilter filter = new QQueryFilter(new QFilterCriteria("id", EQUALS, expression));
+         filter.interpretValues(inputValues, new RemoveFromFilterUseCase());
+         assertEquals(0, filter.getCriteria().get(0).getValues().size());
+         assertEquals(TRUE, filter.getCriteria().get(0).getOperator());
+      }
+
+      //////////////////////////////////////////////////////////////////////////////////////////////
+      // a use-case that says to make-no-matches, which, means translate to a criteria of "FALSE" //
+      //////////////////////////////////////////////////////////////////////////////////////////////
+      {
+         QQueryFilter filter = new QQueryFilter(new QFilterCriteria("id", EQUALS, expression));
+         filter.interpretValues(inputValues, new MakeNoMatchesUseCase());
+         assertEquals(0, filter.getCriteria().get(0).getValues().size());
+         assertEquals(FALSE, filter.getCriteria().get(0).getOperator());
+      }
+
+      ///////////////////////////////////////////
+      // a use-case that says to treat as null //
+      ///////////////////////////////////////////
+      {
+         QQueryFilter filter = new QQueryFilter(new QFilterCriteria("id", EQUALS, expression));
+         filter.interpretValues(inputValues, new InterpretAsNullValueUseCase());
+         assertNull(filter.getCriteria().get(0).getValues().get(0));
+         assertEquals(EQUALS, filter.getCriteria().get(0).getOperator());
+      }
+
+      ///////////////////////////////////
+      // a use-case that says to throw //
+      ///////////////////////////////////
+      {
+         QQueryFilter filter = new QQueryFilter(new QFilterCriteria("id", EQUALS, expression));
+         assertThatThrownBy(() -> filter.interpretValues(inputValues, new ThrowExceptionUseCase()))
+            .isInstanceOf(QUserFacingException.class)
+            .hasMessageContaining("Missing value for variable: clientId");
+      }
+
+      //////////////////////////////////////////////////////////
+      // verify that empty-string is treated as not-found too //
+      //////////////////////////////////////////////////////////
+      inputValues.put("clientId", "");
+      {
+         QQueryFilter filter = new QQueryFilter(new QFilterCriteria("id", EQUALS, expression));
+         assertThatThrownBy(() -> filter.interpretValues(inputValues, new ThrowExceptionUseCase()))
+            .isInstanceOf(QUserFacingException.class)
+            .hasMessageContaining("Missing value for variable: clientId");
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testInterpretValueStringStyleNotFoundUseCases() throws QException
+   {
+      Map<String, Serializable> inputValues = new HashMap<>();
+
+      ////////////////////////////////////////
+      // Control - where the value IS found //
+      ////////////////////////////////////////
+      inputValues.put("clientId", 47);
+      {
+         QQueryFilter filter = new QQueryFilter(new QFilterCriteria("id", EQUALS, "${input.clientId}"));
+         filter.interpretValues(inputValues);
+         assertEquals(47, filter.getCriteria().get(0).getValues().get(0));
+         assertEquals(EQUALS, filter.getCriteria().get(0).getOperator());
+      }
+
+      //////////////////////////////////////////////////////
+      // now - remove the value for the next set of cases //
+      //////////////////////////////////////////////////////
+      inputValues.remove("clientId");
+
+      ////////////////////////////////////////////////////////////////////////////////////////////////
+      // a use-case that says to remove-from-filter, which, means translate to a criteria of "TRUE" //
+      ////////////////////////////////////////////////////////////////////////////////////////////////
+      {
+         QQueryFilter filter = new QQueryFilter(new QFilterCriteria("id", EQUALS, "${input.clientId}"));
+         filter.interpretValues(inputValues, new RemoveFromFilterUseCase());
+         assertEquals(0, filter.getCriteria().get(0).getValues().size());
+         assertEquals(TRUE, filter.getCriteria().get(0).getOperator());
+      }
+
+      //////////////////////////////////////////////////////////////////////////////////////////////
+      // a use-case that says to make-no-matches, which, means translate to a criteria of "FALSE" //
+      //////////////////////////////////////////////////////////////////////////////////////////////
+      {
+         QQueryFilter filter = new QQueryFilter(new QFilterCriteria("id", EQUALS, "${input.clientId}"));
+         filter.interpretValues(inputValues, new MakeNoMatchesUseCase());
+         assertEquals(0, filter.getCriteria().get(0).getValues().size());
+         assertEquals(FALSE, filter.getCriteria().get(0).getOperator());
+      }
+
+      ///////////////////////////////////////////
+      // a use-case that says to treat as null //
+      ///////////////////////////////////////////
+      {
+         QQueryFilter filter = new QQueryFilter(new QFilterCriteria("id", EQUALS, "${input.clientId}"));
+         filter.interpretValues(inputValues, new InterpretAsNullValueUseCase());
+         assertNull(filter.getCriteria().get(0).getValues().get(0));
+         assertEquals(EQUALS, filter.getCriteria().get(0).getOperator());
+      }
+
+      ///////////////////////////////////
+      // a use-case that says to throw //
+      ///////////////////////////////////
+      {
+         QQueryFilter filter = new QQueryFilter(new QFilterCriteria("id", EQUALS, "${input.clientId}"));
+         assertThatThrownBy(() -> filter.interpretValues(inputValues, new ThrowExceptionUseCase()))
+            .isInstanceOf(QUserFacingException.class)
+            .hasMessageContaining("Missing value for criteria on field: id");
+      }
+
+      //////////////////////////////////////////////////////////
+      // verify that empty-string is treated as not-found too //
+      //////////////////////////////////////////////////////////
+      inputValues.put("clientId", "");
+      {
+         QQueryFilter filter = new QQueryFilter(new QFilterCriteria("id", EQUALS, "${input.clientId}"));
+         assertThatThrownBy(() -> filter.interpretValues(inputValues, new ThrowExceptionUseCase()))
+            .isInstanceOf(QUserFacingException.class)
+            .hasMessageContaining("Missing value for criteria on field: id");
+      }
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   private static class RemoveFromFilterUseCase implements FilterUseCase
+   {
+      /***************************************************************************
+       **
+       ***************************************************************************/
+      @Override
+      public CriteriaMissingInputValueBehavior getDefaultCriteriaMissingInputValueBehavior()
+      {
+         return CriteriaMissingInputValueBehavior.REMOVE_FROM_FILTER;
+      }
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   private static class MakeNoMatchesUseCase implements FilterUseCase
+   {
+      /***************************************************************************
+       **
+       ***************************************************************************/
+      @Override
+      public CriteriaMissingInputValueBehavior getDefaultCriteriaMissingInputValueBehavior()
+      {
+         return CriteriaMissingInputValueBehavior.MAKE_NO_MATCHES;
+      }
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   private static class InterpretAsNullValueUseCase implements FilterUseCase
+   {
+      /***************************************************************************
+       **
+       ***************************************************************************/
+      @Override
+      public CriteriaMissingInputValueBehavior getDefaultCriteriaMissingInputValueBehavior()
+      {
+         return CriteriaMissingInputValueBehavior.INTERPRET_AS_NULL_VALUE;
+      }
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   private static class ThrowExceptionUseCase implements FilterUseCase
+   {
+      /***************************************************************************
+       **
+       ***************************************************************************/
+      @Override
+      public CriteriaMissingInputValueBehavior getDefaultCriteriaMissingInputValueBehavior()
+      {
+         return CriteriaMissingInputValueBehavior.THROW_EXCEPTION;
+      }
+   }
 }

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/tables/QueryActionTest.java

@@ -24,6 +24,8 @@ package com.kingsrook.qqq.backend.core.actions.tables;
 
 import java.io.Serializable;
 import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.LinkedHashSet;
 import java.util.List;
 import java.util.Set;
 import com.kingsrook.qqq.backend.core.BaseTest;
@@ -38,6 +40,7 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryJoin;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
 import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
@@ -54,6 +57,7 @@ import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
 import com.kingsrook.qqq.backend.core.utils.TestUtils;
 import org.junit.jupiter.api.Test;
 import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertFalse;
 import static org.junit.jupiter.api.Assertions.assertNotNull;
@@ -541,4 +545,89 @@ class QueryActionTest extends BaseTest
       assertEquals(1, QueryAction.execute(TestUtils.TABLE_NAME_SHAPE, new QQueryFilter(new QFilterCriteria("name", QCriteriaOperator.EQUALS, "SqUaRe"))).size());
    }
 
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testValidateFieldNamesToInclude() throws QException
+   {
+      /////////////////////////////
+      // cases that do not throw //
+      /////////////////////////////
+      QueryAction.validateFieldNamesToInclude(new QueryInput(TestUtils.TABLE_NAME_PERSON)
+         .withFieldNamesToInclude(null));
+
+      QueryAction.validateFieldNamesToInclude(new QueryInput(TestUtils.TABLE_NAME_PERSON)
+         .withFieldNamesToInclude(Set.of("id")));
+
+      QueryAction.validateFieldNamesToInclude(new QueryInput(TestUtils.TABLE_NAME_PERSON)
+         .withFieldNamesToInclude(Set.of("id", "firstName", "lastName", "birthDate", "modifyDate", "createDate")));
+
+      QueryAction.validateFieldNamesToInclude(new QueryInput(TestUtils.TABLE_NAME_PERSON)
+         .withQueryJoin(new QueryJoin(TestUtils.TABLE_NAME_SHAPE).withSelect(true))
+         .withFieldNamesToInclude(Set.of("id", "firstName", "lastName", "birthDate", "modifyDate", "createDate")));
+
+      QueryAction.validateFieldNamesToInclude(new QueryInput(TestUtils.TABLE_NAME_PERSON)
+         .withQueryJoin(new QueryJoin(TestUtils.TABLE_NAME_SHAPE).withSelect(true))
+         .withFieldNamesToInclude(Set.of("id", "firstName", TestUtils.TABLE_NAME_SHAPE + ".id", TestUtils.TABLE_NAME_SHAPE + ".noOfSides")));
+
+      QueryAction.validateFieldNamesToInclude(new QueryInput(TestUtils.TABLE_NAME_PERSON)
+         .withQueryJoin(new QueryJoin(TestUtils.TABLE_NAME_SHAPE).withSelect(true).withAlias("s"))
+         .withFieldNamesToInclude(Set.of("id", "s.id", "s.noOfSides")));
+
+      //////////////////////////
+      // cases that do throw! //
+      //////////////////////////
+      assertThatThrownBy(() -> QueryAction.validateFieldNamesToInclude(new QueryInput(TestUtils.TABLE_NAME_PERSON)
+         .withFieldNamesToInclude(new HashSet<>())))
+         .hasMessageContaining("empty set of fieldNamesToInclude");
+
+      assertThatThrownBy(() -> QueryAction.validateFieldNamesToInclude(new QueryInput(TestUtils.TABLE_NAME_PERSON)
+         .withFieldNamesToInclude(Set.of("notAField"))))
+         .hasMessageContaining("1 unrecognized field name: notAField");
+
+      assertThatThrownBy(() -> QueryAction.validateFieldNamesToInclude(new QueryInput(TestUtils.TABLE_NAME_PERSON)
+         .withFieldNamesToInclude(new LinkedHashSet<>(List.of("notAField", "alsoNot")))))
+         .hasMessageContaining("2 unrecognized field names: notAField,alsoNot");
+
+      assertThatThrownBy(() -> QueryAction.validateFieldNamesToInclude(new QueryInput(TestUtils.TABLE_NAME_PERSON)
+         .withFieldNamesToInclude(new LinkedHashSet<>(List.of("notAField", "alsoNot", "join.not")))))
+         .hasMessageContaining("3 unrecognized field names: notAField,alsoNot,join.not");
+
+      assertThatThrownBy(() -> QueryAction.validateFieldNamesToInclude(new QueryInput(TestUtils.TABLE_NAME_PERSON)
+         .withQueryJoin(new QueryJoin(TestUtils.TABLE_NAME_SHAPE)) // oops, didn't select it!
+         .withFieldNamesToInclude(new LinkedHashSet<>(List.of("id", "firstName", TestUtils.TABLE_NAME_SHAPE + ".id", TestUtils.TABLE_NAME_SHAPE + ".noOfSides")))))
+         .hasMessageContaining("2 unrecognized field names: shape.id,shape.noOfSides");
+
+      assertThatThrownBy(() -> QueryAction.validateFieldNamesToInclude(new QueryInput(TestUtils.TABLE_NAME_PERSON)
+         .withQueryJoin(new QueryJoin(TestUtils.TABLE_NAME_SHAPE).withSelect(true))
+         .withFieldNamesToInclude(Set.of(TestUtils.TABLE_NAME_SHAPE + ".noField"))))
+         .hasMessageContaining("1 unrecognized field name: shape.noField");
+
+      assertThatThrownBy(() -> QueryAction.validateFieldNamesToInclude(new QueryInput(TestUtils.TABLE_NAME_PERSON)
+         .withQueryJoin(new QueryJoin(TestUtils.TABLE_NAME_SHAPE).withSelect(true).withAlias("s")) // oops, not using alias
+         .withFieldNamesToInclude(new LinkedHashSet<>(List.of("id", "firstName", TestUtils.TABLE_NAME_SHAPE + ".id", "s.noOfSides")))))
+         .hasMessageContaining("1 unrecognized field name: shape.id");
+
+      assertThatThrownBy(() -> QueryAction.validateFieldNamesToInclude(new QueryInput(TestUtils.TABLE_NAME_PERSON)
+         .withQueryJoin(new QueryJoin(TestUtils.TABLE_NAME_SHAPE).withSelect(true))
+         .withFieldNamesToInclude(new LinkedHashSet<>(List.of("id", "firstName", TestUtils.TABLE_NAME_SHAPE + ".id", "noOfSides")))))
+         .hasMessageContaining("1 unrecognized field name: noOfSides");
+
+      assertThatThrownBy(() -> QueryAction.validateFieldNamesToInclude(new QueryInput(TestUtils.TABLE_NAME_PERSON)
+         .withQueryJoin(new QueryJoin("noJoinTable").withSelect(true))
+         .withFieldNamesToInclude(Set.of("noJoinTable.id"))))
+         .hasMessageContaining("1 unrecognized field name: noJoinTable.id");
+
+      assertThatThrownBy(() -> QueryAction.validateFieldNamesToInclude(new QueryInput(TestUtils.TABLE_NAME_PERSON)
+         .withFieldNamesToInclude(Set.of("noJoin.id"))))
+         .hasMessageContaining("1 unrecognized field name: noJoin.id");
+
+      assertThatThrownBy(() -> QueryAction.validateFieldNamesToInclude(new QueryInput(TestUtils.TABLE_NAME_PERSON)
+         .withFieldNamesToInclude(Set.of("noDb.noJoin.id"))))
+         .hasMessageContaining("1 unrecognized field name: noDb.noJoin.id");
+   }
+
 }

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidatorTest.java

@@ -22,6 +22,7 @@
 package com.kingsrook.qqq.backend.core.instances;
 
 
+import java.io.Serializable;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashMap;
@@ -38,6 +39,8 @@ import com.kingsrook.qqq.backend.core.actions.dashboard.PersonsByCreateDateBarCh
 import com.kingsrook.qqq.backend.core.actions.dashboard.widgets.AbstractWidgetRenderer;
 import com.kingsrook.qqq.backend.core.actions.dashboard.widgets.ParentWidgetRenderer;
 import com.kingsrook.qqq.backend.core.actions.processes.CancelProcessActionTest;
+import com.kingsrook.qqq.backend.core.actions.reporting.RecordPipe;
+import com.kingsrook.qqq.backend.core.actions.reporting.customizers.ReportCustomRecordSourceInterface;
 import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.exceptions.QInstanceValidationException;
@@ -45,6 +48,7 @@ import com.kingsrook.qqq.backend.core.instances.validation.plugins.AlwaysFailsPr
 import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLineInterface;
 import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
 import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
+import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy;
@@ -1712,7 +1716,7 @@ public class QInstanceValidatorTest extends BaseTest
    @Test
    void testReportDataSourceStaticDataSupplier()
    {
-      assertValidationFailureReasons((qInstance) -> qInstance.getReport(TestUtils.REPORT_NAME_SHAPES_PERSON).getDataSources().get(0).withStaticDataSupplier(new QCodeReference()),
+      assertValidationFailureReasons((qInstance) -> qInstance.getReport(TestUtils.REPORT_NAME_SHAPES_PERSON).getDataSources().get(0).withStaticDataSupplier(new QCodeReference(TestReportStaticDataSupplier.class)),
          "has both a sourceTable and a staticDataSupplier");
 
       assertValidationFailureReasons((qInstance) ->
@@ -1720,16 +1724,43 @@ public class QInstanceValidatorTest extends BaseTest
          QReportDataSource dataSource = qInstance.getReport(TestUtils.REPORT_NAME_SHAPES_PERSON).getDataSources().get(0);
          dataSource.setSourceTable(null);
          dataSource.setStaticDataSupplier(new QCodeReference(null, QCodeType.JAVA));
-      },
-         "missing a code reference name");
+      }, "missing a code reference name");
 
       assertValidationFailureReasons((qInstance) ->
       {
          QReportDataSource dataSource = qInstance.getReport(TestUtils.REPORT_NAME_SHAPES_PERSON).getDataSources().get(0);
          dataSource.setSourceTable(null);
          dataSource.setStaticDataSupplier(new QCodeReference(ArrayList.class));
-      },
-         "is not of the expected type");
+      }, "is not of the expected type");
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testReportDataSourceCustomRecordSource()
+   {
+      assertValidationFailureReasons((qInstance) -> qInstance.getReport(TestUtils.REPORT_NAME_SHAPES_PERSON).getDataSources().get(0)
+            .withSourceTable(null)
+            .withStaticDataSupplier(new QCodeReference(TestReportStaticDataSupplier.class))
+            .withCustomRecordSource(new QCodeReference(TestReportCustomRecordSource.class)),
+         "has both a staticDataSupplier and a customRecordSource");
+
+      assertValidationFailureReasons((qInstance) ->
+      {
+         QReportDataSource dataSource = qInstance.getReport(TestUtils.REPORT_NAME_SHAPES_PERSON).getDataSources().get(0);
+         dataSource.setSourceTable(null);
+         dataSource.setCustomRecordSource(new QCodeReference(null, QCodeType.JAVA));
+      }, "missing a code reference name");
+
+      assertValidationFailureReasons((qInstance) ->
+      {
+         QReportDataSource dataSource = qInstance.getReport(TestUtils.REPORT_NAME_SHAPES_PERSON).getDataSources().get(0);
+         dataSource.setSourceTable(null);
+         dataSource.setCustomRecordSource(new QCodeReference(ArrayList.class));
+      }, "is not of the expected type");
    }
 
 
@@ -2371,5 +2402,38 @@ public class QInstanceValidatorTest extends BaseTest
     *******************************************************************************/
    public static class ValidAuthCustomizer implements QAuthenticationModuleCustomizerInterface {}
 
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   public static class TestReportStaticDataSupplier implements Supplier<List<List<Serializable>>>
+   {
+
+      /***************************************************************************
+       **
+       ***************************************************************************/
+      @Override
+      public List<List<Serializable>> get()
+      {
+         return List.of();
+      }
+   }
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   public static class TestReportCustomRecordSource implements ReportCustomRecordSourceInterface
+   {
+
+      /***************************************************************************
+       **
+       ***************************************************************************/
+      @Override
+      public void execute(ReportInput reportInput, QReportDataSource reportDataSource, RecordPipe recordPipe) throws QException
+      {
+
+      }
+   }
 }
 

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/expressions/NowWithOffsetTest.java

@@ -22,11 +22,16 @@
 package com.kingsrook.qqq.backend.core.model.actions.tables.query.expressions;
 
 
+import java.time.Instant;
+import java.time.LocalDate;
 import java.time.temporal.ChronoUnit;
 import com.kingsrook.qqq.backend.core.BaseTest;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
 import org.assertj.core.data.Offset;
 import org.junit.jupiter.api.Test;
 import static org.assertj.core.api.Assertions.assertThat;
+import static org.junit.jupiter.api.Assertions.assertEquals;
 
 
 /*******************************************************************************
@@ -46,23 +51,39 @@ class NowWithOffsetTest extends BaseTest
    {
       long now = System.currentTimeMillis();
 
-      long oneWeekAgoMillis = NowWithOffset.minus(1, ChronoUnit.WEEKS).evaluate().toEpochMilli();
-      assertThat(oneWeekAgoMillis).isCloseTo(now - (7 * DAY_IN_MILLIS), Offset.offset(10_000L));
+      QFieldMetaData dateTimeField = new QFieldMetaData("myDateTime", QFieldType.DATE_TIME);
+      QFieldMetaData dateField     = new QFieldMetaData("myDate", QFieldType.DATE);
 
-      long oneWeekFromNowMillis = NowWithOffset.plus(2, ChronoUnit.WEEKS).evaluate().toEpochMilli();
-      assertThat(oneWeekFromNowMillis).isCloseTo(now + (14 * DAY_IN_MILLIS), Offset.offset(10_000L));
+      {
+         Offset<Long> allowedDiff            = Offset.offset(100L);
+         Offset<Long> allowedDiffPlusOneDay  = Offset.offset(100L + DAY_IN_MILLIS);
+         Offset<Long> allowedDiffPlusTwoDays = Offset.offset(100L + 2 * DAY_IN_MILLIS);
 
-      long oneMonthAgoMillis = NowWithOffset.minus(1, ChronoUnit.MONTHS).evaluate().toEpochMilli();
-      assertThat(oneMonthAgoMillis).isCloseTo(now - (30 * DAY_IN_MILLIS), Offset.offset(10_000L + 2 * DAY_IN_MILLIS));
+         long oneWeekAgoMillis = ((Instant) NowWithOffset.minus(1, ChronoUnit.WEEKS).evaluate(dateTimeField)).toEpochMilli();
+         assertThat(oneWeekAgoMillis).isCloseTo(now - (7 * DAY_IN_MILLIS), allowedDiff);
 
-      long oneMonthFromNowMillis = NowWithOffset.plus(2, ChronoUnit.MONTHS).evaluate().toEpochMilli();
-      assertThat(oneMonthFromNowMillis).isCloseTo(now + (60 * DAY_IN_MILLIS), Offset.offset(10_000L + 3 * DAY_IN_MILLIS));
+         long twoWeeksFromNowMillis = ((Instant) NowWithOffset.plus(2, ChronoUnit.WEEKS).evaluate(dateTimeField)).toEpochMilli();
+         assertThat(twoWeeksFromNowMillis).isCloseTo(now + (14 * DAY_IN_MILLIS), allowedDiff);
 
-      long oneYearAgoMillis = NowWithOffset.minus(1, ChronoUnit.YEARS).evaluate().toEpochMilli();
-      assertThat(oneYearAgoMillis).isCloseTo(now - (365 * DAY_IN_MILLIS), Offset.offset(10_000L + 2 * DAY_IN_MILLIS));
+         long oneMonthAgoMillis = ((Instant) NowWithOffset.minus(1, ChronoUnit.MONTHS).evaluate(dateTimeField)).toEpochMilli();
+         assertThat(oneMonthAgoMillis).isCloseTo(now - (30 * DAY_IN_MILLIS), allowedDiffPlusOneDay);
 
-      long oneYearFromNowMillis = NowWithOffset.plus(2, ChronoUnit.YEARS).evaluate().toEpochMilli();
-      assertThat(oneYearFromNowMillis).isCloseTo(now + (730 * DAY_IN_MILLIS), Offset.offset(10_000L + 3 * DAY_IN_MILLIS));
+         long twoMonthsFromNowMillis = ((Instant) NowWithOffset.plus(2, ChronoUnit.MONTHS).evaluate(dateTimeField)).toEpochMilli();
+         assertThat(twoMonthsFromNowMillis).isCloseTo(now + (60 * DAY_IN_MILLIS), allowedDiffPlusTwoDays);
+
+         long oneYearAgoMillis = ((Instant) NowWithOffset.minus(1, ChronoUnit.YEARS).evaluate(dateTimeField)).toEpochMilli();
+         assertThat(oneYearAgoMillis).isCloseTo(now - (365 * DAY_IN_MILLIS), allowedDiffPlusOneDay);
+
+         long twoYearsFromNowMillis = ((Instant) NowWithOffset.plus(2, ChronoUnit.YEARS).evaluate(dateTimeField)).toEpochMilli();
+         assertThat(twoYearsFromNowMillis).isCloseTo(now + (730 * DAY_IN_MILLIS), allowedDiffPlusTwoDays);
+      }
+
+      {
+         assertThat(NowWithOffset.minus(1, ChronoUnit.WEEKS).evaluate(dateField)).isInstanceOf(LocalDate.class);
+
+         assertEquals(LocalDate.now().minusDays(1), NowWithOffset.minus(1, ChronoUnit.DAYS).evaluate(dateField));
+         assertEquals(LocalDate.now().minusDays(7), NowWithOffset.minus(1, ChronoUnit.WEEKS).evaluate(dateField));
+      }
    }
 
 }

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/reports/BasicRunReportProcessTest.java

@@ -30,6 +30,7 @@ import com.kingsrook.qqq.backend.core.actions.reporting.GenerateReportActionTest
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput;
 import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessOutput;
+import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportFormat;
 import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
 import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportMetaData;
@@ -43,6 +44,7 @@ import static org.assertj.core.api.Assertions.assertThat;
  *******************************************************************************/
 class BasicRunReportProcessTest extends BaseTest
 {
+
    /*******************************************************************************
     **
     *******************************************************************************/
@@ -74,6 +76,18 @@ class BasicRunReportProcessTest extends BaseTest
       runProcessOutput = new RunProcessAction().execute(runProcessInput);
       assertThat(runProcessOutput.getProcessState().getNextStepName()).isPresent().get().isEqualTo(BasicRunReportProcess.STEP_NAME_ACCESS);
       assertThat(runProcessOutput.getValues()).containsKeys("downloadFileName", "serverFilePath");
+
+      ///////////////////////////////////
+      // assert we get xlsx by default //
+      ///////////////////////////////////
+      assertThat(runProcessOutput.getValueString("downloadFileName")).endsWith(".xlsx");
+
+      /////////////////////////////////////////////////////
+      // re-run, requesting CSV, then assert we get that //
+      /////////////////////////////////////////////////////
+      runProcessInput.addValue(BasicRunReportProcess.FIELD_REPORT_FORMAT, ReportFormat.CSV.name());
+      runProcessOutput = new RunProcessAction().execute(runProcessInput);
+      assertThat(runProcessOutput.getValueString("downloadFileName")).endsWith(".csv");
    }
 
 }

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/ValueUtilsTest.java

@@ -32,12 +32,14 @@ import java.time.LocalDateTime;
 import java.time.LocalTime;
 import java.time.Month;
 import java.time.ZoneId;
+import java.util.ArrayList;
 import java.util.Calendar;
 import java.util.GregorianCalendar;
 import com.kingsrook.qqq.backend.core.BaseTest;
 import com.kingsrook.qqq.backend.core.actions.automation.AutomationStatus;
 import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.exceptions.QValueException;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
 import com.kingsrook.qqq.backend.core.model.session.QSession;
 import org.junit.jupiter.api.Test;
 import static org.assertj.core.api.Assertions.assertThat;
@@ -333,4 +335,28 @@ class ValueUtilsTest extends BaseTest
       assertEquals(ZoneId.of("UTC-05:00"), ValueUtils.getSessionOrInstanceZoneId());
    }
 
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testInferQFieldTypeFromValue()
+   {
+      assertNull(ValueUtils.inferQFieldTypeFromValue(null, null));
+      assertNull(ValueUtils.inferQFieldTypeFromValue(new ArrayList<>(), null));
+      assertEquals(QFieldType.HTML, ValueUtils.inferQFieldTypeFromValue(new ArrayList<>(), QFieldType.HTML));
+
+      assertEquals(QFieldType.STRING, ValueUtils.inferQFieldTypeFromValue("value", null));
+      assertEquals(QFieldType.INTEGER, ValueUtils.inferQFieldTypeFromValue(1, null));
+      assertEquals(QFieldType.INTEGER, ValueUtils.inferQFieldTypeFromValue(Integer.valueOf("1"), null));
+      assertEquals(QFieldType.LONG, ValueUtils.inferQFieldTypeFromValue(10_000_000_000L, null));
+      assertEquals(QFieldType.DECIMAL, ValueUtils.inferQFieldTypeFromValue(BigDecimal.ZERO, null));
+      assertEquals(QFieldType.BOOLEAN, ValueUtils.inferQFieldTypeFromValue(true, null));
+      assertEquals(QFieldType.BOOLEAN, ValueUtils.inferQFieldTypeFromValue(Boolean.FALSE, null));
+      assertEquals(QFieldType.DATE_TIME, ValueUtils.inferQFieldTypeFromValue(Instant.now(), null));
+      assertEquals(QFieldType.DATE, ValueUtils.inferQFieldTypeFromValue(LocalDate.now(), null));
+      assertEquals(QFieldType.TIME, ValueUtils.inferQFieldTypeFromValue(LocalTime.now(), null));
+   }
+
 }

qqq-backend-module-mongodb/src/main/java/com/kingsrook/qqq/backend/module/mongodb/actions/AbstractMongoDBAction.java

@@ -41,6 +41,7 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.query.JoinsContext;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.expressions.AbstractFilterExpression;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
 import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
@@ -176,18 +177,28 @@ public class AbstractMongoDBAction
    /*******************************************************************************
     ** Convert a mongodb document to a QRecord.
     *******************************************************************************/
-   protected QRecord documentToRecord(QTableMetaData table, Document document)
+   protected QRecord documentToRecord(QueryInput queryInput, Document document)
    {
-      QRecord record = new QRecord();
+      QTableMetaData table  = queryInput.getTable();
+      QRecord        record = new QRecord();
+
       record.setTableName(table.getName());
 
+      /////////////////////////////////////////////
+      // build the set of field names to include //
+      /////////////////////////////////////////////
+      Set<String> fieldNamesToInclude = queryInput.getFieldNamesToInclude();
+      List<QFieldMetaData> selectedFields = table.getFields().values()
+         .stream().filter(field -> fieldNamesToInclude == null || fieldNamesToInclude.contains(field.getName()))
+         .toList();
+
       //////////////////////////////////////////////////////////////////////////////////////////////
       // first iterate over the table's fields, looking for them (at their backend name (path,    //
       // if it has dots) inside the document note that we'll remove values from the document      //
       // as we go - then after this loop, will handle all remaining values as unstructured fields //
       //////////////////////////////////////////////////////////////////////////////////////////////
       Map<String, Serializable> values = record.getValues();
-      for(QFieldMetaData field : table.getFields().values())
+      for(QFieldMetaData field : selectedFields)
       {
          String fieldName        = field.getName();
          String fieldBackendName = getFieldBackendName(field);
@@ -558,7 +569,7 @@ public class AbstractMongoDBAction
             {
                try
                {
-                  valueListIterator.set(expression.evaluate());
+                  valueListIterator.set(expression.evaluate(field));
                }
                catch(QException qe)
                {

qqq-backend-module-mongodb/src/main/java/com/kingsrook/qqq/backend/module/mongodb/actions/MongoDBQueryAction.java

@@ -41,6 +41,7 @@ import com.kingsrook.qqq.backend.module.mongodb.model.metadata.MongoDBBackendMet
 import com.mongodb.client.FindIterable;
 import com.mongodb.client.MongoCollection;
 import com.mongodb.client.MongoDatabase;
+import com.mongodb.client.model.Projections;
 import org.bson.Document;
 import org.bson.conversions.Bson;
 
@@ -96,6 +97,15 @@ public class MongoDBQueryAction extends AbstractMongoDBAction implements QueryIn
          ////////////////////////////////////////////////////////////
          FindIterable<Document> cursor = collection.find(mongoClientContainer.getMongoSession(), searchQuery);
 
+         ///////////////////////////////////////////////////////////////////////////////////////////////
+         // if input specifies a set of field names to include, then add a 'projection' to the cursor //
+         ///////////////////////////////////////////////////////////////////////////////////////////////
+         if(queryInput.getFieldNamesToInclude() != null)
+         {
+            List<String> backendFieldNames = queryInput.getFieldNamesToInclude().stream().map(f -> getFieldBackendName(table.getField(f))).toList();
+            cursor.projection(Projections.include(backendFieldNames));
+         }
+
          ///////////////////////////////////
          // add a sort operator if needed //
          ///////////////////////////////////
@@ -138,7 +148,7 @@ public class MongoDBQueryAction extends AbstractMongoDBAction implements QueryIn
             actionTimeoutHelper.cancel();
             setQueryStatFirstResultTime();
 
-            QRecord record = documentToRecord(table, document);
+            QRecord record = documentToRecord(queryInput, document);
             queryOutput.addRecord(record);
 
             if(queryInput.getAsyncJobCallback().wasCancelRequested())

qqq-backend-module-mongodb/src/test/java/com/kingsrook/qqq/backend/module/mongodb/actions/MongoDBQueryActionTest.java

@@ -29,6 +29,7 @@ import java.util.Collections;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
+import java.util.Set;
 import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
 import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
 import com.kingsrook.qqq.backend.core.context.QContext;
@@ -54,6 +55,8 @@ import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
 import static org.assertj.core.api.Assertions.assertThat;
 import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
 
 
 /*******************************************************************************
@@ -994,4 +997,46 @@ class MongoDBQueryActionTest extends BaseTest
          .allMatch(r -> r.getValueInteger("storeKey").equals(1));
    }
 
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testFieldNamesToInclude() throws QException
+   {
+      QQueryFilter  filter     = new QQueryFilter().withCriteria("firstName", QCriteriaOperator.EQUALS, "Darin");
+      QueryInput    queryInput = new QueryInput(TestUtils.TABLE_NAME_PERSON).withFilter(filter);
+
+      QRecord record = new QueryAction().execute(queryInput.withFieldNamesToInclude(null)).getRecords().get(0);
+      assertTrue(record.getValues().containsKey("id"));
+      assertTrue(record.getValues().containsKey("firstName"));
+      assertTrue(record.getValues().containsKey("createDate"));
+      //////////////////////////////////////////////////////////////////////////////
+      // note, we get an extra "metaData" field (??), which, i guess is expected? //
+      //////////////////////////////////////////////////////////////////////////////
+      assertEquals(QContext.getQInstance().getTable(TestUtils.TABLE_NAME_PERSON).getFields().size() + 1, record.getValues().size());
+
+      record = new QueryAction().execute(queryInput.withFieldNamesToInclude(Set.of("id", "firstName"))).getRecords().get(0);
+      assertTrue(record.getValues().containsKey("id"));
+      assertTrue(record.getValues().containsKey("firstName"));
+      assertFalse(record.getValues().containsKey("createDate"));
+      assertEquals(2, record.getValues().size());
+      //////////////////////////////////////////////////////////////////////////////////////////////
+      // here, we'd have put _id (which mongo always returns) as "id", since caller requested it. //
+      //////////////////////////////////////////////////////////////////////////////////////////////
+      assertFalse(record.getValues().containsKey("_id"));
+
+      record = new QueryAction().execute(queryInput.withFieldNamesToInclude(Set.of("homeTown"))).getRecords().get(0);
+      assertFalse(record.getValues().containsKey("id"));
+      assertFalse(record.getValues().containsKey("firstName"));
+      assertFalse(record.getValues().containsKey("createDate"));
+      assertTrue(record.getValues().containsKey("homeTown"));
+      //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      // here, mongo always gives back _id (but, we won't have re-mapped it to "id", since caller didn't request it), so, do expect 2 fields here //
+      //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      assertEquals(2, record.getValues().size());
+      assertTrue(record.getValues().containsKey("_id"));
+   }
+
 }

qqq-backend-module-rdbms/pom.xml

@@ -81,6 +81,11 @@
          <artifactId>junit-jupiter-engine</artifactId>
          <scope>test</scope>
       </dependency>
+      <dependency>
+         <groupId>org.junit.jupiter</groupId>
+         <artifactId>junit-jupiter-params</artifactId>
+         <scope>test</scope>
+      </dependency>
       <dependency>
          <groupId>org.assertj</groupId>
          <artifactId>assertj-core</artifactId>

qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/AbstractRDBMSAction.java

@@ -676,7 +676,7 @@ public abstract class AbstractRDBMSAction
                {
                   try
                   {
-                     valueListIterator.set(expression.evaluate());
+                     valueListIterator.set(expression.evaluate(field));
                   }
                   catch(QException qe)
                   {

qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSInsertAction.java

@@ -146,8 +146,8 @@ public class RDBMSInsertAction extends AbstractRDBMSAction implements InsertInte
             // todo sql customization - can edit sql and/or param list
             // todo - non-serial-id style tables
             // todo - other generated values, e.g., createDate...  maybe need to re-select?
-            List<Integer> idList = QueryManager.executeInsertForGeneratedIds(connection, sql.toString(), params);
-            int           index  = 0;
+            List<Serializable> idList = QueryManager.executeInsertForGeneratedIds(connection, sql.toString(), params, table.getField(table.getPrimaryKeyField()).getType());
+            int                index  = 0;
             for(QRecord record : page)
             {
                QRecord outputRecord = new QRecord(record);
@@ -155,7 +155,7 @@ public class RDBMSInsertAction extends AbstractRDBMSAction implements InsertInte
 
                if(CollectionUtils.nullSafeIsEmpty(record.getErrors()))
                {
-                  Integer id = idList.get(index++);
+                  Serializable id = idList.get(index++);
                   outputRecord.setValue(table.getPrimaryKeyField(), id);
                }
             }

qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSQueryAction.java

@@ -34,6 +34,7 @@ import java.util.HashMap;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 import java.util.concurrent.TimeUnit;
 import java.util.stream.Collectors;
 import com.kingsrook.qqq.backend.core.actions.interfaces.QueryInterface;
@@ -94,7 +95,8 @@ public class RDBMSQueryAction extends AbstractRDBMSAction implements QueryInterf
          QTableMetaData table     = queryInput.getTable();
          String         tableName = queryInput.getTableName();
 
-         StringBuilder sql = new StringBuilder(makeSelectClause(queryInput));
+         Selection     selection = makeSelection(queryInput);
+         StringBuilder sql = new StringBuilder(selection.selectClause());
 
          QQueryFilter filter       = clonedOrNewFilter(queryInput.getFilter());
          JoinsContext joinsContext = new JoinsContext(QContext.getQInstance(), tableName, queryInput.getQueryJoins(), filter);
@@ -135,23 +137,6 @@ public class RDBMSQueryAction extends AbstractRDBMSAction implements QueryInterf
             needToCloseConnection = true;
          }
 
-         ////////////////////////////////////////////////////////////////////////////
-         // build the list of fields that will be processed in the result-set loop //
-         ////////////////////////////////////////////////////////////////////////////
-         List<QFieldMetaData> fieldList = new ArrayList<>(table.getFields().values().stream().toList());
-         for(QueryJoin queryJoin : CollectionUtils.nonNullList(queryInput.getQueryJoins()))
-         {
-            if(queryJoin.getSelect())
-            {
-               QTableMetaData joinTable        = QContext.getQInstance().getTable(queryJoin.getJoinTable());
-               String         tableNameOrAlias = queryJoin.getJoinTableOrItsAlias();
-               for(QFieldMetaData joinField : joinTable.getFields().values())
-               {
-                  fieldList.add(joinField.clone().withName(tableNameOrAlias + "." + joinField.getName()));
-               }
-            }
-         }
-
          Long mark = System.currentTimeMillis();
 
          try
@@ -199,7 +184,7 @@ public class RDBMSQueryAction extends AbstractRDBMSAction implements QueryInterf
 
                   for(int i = 1; i <= metaData.getColumnCount(); i++)
                   {
-                     QFieldMetaData field = fieldList.get(i - 1);
+                     QFieldMetaData field = selection.fields().get(i - 1);
 
                      if(!queryInput.getShouldFetchHeavyFields() && field.getIsHeavy())
                      {
@@ -294,26 +279,62 @@ public class RDBMSQueryAction extends AbstractRDBMSAction implements QueryInterf
 
 
 
+   /***************************************************************************
+    ** output wrapper for makeSelection method.
+    ** - selectClause is everything from SELECT up to (but not including) FROM
+    ** - fields are those being selected, in the same order, and with mutated
+    ** names for join fields.
+    ***************************************************************************/
+   private record Selection(String selectClause, List<QFieldMetaData> fields)
+   {
+
+   }
+
+
+
    /*******************************************************************************
-    **
+    ** For a given queryInput, determine what fields are being selected - returning
+    ** a record containing the SELECT clause, as well as a List of QFieldMetaData
+    ** representing those fields - where - note - the names for fields from join
+    ** tables will be prefixed by the join table nameOrAlias.
     *******************************************************************************/
-   private String makeSelectClause(QueryInput queryInput) throws QException
+   private Selection makeSelection(QueryInput queryInput) throws QException
    {
       QInstance       instance   = QContext.getQInstance();
       String          tableName  = queryInput.getTableName();
       List<QueryJoin> queryJoins = queryInput.getQueryJoins();
       QTableMetaData  table      = instance.getTable(tableName);
 
-      boolean requiresDistinct = queryInput.getSelectDistinct() || doesSelectClauseRequireDistinct(table);
-      String  clausePrefix     = (requiresDistinct) ? "SELECT DISTINCT " : "SELECT ";
+      Set<String> fieldNamesToInclude = queryInput.getFieldNamesToInclude();
 
-      List<QFieldMetaData> fieldList = new ArrayList<>(table.getFields().values());
+      ///////////////////////////////////////////////////////////////////////////////////////////////
+      // start with the main table's fields, optionally filtered by the set of fieldNamesToInclude //
+      ///////////////////////////////////////////////////////////////////////////////////////////////
+      List<QFieldMetaData> fieldList = table.getFields().values()
+         .stream().filter(field -> fieldNamesToInclude == null || fieldNamesToInclude.contains(field.getName()))
+         .toList();
+
+      //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      // map those field names to columns, joined with ", ".                                                              //
+      // if a field is heavy, and heavy fields aren't being selected, then replace that field name with a LENGTH function //
+      //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
       String columns = fieldList.stream()
          .map(field -> Pair.of(field, escapeIdentifier(tableName) + "." + escapeIdentifier(getColumnName(field))))
          .map(pair -> wrapHeavyFieldsWithLengthFunctionIfNeeded(pair, queryInput.getShouldFetchHeavyFields()))
          .collect(Collectors.joining(", "));
-      StringBuilder rs = new StringBuilder(clausePrefix).append(columns);
 
+      ///////////////////////////////////////////////////////////////////////////////////////////////////////////
+      // figure out if distinct is being used.  then start building the select clause with the table's columns //
+      ///////////////////////////////////////////////////////////////////////////////////////////////////////////
+      boolean              requiresDistinct   = queryInput.getSelectDistinct() || doesSelectClauseRequireDistinct(table);
+      StringBuilder        selectClause       = new StringBuilder((requiresDistinct) ? "SELECT DISTINCT " : "SELECT ").append(columns);
+      List<QFieldMetaData> selectionFieldList = new ArrayList<>(fieldList);
+
+      boolean needCommaBeforeJoinFields = !columns.isEmpty();
+
+      ///////////////////////////////////
+      // add any 'selected' queryJoins //
+      ///////////////////////////////////
       for(QueryJoin queryJoin : CollectionUtils.nonNullList(queryJoins))
       {
          if(queryJoin.getSelect())
@@ -325,16 +346,41 @@ public class RDBMSQueryAction extends AbstractRDBMSAction implements QueryInterf
                throw new QException("Requested join table [" + queryJoin.getJoinTable() + "] is not a defined table.");
             }
 
-            List<QFieldMetaData> joinFieldList = new ArrayList<>(joinTable.getFields().values());
+            ///////////////////////////////////
+            // filter by fieldNamesToInclude //
+            ///////////////////////////////////
+            List<QFieldMetaData> joinFieldList = joinTable.getFields().values()
+               .stream().filter(field -> fieldNamesToInclude == null || fieldNamesToInclude.contains(tableNameOrAlias + "." + field.getName()))
+               .toList();
+            if(joinFieldList.isEmpty())
+            {
+               continue;
+            }
+
+            /////////////////////////////////////////////////////
+            // map to columns, wrapping heavy fields as needed //
+            /////////////////////////////////////////////////////
             String joinColumns = joinFieldList.stream()
                .map(field -> Pair.of(field, escapeIdentifier(tableNameOrAlias) + "." + escapeIdentifier(getColumnName(field))))
                .map(pair -> wrapHeavyFieldsWithLengthFunctionIfNeeded(pair, queryInput.getShouldFetchHeavyFields()))
                .collect(Collectors.joining(", "));
-            rs.append(", ").append(joinColumns);
+
+            ////////////////////////////////////////////////////////////////////////////////////////////////
+            // append to output objects.                                                                  //
+            // note that fields are cloned, since we are changing their names to have table/alias prefix. //
+            ////////////////////////////////////////////////////////////////////////////////////////////////
+            if(needCommaBeforeJoinFields)
+            {
+               selectClause.append(", ");
+            }
+            selectClause.append(joinColumns);
+            needCommaBeforeJoinFields = true;
+
+            selectionFieldList.addAll(joinFieldList.stream().map(field -> field.clone().withName(tableNameOrAlias + "." + field.getName())).toList());
          }
       }
 
-      return (rs.toString());
+      return (new Selection(selectClause.toString(), selectionFieldList));
    }
 
 

qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/jdbc/QueryManager.java

@@ -53,6 +53,7 @@ import java.util.List;
 import java.util.Map;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.logging.QLogger;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
 import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.PossibleValueEnum;
 import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
 import com.kingsrook.qqq.backend.core.utils.StringUtils;
@@ -526,18 +527,45 @@ public class QueryManager
    /*******************************************************************************
     ** todo - needs (specific) unit test
     *******************************************************************************/
-   public static List<Integer> executeInsertForGeneratedIds(Connection connection, String sql, List<Object> params) throws SQLException
+   public static List<Serializable> executeInsertForGeneratedIds(Connection connection, String sql, List<Object> params, QFieldType idType) throws SQLException
    {
       try(PreparedStatement statement = connection.prepareStatement(sql, Statement.RETURN_GENERATED_KEYS))
       {
          bindParams(params.toArray(), statement);
          incrementStatistic(STAT_QUERIES_RAN);
          statement.executeUpdate();
-         ResultSet     generatedKeys = statement.getGeneratedKeys();
-         List<Integer> rs            = new ArrayList<>();
+
+         /////////////////////////////////////////////////////////////
+         // We default to idType of INTEGER if it was not passed in //
+         /////////////////////////////////////////////////////////////
+         if(idType == null)
+         {
+            idType = QFieldType.INTEGER;
+         }
+
+         ResultSet          generatedKeys = statement.getGeneratedKeys();
+         List<Serializable> rs            = new ArrayList<>();
          while(generatedKeys.next())
          {
-            rs.add(getInteger(generatedKeys, 1));
+            switch(idType)
+            {
+               case INTEGER:
+               {
+                  rs.add(getInteger(generatedKeys, 1));
+                  break;
+               }
+               case LONG:
+               {
+                  rs.add(getLong(generatedKeys, 1));
+                  break;
+               }
+               default:
+               {
+                  LOG.warn("Unknown id data type, attempting to getInteger.", logPair("sql", sql));
+                  rs.add(getInteger(generatedKeys, 1));
+                  break;
+               }
+            }
          }
          return (rs);
       }

qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSQueryActionJoinsTest.java

@@ -25,6 +25,7 @@ package com.kingsrook.qqq.backend.module.rdbms.actions;
 import java.util.Collections;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 import java.util.concurrent.atomic.AtomicInteger;
 import com.kingsrook.qqq.backend.core.actions.tables.CountAction;
 import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
@@ -1052,4 +1053,48 @@ public class RDBMSQueryActionJoinsTest extends RDBMSActionTest
       assertEquals(5, new QueryAction().execute(new QueryInput(TestUtils.TABLE_NAME_PERSON)).getRecords().size());
    }
 
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testFieldNamesToInclude() throws QException
+   {
+      QueryInput queryInput = initQueryRequest();
+      queryInput.withQueryJoin(new QueryJoin(TestUtils.TABLE_NAME_PERSONAL_ID_CARD).withSelect(true));
+      queryInput.withFieldNamesToInclude(Set.of("firstName", TestUtils.TABLE_NAME_PERSONAL_ID_CARD + ".idNumber"));
+      QueryOutput queryOutput = new QueryAction().execute(queryInput);
+      assertThat(queryOutput.getRecords()).allMatch(r -> r.getValues().containsKey("firstName"));
+      assertThat(queryOutput.getRecords()).allMatch(r -> r.getValues().containsKey(TestUtils.TABLE_NAME_PERSONAL_ID_CARD + ".idNumber"));
+      assertThat(queryOutput.getRecords()).allMatch(r -> r.getValues().size() == 2);
+
+      ////////////////////////////////////////////////////////////////////////////////////////////////////
+      // re-run w/ null fieldNamesToInclude -- and should still see those 2, and more (values size > 2) //
+      ////////////////////////////////////////////////////////////////////////////////////////////////////
+      queryInput.withFieldNamesToInclude(null);
+      queryOutput = new QueryAction().execute(queryInput);
+      assertThat(queryOutput.getRecords()).allMatch(r -> r.getValues().containsKey("firstName"));
+      assertThat(queryOutput.getRecords()).allMatch(r -> r.getValues().containsKey(TestUtils.TABLE_NAME_PERSONAL_ID_CARD + ".idNumber"));
+      assertThat(queryOutput.getRecords()).allMatch(r -> r.getValues().size() > 2);
+
+      ////////////////////////////////////////////////////////////////////////////
+      // regression from original build - where only join fields made sql error //
+      ////////////////////////////////////////////////////////////////////////////
+      queryInput.withFieldNamesToInclude(Set.of(TestUtils.TABLE_NAME_PERSONAL_ID_CARD + ".idNumber"));
+      queryOutput = new QueryAction().execute(queryInput);
+      assertThat(queryOutput.getRecords()).allMatch(r -> r.getValues().containsKey(TestUtils.TABLE_NAME_PERSONAL_ID_CARD + ".idNumber"));
+      assertThat(queryOutput.getRecords()).allMatch(r -> r.getValues().size() == 1);
+
+      //////////////////////////////////////////////////////////////////////////////////////////
+      // similar regression to above, if one of the join tables didn't have anything selected //
+      //////////////////////////////////////////////////////////////////////////////////////////
+      queryInput.withQueryJoin(new QueryJoin(TestUtils.TABLE_NAME_PERSONAL_ID_CARD).withAlias("id2").withSelect(true));
+      queryInput.withFieldNamesToInclude(Set.of("firstName", "id2.idNumber"));
+      queryOutput = new QueryAction().execute(queryInput);
+      assertThat(queryOutput.getRecords()).allMatch(r -> r.getValues().containsKey("firstName"));
+      assertThat(queryOutput.getRecords()).allMatch(r -> r.getValues().containsKey("id2.idNumber"));
+      assertThat(queryOutput.getRecords()).allMatch(r -> r.getValues().size() == 2);
+   }
+
 }

qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSQueryActionTest.java

@@ -24,10 +24,14 @@ package com.kingsrook.qqq.backend.module.rdbms.actions;
 
 import java.io.Serializable;
 import java.time.Instant;
+import java.time.LocalDate;
+import java.time.ZoneId;
 import java.time.temporal.ChronoUnit;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
+import java.util.Set;
+import java.util.function.Consumer;
 import java.util.function.Predicate;
 import com.kingsrook.qqq.backend.core.actions.QBackendTransaction;
 import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
@@ -37,22 +41,29 @@ import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.expressions.Now;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.expressions.NowWithOffset;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.expressions.ThisOrLastPeriod;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
 import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
 import com.kingsrook.qqq.backend.core.model.metadata.security.RecordSecurityLock;
 import com.kingsrook.qqq.backend.core.model.session.QSession;
+import com.kingsrook.qqq.backend.core.utils.lambdas.UnsafeFunction;
 import com.kingsrook.qqq.backend.module.rdbms.TestUtils;
 import org.junit.jupiter.api.AfterEach;
-import org.junit.jupiter.api.Assertions;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
 import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
 import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
 
 
 /*******************************************************************************
@@ -82,6 +93,7 @@ public class RDBMSQueryActionTest extends RDBMSActionTest
    void afterEach()
    {
       AbstractRDBMSAction.setLogSQL(false);
+      QContext.getQSession().removeValue(QSession.VALUE_KEY_USER_TIMEZONE);
    }
 
 
@@ -166,7 +178,7 @@ public class RDBMSQueryActionTest extends RDBMSActionTest
       );
       QueryOutput queryOutput = new RDBMSQueryAction().execute(queryInput);
       assertEquals(4, queryOutput.getRecords().size(), "Expected # of rows");
-      Assertions.assertTrue(queryOutput.getRecords().stream().noneMatch(r -> r.getValueString("email").equals(email)), "Should NOT find expected email address");
+      assertTrue(queryOutput.getRecords().stream().noneMatch(r -> r.getValueString("email").equals(email)), "Should NOT find expected email address");
    }
 
 
@@ -199,7 +211,7 @@ public class RDBMSQueryActionTest extends RDBMSActionTest
             .withValues(List.of(1_000_000))));
       queryOutput = new RDBMSQueryAction().execute(queryInput);
       assertEquals(4, queryOutput.getRecords().size(), "Expected # of rows");
-      Assertions.assertTrue(queryOutput.getRecords().stream().noneMatch(r -> Objects.equals(1_000_000, r.getValueInteger("annualSalary"))), "Should NOT find expected salary");
+      assertTrue(queryOutput.getRecords().stream().noneMatch(r -> Objects.equals(1_000_000, r.getValueInteger("annualSalary"))), "Should NOT find expected salary");
    }
 
 
@@ -219,7 +231,7 @@ public class RDBMSQueryActionTest extends RDBMSActionTest
       );
       QueryOutput queryOutput = new RDBMSQueryAction().execute(queryInput);
       assertEquals(2, queryOutput.getRecords().size(), "Expected # of rows");
-      Assertions.assertTrue(queryOutput.getRecords().stream().allMatch(r -> r.getValueInteger("id").equals(2) || r.getValueInteger("id").equals(4)), "Should find expected ids");
+      assertTrue(queryOutput.getRecords().stream().allMatch(r -> r.getValueInteger("id").equals(2) || r.getValueInteger("id").equals(4)), "Should find expected ids");
    }
 
 
@@ -239,7 +251,7 @@ public class RDBMSQueryActionTest extends RDBMSActionTest
       );
       QueryOutput queryOutput = new RDBMSQueryAction().execute(queryInput);
       assertEquals(2, queryOutput.getRecords().size(), "Expected # of rows");
-      Assertions.assertTrue(queryOutput.getRecords().stream().allMatch(r -> r.getValueInteger("id").equals(1) || r.getValueInteger("id").equals(5)), "Should find expected ids");
+      assertTrue(queryOutput.getRecords().stream().allMatch(r -> r.getValueInteger("id").equals(1) || r.getValueInteger("id").equals(5)), "Should find expected ids");
    }
 
 
@@ -259,7 +271,7 @@ public class RDBMSQueryActionTest extends RDBMSActionTest
       );
       QueryOutput queryOutput = new RDBMSQueryAction().execute(queryInput);
       assertEquals(1, queryOutput.getRecords().size(), "Expected # of rows");
-      Assertions.assertTrue(queryOutput.getRecords().stream().allMatch(r -> r.getValueString("email").matches("darin.*")), "Should find matching email address");
+      assertTrue(queryOutput.getRecords().stream().allMatch(r -> r.getValueString("email").matches("darin.*")), "Should find matching email address");
    }
 
 
@@ -279,7 +291,7 @@ public class RDBMSQueryActionTest extends RDBMSActionTest
       );
       QueryOutput queryOutput = new RDBMSQueryAction().execute(queryInput);
       assertEquals(1, queryOutput.getRecords().size(), "Expected # of rows");
-      Assertions.assertTrue(queryOutput.getRecords().stream().allMatch(r -> r.getValueString("email").matches(".*kelkhoff.*")), "Should find matching email address");
+      assertTrue(queryOutput.getRecords().stream().allMatch(r -> r.getValueString("email").matches(".*kelkhoff.*")), "Should find matching email address");
    }
 
 
@@ -299,7 +311,7 @@ public class RDBMSQueryActionTest extends RDBMSActionTest
       );
       QueryOutput queryOutput = new RDBMSQueryAction().execute(queryInput);
       assertEquals(1, queryOutput.getRecords().size(), "Expected # of rows");
-      Assertions.assertTrue(queryOutput.getRecords().stream().allMatch(r -> r.getValueString("email").matches(".*kelkhoff.*")), "Should find matching email address");
+      assertTrue(queryOutput.getRecords().stream().allMatch(r -> r.getValueString("email").matches(".*kelkhoff.*")), "Should find matching email address");
    }
 
 
@@ -319,7 +331,7 @@ public class RDBMSQueryActionTest extends RDBMSActionTest
       );
       QueryOutput queryOutput = new RDBMSQueryAction().execute(queryInput);
       assertEquals(4, queryOutput.getRecords().size(), "Expected # of rows");
-      Assertions.assertTrue(queryOutput.getRecords().stream().noneMatch(r -> r.getValueString("email").matches(".*kelkhoff.*")), "Should find matching email address");
+      assertTrue(queryOutput.getRecords().stream().noneMatch(r -> r.getValueString("email").matches(".*kelkhoff.*")), "Should find matching email address");
    }
 
 
@@ -339,7 +351,7 @@ public class RDBMSQueryActionTest extends RDBMSActionTest
       );
       QueryOutput queryOutput = new RDBMSQueryAction().execute(queryInput);
       assertEquals(1, queryOutput.getRecords().size(), "Expected # of rows");
-      Assertions.assertTrue(queryOutput.getRecords().stream().allMatch(r -> r.getValueString("email").matches(".*gmail.com")), "Should find matching email address");
+      assertTrue(queryOutput.getRecords().stream().allMatch(r -> r.getValueString("email").matches(".*gmail.com")), "Should find matching email address");
    }
 
 
@@ -359,7 +371,7 @@ public class RDBMSQueryActionTest extends RDBMSActionTest
       );
       QueryOutput queryOutput = new RDBMSQueryAction().execute(queryInput);
       assertEquals(4, queryOutput.getRecords().size(), "Expected # of rows");
-      Assertions.assertTrue(queryOutput.getRecords().stream().noneMatch(r -> r.getValueString("email").matches("darin.*")), "Should find matching email address");
+      assertTrue(queryOutput.getRecords().stream().noneMatch(r -> r.getValueString("email").matches("darin.*")), "Should find matching email address");
    }
 
 
@@ -379,7 +391,7 @@ public class RDBMSQueryActionTest extends RDBMSActionTest
       );
       QueryOutput queryOutput = new RDBMSQueryAction().execute(queryInput);
       assertEquals(4, queryOutput.getRecords().size(), "Expected # of rows");
-      Assertions.assertTrue(queryOutput.getRecords().stream().noneMatch(r -> r.getValueString("email").matches(".*kelkhoff.*")), "Should find matching email address");
+      assertTrue(queryOutput.getRecords().stream().noneMatch(r -> r.getValueString("email").matches(".*kelkhoff.*")), "Should find matching email address");
    }
 
 
@@ -399,7 +411,7 @@ public class RDBMSQueryActionTest extends RDBMSActionTest
       );
       QueryOutput queryOutput = new RDBMSQueryAction().execute(queryInput);
       assertEquals(4, queryOutput.getRecords().size(), "Expected # of rows");
-      Assertions.assertTrue(queryOutput.getRecords().stream().noneMatch(r -> r.getValueString("email").matches(".*gmail.com")), "Should find matching email address");
+      assertTrue(queryOutput.getRecords().stream().noneMatch(r -> r.getValueString("email").matches(".*gmail.com")), "Should find matching email address");
    }
 
 
@@ -419,7 +431,7 @@ public class RDBMSQueryActionTest extends RDBMSActionTest
       );
       QueryOutput queryOutput = new RDBMSQueryAction().execute(queryInput);
       assertEquals(2, queryOutput.getRecords().size(), "Expected # of rows");
-      Assertions.assertTrue(queryOutput.getRecords().stream().allMatch(r -> r.getValueInteger("id").equals(1) || r.getValueInteger("id").equals(2)), "Should find expected ids");
+      assertTrue(queryOutput.getRecords().stream().allMatch(r -> r.getValueInteger("id").equals(1) || r.getValueInteger("id").equals(2)), "Should find expected ids");
    }
 
 
@@ -439,7 +451,7 @@ public class RDBMSQueryActionTest extends RDBMSActionTest
       );
       QueryOutput queryOutput = new RDBMSQueryAction().execute(queryInput);
       assertEquals(2, queryOutput.getRecords().size(), "Expected # of rows");
-      Assertions.assertTrue(queryOutput.getRecords().stream().allMatch(r -> r.getValueInteger("id").equals(1) || r.getValueInteger("id").equals(2)), "Should find expected ids");
+      assertTrue(queryOutput.getRecords().stream().allMatch(r -> r.getValueInteger("id").equals(1) || r.getValueInteger("id").equals(2)), "Should find expected ids");
    }
 
 
@@ -459,7 +471,7 @@ public class RDBMSQueryActionTest extends RDBMSActionTest
       );
       QueryOutput queryOutput = new RDBMSQueryAction().execute(queryInput);
       assertEquals(2, queryOutput.getRecords().size(), "Expected # of rows");
-      Assertions.assertTrue(queryOutput.getRecords().stream().allMatch(r -> r.getValueInteger("id").equals(4) || r.getValueInteger("id").equals(5)), "Should find expected ids");
+      assertTrue(queryOutput.getRecords().stream().allMatch(r -> r.getValueInteger("id").equals(4) || r.getValueInteger("id").equals(5)), "Should find expected ids");
    }
 
 
@@ -479,7 +491,7 @@ public class RDBMSQueryActionTest extends RDBMSActionTest
       );
       QueryOutput queryOutput = new RDBMSQueryAction().execute(queryInput);
       assertEquals(2, queryOutput.getRecords().size(), "Expected # of rows");
-      Assertions.assertTrue(queryOutput.getRecords().stream().allMatch(r -> r.getValueInteger("id").equals(4) || r.getValueInteger("id").equals(5)), "Should find expected ids");
+      assertTrue(queryOutput.getRecords().stream().allMatch(r -> r.getValueInteger("id").equals(4) || r.getValueInteger("id").equals(5)), "Should find expected ids");
    }
 
 
@@ -498,7 +510,7 @@ public class RDBMSQueryActionTest extends RDBMSActionTest
          ));
       QueryOutput queryOutput = new RDBMSQueryAction().execute(queryInput);
       assertEquals(1, queryOutput.getRecords().size(), "Expected # of rows");
-      Assertions.assertTrue(queryOutput.getRecords().stream().allMatch(r -> r.getValue("birthDate") == null), "Should find expected row");
+      assertTrue(queryOutput.getRecords().stream().allMatch(r -> r.getValue("birthDate") == null), "Should find expected row");
    }
 
 
@@ -517,7 +529,7 @@ public class RDBMSQueryActionTest extends RDBMSActionTest
          ));
       QueryOutput queryOutput = new RDBMSQueryAction().execute(queryInput);
       assertEquals(5, queryOutput.getRecords().size(), "Expected # of rows");
-      Assertions.assertTrue(queryOutput.getRecords().stream().allMatch(r -> r.getValue("firstName") != null), "Should find expected rows");
+      assertTrue(queryOutput.getRecords().stream().allMatch(r -> r.getValue("firstName") != null), "Should find expected rows");
    }
 
 
@@ -537,7 +549,7 @@ public class RDBMSQueryActionTest extends RDBMSActionTest
          ));
       QueryOutput queryOutput = new RDBMSQueryAction().execute(queryInput);
       assertEquals(3, queryOutput.getRecords().size(), "Expected # of rows");
-      Assertions.assertTrue(queryOutput.getRecords().stream().allMatch(r -> r.getValueInteger("id").equals(2) || r.getValueInteger("id").equals(3) || r.getValueInteger("id").equals(4)), "Should find expected ids");
+      assertTrue(queryOutput.getRecords().stream().allMatch(r -> r.getValueInteger("id").equals(2) || r.getValueInteger("id").equals(3) || r.getValueInteger("id").equals(4)), "Should find expected ids");
    }
 
 
@@ -557,7 +569,7 @@ public class RDBMSQueryActionTest extends RDBMSActionTest
          ));
       QueryOutput queryOutput = new RDBMSQueryAction().execute(queryInput);
       assertEquals(2, queryOutput.getRecords().size(), "Expected # of rows");
-      Assertions.assertTrue(queryOutput.getRecords().stream().allMatch(r -> r.getValueInteger("id").equals(1) || r.getValueInteger("id").equals(5)), "Should find expected ids");
+      assertTrue(queryOutput.getRecords().stream().allMatch(r -> r.getValueInteger("id").equals(1) || r.getValueInteger("id").equals(5)), "Should find expected ids");
    }
 
 
@@ -583,7 +595,7 @@ public class RDBMSQueryActionTest extends RDBMSActionTest
             .withCriteria(new QFilterCriteria().withFieldName("birthDate").withOperator(QCriteriaOperator.LESS_THAN).withValues(List.of(new Now()))));
          QueryOutput queryOutput = new RDBMSQueryAction().execute(queryInput);
          assertEquals(1, queryOutput.getRecords().size(), "Expected # of rows");
-         Assertions.assertTrue(queryOutput.getRecords().stream().anyMatch(r -> r.getValue("firstName").equals("past")), "Should find expected row");
+         assertTrue(queryOutput.getRecords().stream().anyMatch(r -> r.getValue("firstName").equals("past")), "Should find expected row");
       }
 
       {
@@ -593,7 +605,7 @@ public class RDBMSQueryActionTest extends RDBMSActionTest
             .withCriteria(new QFilterCriteria().withFieldName("birthDate").withOperator(QCriteriaOperator.LESS_THAN).withValues(List.of(NowWithOffset.plus(2, ChronoUnit.DAYS)))));
          QueryOutput queryOutput = new RDBMSQueryAction().execute(queryInput);
          assertEquals(1, queryOutput.getRecords().size(), "Expected # of rows");
-         Assertions.assertTrue(queryOutput.getRecords().stream().anyMatch(r -> r.getValue("firstName").equals("past")), "Should find expected row");
+         assertTrue(queryOutput.getRecords().stream().anyMatch(r -> r.getValue("firstName").equals("past")), "Should find expected row");
       }
 
       {
@@ -603,13 +615,108 @@ public class RDBMSQueryActionTest extends RDBMSActionTest
             .withCriteria(new QFilterCriteria().withFieldName("birthDate").withOperator(QCriteriaOperator.GREATER_THAN).withValues(List.of(NowWithOffset.minus(5, ChronoUnit.DAYS)))));
          QueryOutput queryOutput = new RDBMSQueryAction().execute(queryInput);
          assertEquals(2, queryOutput.getRecords().size(), "Expected # of rows");
-         Assertions.assertTrue(queryOutput.getRecords().stream().anyMatch(r -> r.getValue("firstName").equals("past")), "Should find expected row");
-         Assertions.assertTrue(queryOutput.getRecords().stream().anyMatch(r -> r.getValue("firstName").equals("future")), "Should find expected row");
+         assertTrue(queryOutput.getRecords().stream().anyMatch(r -> r.getValue("firstName").equals("past")), "Should find expected row");
+         assertTrue(queryOutput.getRecords().stream().anyMatch(r -> r.getValue("firstName").equals("future")), "Should find expected row");
       }
    }
 
 
 
+   /*******************************************************************************
+    ** Adding additional test conditions, specifically for DATE-precision
+    *******************************************************************************/
+   @ParameterizedTest()
+   @ValueSource(strings = { "UTC", "US/Eastern", "UTC+12" })
+   void testMoreFilterExpressions(String userTimezone) throws QException
+   {
+      QContext.getQSession().setValue(QSession.VALUE_KEY_USER_TIMEZONE, userTimezone);
+
+      LocalDate today     = Instant.now().atZone(ZoneId.of(userTimezone)).toLocalDate();
+      LocalDate yesterday = today.minusDays(1);
+      LocalDate tomorrow  = today.plusDays(1);
+
+      new InsertAction().execute(new InsertInput(TestUtils.TABLE_NAME_PERSON).withRecords(List.of(
+         new QRecord().withValue("email", "-").withValue("firstName", "yesterday").withValue("lastName", "ExpressionTest").withValue("birthDate", yesterday),
+         new QRecord().withValue("email", "-").withValue("firstName", "today").withValue("lastName", "ExpressionTest").withValue("birthDate", today),
+         new QRecord().withValue("email", "-").withValue("firstName", "tomorrow").withValue("lastName", "ExpressionTest").withValue("birthDate", tomorrow))
+      ));
+
+      UnsafeFunction<Consumer<QQueryFilter>, List<QRecord>, QException> testFunction = (filterConsumer) ->
+      {
+         QQueryFilter filter = new QQueryFilter().withCriteria("lastName", QCriteriaOperator.EQUALS, "ExpressionTest");
+         filter.withOrderBy(new QFilterOrderBy("birthDate"));
+         filterConsumer.accept(filter);
+
+         return QueryAction.execute(TestUtils.TABLE_NAME_PERSON, filter);
+      };
+
+      assertOneRecordWithFirstName("today", testFunction.apply(filter -> filter.withCriteria(new QFilterCriteria("birthDate", QCriteriaOperator.EQUALS, new Now()))));
+      assertOneRecordWithFirstName("tomorrow", testFunction.apply(filter -> filter.withCriteria(new QFilterCriteria("birthDate", QCriteriaOperator.GREATER_THAN, new Now()))));
+      assertOneRecordWithFirstName("yesterday", testFunction.apply(filter -> filter.withCriteria(new QFilterCriteria("birthDate", QCriteriaOperator.LESS_THAN, new Now()))));
+      assertTwoRecordsWithFirstNames("yesterday", "today", testFunction.apply(filter -> filter.withCriteria(new QFilterCriteria("birthDate", QCriteriaOperator.LESS_THAN_OR_EQUALS, new Now()))));
+      assertTwoRecordsWithFirstNames("today", "tomorrow", testFunction.apply(filter -> filter.withCriteria(new QFilterCriteria("birthDate", QCriteriaOperator.GREATER_THAN_OR_EQUALS, new Now()))));
+
+      assertNoOfRecords(0, testFunction.apply(filter -> filter.withCriteria(new QFilterCriteria("birthDate", QCriteriaOperator.LESS_THAN, NowWithOffset.minus(1, ChronoUnit.DAYS)))));
+      assertNoOfRecords(3, testFunction.apply(filter -> filter.withCriteria(new QFilterCriteria("birthDate", QCriteriaOperator.LESS_THAN_OR_EQUALS, NowWithOffset.plus(1, ChronoUnit.DAYS)))));
+      assertOneRecordWithFirstName("yesterday", testFunction.apply(filter -> filter.withCriteria(new QFilterCriteria("birthDate", QCriteriaOperator.EQUALS, NowWithOffset.minus(1, ChronoUnit.DAYS)))));
+      assertOneRecordWithFirstName("tomorrow", testFunction.apply(filter -> filter.withCriteria(new QFilterCriteria("birthDate", QCriteriaOperator.EQUALS, NowWithOffset.plus(1, ChronoUnit.DAYS)))));
+      assertNoOfRecords(3, testFunction.apply(filter -> filter.withCriteria(new QFilterCriteria("birthDate", QCriteriaOperator.LESS_THAN, NowWithOffset.plus(1, ChronoUnit.WEEKS)))));
+      assertNoOfRecords(3, testFunction.apply(filter -> filter.withCriteria(new QFilterCriteria("birthDate", QCriteriaOperator.LESS_THAN, NowWithOffset.plus(1, ChronoUnit.MONTHS)))));
+      assertNoOfRecords(3, testFunction.apply(filter -> filter.withCriteria(new QFilterCriteria("birthDate", QCriteriaOperator.LESS_THAN, NowWithOffset.plus(1, ChronoUnit.YEARS)))));
+
+      assertThatThrownBy(() -> testFunction.apply(filter -> filter.withCriteria(new QFilterCriteria("birthDate", QCriteriaOperator.LESS_THAN, NowWithOffset.plus(1, ChronoUnit.HOURS)))))
+         .hasRootCauseMessage("Unsupported unit: Hours");
+
+      assertOneRecordWithFirstName("today", testFunction.apply(filter -> filter.withCriteria(new QFilterCriteria("birthDate", QCriteriaOperator.EQUALS, ThisOrLastPeriod.this_(ChronoUnit.DAYS)))));
+      assertOneRecordWithFirstName("yesterday", testFunction.apply(filter -> filter.withCriteria(new QFilterCriteria("birthDate", QCriteriaOperator.EQUALS, ThisOrLastPeriod.last(ChronoUnit.DAYS)))));
+      assertNoOfRecords(3, testFunction.apply(filter -> filter.withCriteria(new QFilterCriteria("birthDate", QCriteriaOperator.GREATER_THAN, ThisOrLastPeriod.last(ChronoUnit.WEEKS)))));
+      assertNoOfRecords(3, testFunction.apply(filter -> filter.withCriteria(new QFilterCriteria("birthDate", QCriteriaOperator.GREATER_THAN, ThisOrLastPeriod.last(ChronoUnit.MONTHS)))));
+      assertNoOfRecords(3, testFunction.apply(filter -> filter.withCriteria(new QFilterCriteria("birthDate", QCriteriaOperator.GREATER_THAN, ThisOrLastPeriod.last(ChronoUnit.YEARS)))));
+      assertNoOfRecords(0, testFunction.apply(filter -> filter.withCriteria(new QFilterCriteria("birthDate", QCriteriaOperator.LESS_THAN, ThisOrLastPeriod.last(ChronoUnit.WEEKS)))));
+      assertNoOfRecords(0, testFunction.apply(filter -> filter.withCriteria(new QFilterCriteria("birthDate", QCriteriaOperator.LESS_THAN, ThisOrLastPeriod.last(ChronoUnit.MONTHS)))));
+      assertNoOfRecords(0, testFunction.apply(filter -> filter.withCriteria(new QFilterCriteria("birthDate", QCriteriaOperator.LESS_THAN, ThisOrLastPeriod.last(ChronoUnit.YEARS)))));
+
+      assertThatThrownBy(() -> testFunction.apply(filter -> filter.withCriteria(new QFilterCriteria("birthDate", QCriteriaOperator.LESS_THAN, ThisOrLastPeriod.this_(ChronoUnit.HOURS)))))
+         .hasRootCauseMessage("Unsupported unit: Hours");
+      assertThatThrownBy(() -> testFunction.apply(filter -> filter.withCriteria(new QFilterCriteria("birthDate", QCriteriaOperator.LESS_THAN, ThisOrLastPeriod.last(ChronoUnit.MINUTES)))))
+         .hasRootCauseMessage("Unsupported unit: Minutes");
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   private void assertNoOfRecords(Integer expectedSize, List<QRecord> actualRecords)
+   {
+      assertEquals(expectedSize, actualRecords.size());
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   private void assertOneRecordWithFirstName(String expectedFirstName, List<QRecord> actualRecords)
+   {
+      assertEquals(1, actualRecords.size());
+      assertEquals(expectedFirstName, actualRecords.get(0).getValueString("firstName"));
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   private void assertTwoRecordsWithFirstNames(String expectedFirstName0, String expectedFirstName1, List<QRecord> actualRecords)
+   {
+      assertEquals(2, actualRecords.size());
+      assertEquals(expectedFirstName0, actualRecords.get(0).getValueString("firstName"));
+      assertEquals(expectedFirstName1, actualRecords.get(1).getValueString("firstName"));
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/
@@ -1005,7 +1112,36 @@ public class RDBMSQueryActionTest extends RDBMSActionTest
       queryInput.setShouldFetchHeavyFields(true);
       records = new QueryAction().execute(queryInput).getRecords();
       assertThat(records).describedAs("Some records should have the heavy homeTown field set when heavies are requested").anyMatch(r -> r.getValue("homeTown") != null);
+   }
 
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testFieldNamesToInclude() throws QException
+   {
+      QQueryFilter filter     = new QQueryFilter().withCriteria("id", QCriteriaOperator.EQUALS, 1);
+      QueryInput   queryInput = new QueryInput(TestUtils.TABLE_NAME_PERSON).withFilter(filter);
+
+      QRecord record = new QueryAction().execute(queryInput.withFieldNamesToInclude(null)).getRecords().get(0);
+      assertTrue(record.getValues().containsKey("id"));
+      assertTrue(record.getValues().containsKey("firstName"));
+      assertTrue(record.getValues().containsKey("createDate"));
+      assertEquals(QContext.getQInstance().getTable(TestUtils.TABLE_NAME_PERSON).getFields().size(), record.getValues().size());
+
+      record = new QueryAction().execute(queryInput.withFieldNamesToInclude(Set.of("id", "firstName"))).getRecords().get(0);
+      assertTrue(record.getValues().containsKey("id"));
+      assertTrue(record.getValues().containsKey("firstName"));
+      assertFalse(record.getValues().containsKey("createDate"));
+      assertEquals(2, record.getValues().size());
+
+      record = new QueryAction().execute(queryInput.withFieldNamesToInclude(Set.of("homeTown"))).getRecords().get(0);
+      assertFalse(record.getValues().containsKey("id"));
+      assertFalse(record.getValues().containsKey("firstName"));
+      assertFalse(record.getValues().containsKey("createDate"));
+      assertEquals(1, record.getValues().size());
    }
 
 }

qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/jdbc/C3P0PooledConnectionProviderTest.java

@@ -102,6 +102,14 @@ class C3P0PooledConnectionProviderTest extends BaseTest
       backend.setConnectionProvider(new QCodeReference(C3P0PooledConnectionProvider.class));
       QContext.init(qInstance, new QSession());
 
+      /////////////////////////////////////////////////////////////////////////////////
+      // sometimes we're seeing this test fail w/ only 2 connections in the pool...  //
+      // theory is, maybe, the pool doesn't quite have enough time to open them all? //
+      // so, try adding a little sleep here.                                         //
+      /////////////////////////////////////////////////////////////////////////////////
+      new QueryAction().execute(new QueryInput(TestUtils.TABLE_NAME_PERSON));
+      SleepUtils.sleep(500, TimeUnit.MILLISECONDS);
+
       for(int i = 0; i < 5; i++)
       {
          new QueryAction().execute(new QueryInput(TestUtils.TABLE_NAME_PERSON));
@@ -110,17 +118,18 @@ class C3P0PooledConnectionProviderTest extends BaseTest
       JSONObject debugValues = getDebugStateValues(true);
       assertThat(debugValues.getInt("numConnections")).isBetween(3, 6); // due to potential timing issues, sometimes pool will acquire another 3 conns, so 3 or 6 seems ok.
 
-      ////////////////////////////////////////////////////////////////////
-      // open up 4 transactions - confirm the pool opens some new conns //
-      ////////////////////////////////////////////////////////////////////
+      //////////////////////////////////////////////////////////////////////////
+      // open up several transactions - confirm the pool opens some new conns //
+      //////////////////////////////////////////////////////////////////////////
+      int noTransactions = 7;
       List<QBackendTransaction> transactions = new ArrayList<>();
-      for(int i = 0; i < 5; i++)
+      for(int i = 0; i < noTransactions; i++)
       {
          transactions.add(QBackendTransaction.openFor(new InsertInput(TestUtils.TABLE_NAME_PERSON)));
       }
 
       debugValues = getDebugStateValues(true);
-      assertThat(debugValues.getInt("numConnections")).isGreaterThan(3);
+      assertThat(debugValues.getInt("numConnections")).isGreaterThanOrEqualTo(noTransactions);
 
       transactions.forEach(transaction -> transaction.close());
 
@@ -128,7 +137,7 @@ class C3P0PooledConnectionProviderTest extends BaseTest
       // might take a second for the pool to re-claim the closed connections //
       /////////////////////////////////////////////////////////////////////////
       boolean foundMatch = false;
-      for(int i = 0; i < 5; i++)
+      for(int i = 0; i < noTransactions; i++)
       {
          debugValues = getDebugStateValues(true);
          if(debugValues.getInt("numConnections") == debugValues.getInt("numIdleConnections"))

qqq-dev-tools/CURRENT-SNAPSHOT-VERSION

@@ -1 +1 @@
-0.21.0
+0.23.0

qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinImplementation.java

@@ -114,6 +114,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.fields.FieldAdornment;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
 import com.kingsrook.qqq.backend.core.model.metadata.frontend.QFrontendVariant;
+import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.PossibleValueSearchFilterUseCase;
 import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSource;
 import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
@@ -125,6 +126,7 @@ import com.kingsrook.qqq.backend.core.modules.authentication.implementations.Aut
 import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
 import com.kingsrook.qqq.backend.core.utils.ExceptionUtils;
 import com.kingsrook.qqq.backend.core.utils.JsonUtils;
+import com.kingsrook.qqq.backend.core.utils.ObjectUtils;
 import com.kingsrook.qqq.backend.core.utils.StringUtils;
 import com.kingsrook.qqq.backend.core.utils.ValueUtils;
 import com.kingsrook.qqq.backend.core.utils.collections.MapBuilder;
@@ -1792,7 +1794,11 @@ public class QJavalinImplementation
          }
 
          defaultQueryFilter = field.getPossibleValueSourceFilter().clone();
-         defaultQueryFilter.interpretValues(values);
+
+         String                           useCaseParam = QJavalinUtils.getQueryParamOrFormParam(context, "useCase");
+         PossibleValueSearchFilterUseCase useCase      = ObjectUtils.tryElse(() -> PossibleValueSearchFilterUseCase.valueOf(useCaseParam.toUpperCase()), PossibleValueSearchFilterUseCase.FORM);
+
+         defaultQueryFilter.interpretValues(values, useCase);
       }
 
       finishPossibleValuesRequest(context, field.getPossibleValueSourceName(), defaultQueryFilter);

qqq-middleware-javalin/src/test/java/com/kingsrook/qqq/backend/javalin/QJavalinImplementationTest.java

@@ -950,24 +950,138 @@ class QJavalinImplementationTest extends QJavalinTestBase
 
 
 
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   private JSONArray assertPossibleValueSuccessfulResponseAndGetOptionsArray(HttpResponse<String> response)
+   {
+      assertEquals(200, response.getStatus());
+      JSONObject jsonObject = JsonUtils.toJSONObject(response.getBody());
+      assertNotNull(jsonObject);
+      return (jsonObject.getJSONArray("options"));
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   private void assertPossibleValueSuccessfulResponseWithNoOptions(HttpResponse<String> response)
+   {
+      assertEquals(200, response.getStatus());
+      JSONObject jsonObject = JsonUtils.toJSONObject(response.getBody());
+      assertNotNull(jsonObject);
+      assertFalse(jsonObject.has("options")); // no results comes back as result w/o options array.
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/
    @Test
    void testPossibleValueWithFilter()
    {
+      /////////////////////////////////////////////////////////////
+      // post with no search term, and values that find a result //
+      /////////////////////////////////////////////////////////////
       HttpResponse<String> response = Unirest.post(BASE_URL + "/data/pet/possibleValues/ownerPersonId?searchTerm=")
          .field("values", """
-         {"email":"tsamples@mmltholdings.com"}
-         """)
+            {"email":"tsamples@mmltholdings.com"}
+            """)
          .asString();
 
-      assertEquals(200, response.getStatus());
-      JSONObject jsonObject = JsonUtils.toJSONObject(response.getBody());
-      assertNotNull(jsonObject);
-      assertNotNull(jsonObject.getJSONArray("options"));
-      assertEquals(1, jsonObject.getJSONArray("options").length());
-      assertEquals("Tyler Samples (4)", jsonObject.getJSONArray("options").getJSONObject(0).getString("label"));
+      JSONArray options = assertPossibleValueSuccessfulResponseAndGetOptionsArray(response);
+      assertNotNull(options);
+      assertEquals(1, options.length());
+      assertEquals("Tyler Samples (4)", options.getJSONObject(0).getString("label"));
+
+      ///////////////////////////////////////////////////////////
+      // post with search term and values that find no results //
+      ///////////////////////////////////////////////////////////
+      response = Unirest.post(BASE_URL + "/data/pet/possibleValues/ownerPersonId?searchTerm=notFound")
+         .field("values", """
+            {"email":"tsamples@mmltholdings.com"}
+            """)
+         .asString();
+      assertPossibleValueSuccessfulResponseWithNoOptions(response);
+
+      ////////////////////////////////////////////////////////////////
+      // post with no search term, but values that cause no matches //
+      ////////////////////////////////////////////////////////////////
+      response = Unirest.post(BASE_URL + "/data/pet/possibleValues/ownerPersonId?searchTerm=")
+         .field("values", """
+            {"email":"noUser@mmltholdings.com"}
+            """)
+         .asString();
+      assertPossibleValueSuccessfulResponseWithNoOptions(response);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testPossibleValueWithFilterMissingValue()
+   {
+      /////////////////////////////////////////////////////////////
+      // filter use-case, with no values, should return options. //
+      /////////////////////////////////////////////////////////////
+      HttpResponse<String> response = Unirest.post(BASE_URL + "/data/pet/possibleValues/ownerPersonId?searchTerm=&useCase=filter").asString();
+      JSONArray options = assertPossibleValueSuccessfulResponseAndGetOptionsArray(response);
+      assertNotNull(options);
+      assertThat(options.length()).isGreaterThanOrEqualTo(5);
+
+      ///////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      // similarly, values map, but not the 'email' value, that this PVS field is based on, should return options. //
+      ///////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      response = Unirest.post(BASE_URL + "/data/pet/possibleValues/ownerPersonId?searchTerm=&useCase=filter")
+         .field("values", """
+            {"userId":"123"}
+            """)
+         .asString();
+      options = assertPossibleValueSuccessfulResponseAndGetOptionsArray(response);
+      assertNotNull(options);
+      assertThat(options.length()).isGreaterThanOrEqualTo(5);
+
+      ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      // similarly, values map, with the email value, but an empty string in there - should act the same as if it's missing, and not filter the values. //
+      ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      response = Unirest.post(BASE_URL + "/data/pet/possibleValues/ownerPersonId?searchTerm=&useCase=filter")
+         .field("values", """
+            {"email":""}
+            """)
+         .asString();
+      options = assertPossibleValueSuccessfulResponseAndGetOptionsArray(response);
+      assertNotNull(options);
+      assertThat(options.length()).isGreaterThanOrEqualTo(5);
+
+      /////////////////////////////////////////////////////////////////////////
+      // versus form use-case with no values, which should return 0 options. //
+      /////////////////////////////////////////////////////////////////////////
+      response = Unirest.post(BASE_URL + "/data/pet/possibleValues/ownerPersonId?searchTerm=&useCase=form").asString();
+      assertPossibleValueSuccessfulResponseWithNoOptions(response);
+
+      /////////////////////////////////////////////////////////////////////////////////
+      // versus form use-case with expected value, which should return some options. //
+      /////////////////////////////////////////////////////////////////////////////////
+      response = Unirest.post(BASE_URL + "/data/pet/possibleValues/ownerPersonId?searchTerm=&useCase=form")
+         .field("values", """
+            {"email":"tsamples@mmltholdings.com"}
+            """)
+         .asString();
+      options = assertPossibleValueSuccessfulResponseAndGetOptionsArray(response);
+      assertNotNull(options);
+      assertEquals(1, options.length());
+      assertEquals("Tyler Samples (4)", options.getJSONObject(0).getString("label"));
+
+      ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      // finally an unrecognized useCase (or missing or empty), should behave the same as a form, and return 0 options if the filter-value is missing. //
+      ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      assertPossibleValueSuccessfulResponseWithNoOptions(Unirest.post(BASE_URL + "/data/pet/possibleValues/ownerPersonId?searchTerm=&useCase=notAUseCase").asString());
+      assertPossibleValueSuccessfulResponseWithNoOptions(Unirest.post(BASE_URL + "/data/pet/possibleValues/ownerPersonId?searchTerm=").asString());
+      assertPossibleValueSuccessfulResponseWithNoOptions(Unirest.post(BASE_URL + "/data/pet/possibleValues/ownerPersonId?searchTerm=&useCase=").asString());
    }