CE-1772: fix aws expecting content type if object metadata is given

hotfix: allow basepull override values from jobs

checkstyle/config.xml

@@ -213,18 +213,6 @@
          <property name="tokens" value="VARIABLE_DEF"/>
          <property name="allowSamelineMultipleAnnotations" value="true"/>
       </module>
-      <module name="NonEmptyAtclauseDescription"/>
-      <!-- <module name="JavadocTagContinuationIndentation"/> -->
-      <!--
-      <module name="SummaryJavadoc">
-         <property name="forbiddenSummaryFragments" value="^@return the *|^This method returns |^A [{]@code [a-zA-Z0-9]+[}]( is a )"/>
-      </module>
-      -->
-      <!-- <module name="JavadocParagraph"/> -->
-      <module name="AtclauseOrder">
-         <property name="tagOrder" value="@param, @return, @throws, @deprecated"/>
-         <property name="target" value="CLASS_DEF, INTERFACE_DEF, ENUM_DEF, METHOD_DEF, CTOR_DEF, VARIABLE_DEF"/>
-      </module>
       <module name="JavadocMethod">
          <property name="allowMissingParamTags" value="true"/>
          <property name="allowMissingReturnTag" value="true"/>
@@ -233,23 +221,14 @@
       <module name="MissingJavadocMethod">
          <property name="scope" value="private"/>
       </module>
+      <module name="MissingJavadocType">
+         <property name="scope" value="private"/>
+      </module>
       <module name="MethodName">
          <property name="format" value="^[a-z][a-zA-Z0-9_]*$"/>
          <message key="name.invalidPattern"
           value="Method name ''{0}'' must match pattern ''{1}''."/>
       </module>
-      <module name="SingleLineJavadoc">
-         <property name="ignoreInlineTags" value="false"/>
-      </module>
-
-      <module name="MagicNumber">
-         <property name="severity" value="info"/>
-         <property name="tokens" value="NUM_DOUBLE, NUM_FLOAT, NUM_INT"/>
-         <property name="ignoreNumbers" value="0, 1, 2, 3, 4, 5, 6, 7, 8"/>
-         <property name="ignoreFieldDeclaration" value="true"/>
-         <property name="ignoreAnnotation" value="true"/>
-      </module>
-
       <module name="EmptyCatchBlock">
          <property name="exceptionVariableName" value="expected"/>
       </module>

docs/actions/QueryAction.adoc

@@ -33,9 +33,6 @@ If the {link-table} has a `POST_QUERY_CUSTOMIZER` defined, then after records ar
 * `table` - *String, Required* - Name of the table being queried against.
 * `filter` - *<<QQueryFilter>> object* - Specification for what records should be returned, based on *<<QFilterCriteria>>* objects, and how they should be sorted, based on *<<QFilterOrderBy>>* objects.
 If a `filter` is not given, then all rows in the table will be returned by the query.
-* `skip` - *Integer* - Optional number of records to be skipped at the beginning of the result set.
-e.g., for implementing pagination.
-* `limit` - *Integer* - Optional maximum number of records to be returned by the query.
 * `transaction` - *QBackendTransaction object* - Optional transaction object.
 ** Behavior for this object is backend-dependant.
 In an RDBMS backend, this object is generally needed if you want your query to see data that may have been modified within the same transaction.
@@ -55,6 +52,14 @@ But if running a query to provide data as part of a process, then this can gener
 * `shouldMaskPassword` - *boolean, default: true* - Controls whether or not fields with `type` = `PASSWORD` should be masked, or if their actual values should be returned.
 * `queryJoins` - *List of <<QueryJoin>> objects* - Optional list of tables to be joined with the main table being queried.
 See QueryJoin below for further details.
+* `fieldNamesToInclude` - *Set of String* - Optional set of field names to be included in the records.
+** Fields from a queryJoin must be prefixed by the join table's name or alias, and a period.
+Field names from the table being queried should not have any sort of prefix.
+** A `null` set here (default) means to include all fields from the table and any queryJoins set as select=true.
+** An empty set will cause an error, as well any unrecognized field names.
+** `QueryAction` will validate the set of field names, and throw an exception if any unrecognized names are given.
+** _Note that this is an optional feature, which some backend modules may not implement.
+Meaning, they would always return all fields._
 
 ==== QQueryFilter
 A key component of *<<QueryInput>>*, a *QQueryFilter* defines both what records should be included in a query's results (e.g., an SQL `WHERE`), as well as how those results should be sorted (SQL `ORDER BY`).
@@ -68,6 +73,9 @@ In general, multiple *orderBys* can be given (depending on backend implementatio
 ** Each *subFilter* can include its own additional *subFilters*.
 ** Each *subFilter* can specify a different *booleanOperator*.
 ** For example, consider the following *QQueryFilter*, that uses two *subFilters*, and a mix of *booleanOperators*
+* `skip` - *Integer* - Optional number of records to be skipped at the beginning of the result set.
+e.g., for implementing pagination.
+* `limit` - *Integer* - Optional maximum number of records to be returned by the query.
 
 [source,java]
 ----

pom.xml

@@ -46,12 +46,11 @@
    </modules>
 
    <properties>
-      <revision>0.21.0-SNAPSHOT</revision>
+      <revision>0.23.0-SNAPSHOT</revision>
 
       <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
       <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
-      <maven.compiler.source>17</maven.compiler.source>
-      <maven.compiler.target>17</maven.compiler.target>
+      <maven.compiler.release>17</maven.compiler.release>
       <maven.compiler.showDeprecation>true</maven.compiler.showDeprecation>
       <maven.compiler.showWarnings>true</maven.compiler.showWarnings>
       <coverage.haltOnFailure>true</coverage.haltOnFailure>
@@ -168,6 +167,7 @@
                      <violationSeverity>warning</violationSeverity>
                      <excludes>**/target/generated-sources/*.*</excludes>
                      <!-- <linkXRef>false</linkXRef> -->
+                     <includeTestSourceDirectory>true</includeTestSourceDirectory>
                   </configuration>
                   <goals>
                      <goal>check</goal>

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/QBackendTransaction.java

@@ -37,7 +37,7 @@ import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleInterface;
  **
  ** Note:  One would imagine that this class shouldn't ever implement Serializable...
  *******************************************************************************/
-public class QBackendTransaction
+public class QBackendTransaction implements AutoCloseable
 {
 
    /*******************************************************************************

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/audits/AuditAction.java

@@ -225,7 +225,13 @@ public class AuditAction extends AbstractQActionFunction<AuditInput, AuditOutput
                {
                   if(auditSingleInput.getSecurityKeyValues() == null || !auditSingleInput.getSecurityKeyValues().containsKey(recordSecurityLock.getSecurityKeyType()))
                   {
-                     throw (new QException("Missing securityKeyValue [" + recordSecurityLock.getSecurityKeyType() + "] in audit request for table " + auditSingleInput.getAuditTableName()));
+                     ///////////////////////////////////////////////////////
+                     // originally, this case threw...                    //
+                     // but i think it's better to record the audit, just //
+                     // missing its security key value, then to fail...   //
+                     ///////////////////////////////////////////////////////
+                     // throw (new QException("Missing securityKeyValue [" + recordSecurityLock.getSecurityKeyType() + "] in audit request for table " + auditSingleInput.getAuditTableName()));
+                     LOG.info("Missing securityKeyValue in audit request", logPair("table", auditSingleInput.getAuditTableName()), logPair("securityKey", recordSecurityLock.getSecurityKeyType()));
                   }
                }
 
@@ -272,7 +278,7 @@ public class AuditAction extends AbstractQActionFunction<AuditInput, AuditOutput
             List<QRecord> auditDetailRecords = new ArrayList<>();
             for(AuditSingleInput auditSingleInput : CollectionUtils.nonNullList(input.getAuditSingleInputList()))
             {
-               Integer auditId = insertOutput.getRecords().get(i++).getValueInteger("id");
+               Long auditId = insertOutput.getRecords().get(i++).getValueLong("id");
                if(auditId == null)
                {
                   LOG.warn("Missing an id for inserted audit - so won't be able to store its child details...");

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/automation/RecordAutomationStatusUpdater.java

@@ -344,6 +344,9 @@ public class RecordAutomationStatusUpdater
 
 
 
+   /***************************************************************************
+    **
+    ***************************************************************************/
    private record Key(QTableMetaData table, TriggerEvent triggerEvent) {}
 
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/AbstractPreInsertCustomizer.java

@@ -55,10 +55,10 @@ public abstract class AbstractPreInsertCustomizer implements TableCustomizerInte
 
 
 
-   /////////////////////////////////////////////////////////////////////////////////
-   // allow the customizer to specify when it should be executed as part of the   //
-   // insert action.  default (per method in this class) is AFTER_ALL_VALIDATIONS //
-   /////////////////////////////////////////////////////////////////////////////////
+   /***************************************************************************
+    ** allow the customizer to specify when it should be executed as part of the
+    ** insert action.  default (per method in this class) is AFTER_ALL_VALIDATIONS
+    ***************************************************************************/
    public enum WhenToRun
    {
       BEFORE_ALL_VALIDATIONS,

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/ChildInserterPostInsertCustomizer.java

@@ -28,6 +28,7 @@ import java.util.Iterator;
 import java.util.List;
 import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
 import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction;
+import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.exceptions.QRuntimeException;
 import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
@@ -49,6 +50,9 @@ import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
  *******************************************************************************/
 public abstract class ChildInserterPostInsertCustomizer extends AbstractPostInsertCustomizer
 {
+   /***************************************************************************
+    **
+    ***************************************************************************/
    public enum RelationshipType
    {
       PARENT_POINTS_AT_CHILD,
@@ -97,7 +101,7 @@ public abstract class ChildInserterPostInsertCustomizer extends AbstractPostInse
          List<QRecord>  rs               = records;
          List<QRecord>  childrenToInsert = new ArrayList<>();
          QTableMetaData table            = getInsertInput().getTable();
-         QTableMetaData childTable       = getInsertInput().getInstance().getTable(getChildTableName());
+         QTableMetaData childTable       = QContext.getQInstance().getTable(getChildTableName());
 
          ////////////////////////////////////////////////////////////////////////////////
          // iterate over the inserted records, building a list child records to insert //

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/RecordCustomizerUtilityInterface.java

@@ -24,10 +24,12 @@ package com.kingsrook.qqq.backend.core.actions.customizers;
 
 import java.io.Serializable;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 import java.util.Objects;
 import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.logging.QLogger;
+import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
@@ -143,4 +145,19 @@ public interface RecordCustomizerUtilityInterface
       }
    }
 
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   default Map<Serializable, QRecord> getOldRecordMap(List<QRecord> oldRecordList, UpdateInput updateInput)
+   {
+      Map<Serializable, QRecord> oldRecordMap = new HashMap<>();
+      for(QRecord qRecord : oldRecordList)
+      {
+         oldRecordMap.put(qRecord.getValue(updateInput.getTable().getPrimaryKeyField()), qRecord);
+      }
+
+      return (oldRecordMap);
+   }
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/dashboard/widgets/AbstractWidgetRenderer.java

@@ -31,6 +31,7 @@ import java.util.Map;
 import java.util.Set;
 import com.kingsrook.qqq.backend.core.actions.values.QValueFormatter;
 import com.kingsrook.qqq.backend.core.actions.values.SearchPossibleValueSourceAction;
+import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
@@ -102,7 +103,7 @@ public abstract class AbstractWidgetRenderer
             String possibleValueSourceName = dropdownData.getPossibleValueSourceName();
             if(possibleValueSourceName != null)
             {
-               QPossibleValueSource possibleValueSource = input.getInstance().getPossibleValueSource(possibleValueSourceName);
+               QPossibleValueSource possibleValueSource = QContext.getQInstance().getPossibleValueSource(possibleValueSourceName);
 
                /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
                // this looks complicated, but is just look for a label in the dropdown data and if found use it,                                                                                  //

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/dashboard/widgets/ChildRecordListRenderer.java

@@ -34,6 +34,7 @@ import com.kingsrook.qqq.backend.core.actions.tables.CountAction;
 import com.kingsrook.qqq.backend.core.actions.tables.GetAction;
 import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
 import com.kingsrook.qqq.backend.core.actions.values.QValueFormatter;
+import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.exceptions.QNotFoundException;
 import com.kingsrook.qqq.backend.core.logging.QLogger;
@@ -181,10 +182,10 @@ public class ChildRecordListRenderer extends AbstractWidgetRenderer
       {
          String         widgetLabel = input.getQueryParams().get("widgetLabel");
          String         joinName    = input.getQueryParams().get("joinName");
-         QJoinMetaData  join        = input.getInstance().getJoin(joinName);
+         QJoinMetaData  join        = QContext.getQInstance().getJoin(joinName);
          String         id          = input.getQueryParams().get("id");
-         QTableMetaData leftTable   = input.getInstance().getTable(join.getLeftTable());
-         QTableMetaData rightTable  = input.getInstance().getTable(join.getRightTable());
+         QTableMetaData leftTable   = QContext.getQInstance().getTable(join.getLeftTable());
+         QTableMetaData rightTable  = QContext.getQInstance().getTable(join.getRightTable());
 
          Integer maxRows = null;
          if(StringUtils.hasContent(input.getQueryParams().get("maxRows")))
@@ -252,7 +253,7 @@ public class ChildRecordListRenderer extends AbstractWidgetRenderer
             }
          }
 
-         String tablePath   = input.getInstance().getTablePath(rightTable.getName());
+         String tablePath   = QContext.getQInstance().getTablePath(rightTable.getName());
          String viewAllLink = tablePath == null ? null : (tablePath + "?filter=" + URLEncoder.encode(JsonUtils.toJson(filter), Charset.defaultCharset()));
 
          ChildRecordListData widgetData = new ChildRecordListData(widgetLabel, queryOutput, rightTable, tablePath, viewAllLink, totalRows);
@@ -278,7 +279,9 @@ public class ChildRecordListRenderer extends AbstractWidgetRenderer
             Map<String, Serializable> widgetValues = input.getWidgetMetaData().getDefaultValues();
             if(widgetValues.containsKey("disabledFieldsForNewChildRecords"))
             {
-               widgetData.setDisabledFieldsForNewChildRecords((Set<String>) widgetValues.get("disabledFieldsForNewChildRecords"));
+               @SuppressWarnings("unchecked")
+               Set<String> disabledFieldsForNewChildRecords = (Set<String>) widgetValues.get("disabledFieldsForNewChildRecords");
+               widgetData.setDisabledFieldsForNewChildRecords(disabledFieldsForNewChildRecords);
             }
             else
             {

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/dashboard/widgets/ProcessWidgetRenderer.java

@@ -24,6 +24,7 @@ package com.kingsrook.qqq.backend.core.actions.dashboard.widgets;
 
 import java.util.HashMap;
 import com.kingsrook.qqq.backend.core.actions.ActionHelper;
+import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.model.actions.widgets.RenderWidgetInput;
 import com.kingsrook.qqq.backend.core.model.actions.widgets.RenderWidgetOutput;
@@ -57,7 +58,7 @@ public class ProcessWidgetRenderer extends AbstractWidgetRenderer
             setupDropdowns(input, widgetMetaData, data);
 
             String           processName     = (String) widgetMetaData.getDefaultValues().get(WIDGET_PROCESS_NAME);
-            QProcessMetaData processMetaData = input.getInstance().getProcess(processName);
+            QProcessMetaData processMetaData = QContext.getQInstance().getProcess(processName);
             data.setProcessMetaData(processMetaData);
 
             data.setDefaultValues(new HashMap<>(input.getQueryParams()));

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/metadata/MetaDataAction.java

@@ -30,6 +30,7 @@ import java.util.Map;
 import com.kingsrook.qqq.backend.core.actions.ActionHelper;
 import com.kingsrook.qqq.backend.core.actions.permissions.PermissionCheckResult;
 import com.kingsrook.qqq.backend.core.actions.permissions.PermissionsHelper;
+import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.model.actions.metadata.MetaDataInput;
 import com.kingsrook.qqq.backend.core.model.actions.metadata.MetaDataOutput;
@@ -72,7 +73,7 @@ public class MetaDataAction
       // map tables to frontend metadata //
       /////////////////////////////////////
       Map<String, QFrontendTableMetaData> tables = new LinkedHashMap<>();
-      for(Map.Entry<String, QTableMetaData> entry : metaDataInput.getInstance().getTables().entrySet())
+      for(Map.Entry<String, QTableMetaData> entry : QContext.getQInstance().getTables().entrySet())
       {
          String         tableName = entry.getKey();
          QTableMetaData table     = entry.getValue();
@@ -83,7 +84,7 @@ public class MetaDataAction
             continue;
          }
 
-         QBackendMetaData backendForTable = metaDataInput.getInstance().getBackendForTable(tableName);
+         QBackendMetaData backendForTable = QContext.getQInstance().getBackendForTable(tableName);
          tables.put(tableName, new QFrontendTableMetaData(metaDataInput, backendForTable, table, false, false));
          treeNodes.put(tableName, new AppTreeNode(table));
       }
@@ -96,7 +97,7 @@ public class MetaDataAction
       // map processes to frontend metadata //
       ////////////////////////////////////////
       Map<String, QFrontendProcessMetaData> processes = new LinkedHashMap<>();
-      for(Map.Entry<String, QProcessMetaData> entry : metaDataInput.getInstance().getProcesses().entrySet())
+      for(Map.Entry<String, QProcessMetaData> entry : QContext.getQInstance().getProcesses().entrySet())
       {
          String           processName = entry.getKey();
          QProcessMetaData process     = entry.getValue();
@@ -116,7 +117,7 @@ public class MetaDataAction
       // map reports to frontend metadata //
       //////////////////////////////////////
       Map<String, QFrontendReportMetaData> reports = new LinkedHashMap<>();
-      for(Map.Entry<String, QReportMetaData> entry : metaDataInput.getInstance().getReports().entrySet())
+      for(Map.Entry<String, QReportMetaData> entry : QContext.getQInstance().getReports().entrySet())
       {
          String          reportName = entry.getKey();
          QReportMetaData report     = entry.getValue();
@@ -136,7 +137,7 @@ public class MetaDataAction
       // map widgets to frontend metadata //
       //////////////////////////////////////
       Map<String, QFrontendWidgetMetaData> widgets = new LinkedHashMap<>();
-      for(Map.Entry<String, QWidgetMetaDataInterface> entry : metaDataInput.getInstance().getWidgets().entrySet())
+      for(Map.Entry<String, QWidgetMetaDataInterface> entry : QContext.getQInstance().getWidgets().entrySet())
       {
          String                   widgetName = entry.getKey();
          QWidgetMetaDataInterface widget     = entry.getValue();
@@ -154,7 +155,7 @@ public class MetaDataAction
       ///////////////////////////////////////////////////////
       // sort apps - by sortOrder (integer), then by label //
       ///////////////////////////////////////////////////////
-      List<QAppMetaData> sortedApps = metaDataInput.getInstance().getApps().values().stream()
+      List<QAppMetaData> sortedApps = QContext.getQInstance().getApps().values().stream()
          .sorted(Comparator.comparing((QAppMetaData a) -> a.getSortOrder())
             .thenComparing((QAppMetaData a) -> a.getLabel()))
          .toList();
@@ -211,14 +212,14 @@ public class MetaDataAction
       ////////////////////////////////////
       // add branding metadata if found //
       ////////////////////////////////////
-      if(metaDataInput.getInstance().getBranding() != null)
+      if(QContext.getQInstance().getBranding() != null)
       {
-         metaDataOutput.setBranding(metaDataInput.getInstance().getBranding());
+         metaDataOutput.setBranding(QContext.getQInstance().getBranding());
       }
 
-      metaDataOutput.setEnvironmentValues(metaDataInput.getInstance().getEnvironmentValues());
+      metaDataOutput.setEnvironmentValues(QContext.getQInstance().getEnvironmentValues());
 
-      metaDataOutput.setHelpContents(metaDataInput.getInstance().getHelpContent());
+      metaDataOutput.setHelpContents(QContext.getQInstance().getHelpContent());
 
       // todo post-customization - can do whatever w/ the result if you want?
 

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/metadata/ProcessMetaDataAction.java

@@ -23,6 +23,7 @@ package com.kingsrook.qqq.backend.core.actions.metadata;
 
 
 import com.kingsrook.qqq.backend.core.actions.ActionHelper;
+import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.exceptions.QNotFoundException;
 import com.kingsrook.qqq.backend.core.model.actions.metadata.ProcessMetaDataInput;
@@ -47,7 +48,7 @@ public class ProcessMetaDataAction
       // todo pre-customization - just get to modify the request?
       ProcessMetaDataOutput processMetaDataOutput = new ProcessMetaDataOutput();
 
-      QProcessMetaData process = processMetaDataInput.getInstance().getProcess(processMetaDataInput.getProcessName());
+      QProcessMetaData process = QContext.getQInstance().getProcess(processMetaDataInput.getProcessName());
       if(process == null)
       {
          throw (new QNotFoundException("Process [" + processMetaDataInput.getProcessName() + "] was not found."));

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/metadata/TableMetaDataAction.java

@@ -23,6 +23,7 @@ package com.kingsrook.qqq.backend.core.actions.metadata;
 
 
 import com.kingsrook.qqq.backend.core.actions.ActionHelper;
+import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.exceptions.QNotFoundException;
 import com.kingsrook.qqq.backend.core.model.actions.metadata.TableMetaDataInput;
@@ -48,12 +49,12 @@ public class TableMetaDataAction
       // todo pre-customization - just get to modify the request?
       TableMetaDataOutput tableMetaDataOutput = new TableMetaDataOutput();
 
-      QTableMetaData table = tableMetaDataInput.getInstance().getTable(tableMetaDataInput.getTableName());
+      QTableMetaData table = QContext.getQInstance().getTable(tableMetaDataInput.getTableName());
       if(table == null)
       {
          throw (new QNotFoundException("Table [" + tableMetaDataInput.getTableName() + "] was not found."));
       }
-      QBackendMetaData backendForTable = tableMetaDataInput.getInstance().getBackendForTable(table.getName());
+      QBackendMetaData backendForTable = QContext.getQInstance().getBackendForTable(table.getName());
       tableMetaDataOutput.setTable(new QFrontendTableMetaData(tableMetaDataInput, backendForTable, table, true, true));
 
       // todo post-customization - can do whatever w/ the result if you want

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/permissions/PermissionsHelper.java

@@ -49,6 +49,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.tables.Capability;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 import com.kingsrook.qqq.backend.core.model.session.QSession;
 import com.kingsrook.qqq.backend.core.utils.StringUtils;
+import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
 
 
 /*******************************************************************************
@@ -78,6 +79,12 @@ public class PermissionsHelper
       warnAboutPermissionSubTypeForTables(permissionSubType);
       QTableMetaData table = QContext.getQInstance().getTable(tableName);
 
+      if(table == null)
+      {
+         LOG.info("Throwing a permission denied exception in response to a non-existent table name", logPair("tableName", tableName));
+         throw (new QPermissionDeniedException("Permission denied."));
+      }
+
       commonCheckPermissionThrowing(getEffectivePermissionRules(table, QContext.getQInstance()), permissionSubType, table.getName());
    }
 
@@ -185,6 +192,13 @@ public class PermissionsHelper
    public static void checkProcessPermissionThrowing(AbstractActionInput actionInput, String processName, Map<String, Serializable> processValues) throws QPermissionDeniedException
    {
       QProcessMetaData process = QContext.getQInstance().getProcess(processName);
+
+      if(process == null)
+      {
+         LOG.info("Throwing a permission denied exception in response to a non-existent process name", logPair("processName", processName));
+         throw (new QPermissionDeniedException("Permission denied."));
+      }
+
       QPermissionRules effectivePermissionRules = getEffectivePermissionRules(process, QContext.getQInstance());
 
       if(effectivePermissionRules.getCustomPermissionChecker() != null)
@@ -226,6 +240,13 @@ public class PermissionsHelper
    public static void checkAppPermissionThrowing(AbstractActionInput actionInput, String appName) throws QPermissionDeniedException
    {
       QAppMetaData app = QContext.getQInstance().getApp(appName);
+
+      if(app == null)
+      {
+         LOG.info("Throwing a permission denied exception in response to a non-existent app name", logPair("appName", appName));
+         throw (new QPermissionDeniedException("Permission denied."));
+      }
+
       commonCheckPermissionThrowing(getEffectivePermissionRules(app, QContext.getQInstance()), PrivatePermissionSubType.HAS_ACCESS, app.getName());
    }
 
@@ -255,6 +276,13 @@ public class PermissionsHelper
    public static void checkReportPermissionThrowing(AbstractActionInput actionInput, String reportName) throws QPermissionDeniedException
    {
       QReportMetaData report = QContext.getQInstance().getReport(reportName);
+
+      if(report == null)
+      {
+         LOG.info("Throwing a permission denied exception in response to a non-existent process name", logPair("reportName", reportName));
+         throw (new QPermissionDeniedException("Permission denied."));
+      }
+
       commonCheckPermissionThrowing(getEffectivePermissionRules(report, QContext.getQInstance()), PrivatePermissionSubType.HAS_ACCESS, report.getName());
    }
 
@@ -284,6 +312,13 @@ public class PermissionsHelper
    public static void checkWidgetPermissionThrowing(AbstractActionInput actionInput, String widgetName) throws QPermissionDeniedException
    {
       QWidgetMetaDataInterface widget = QContext.getQInstance().getWidget(widgetName);
+
+      if(widget == null)
+      {
+         LOG.info("Throwing a permission denied exception in response to a non-existent widget name", logPair("widgetName", widgetName));
+         throw (new QPermissionDeniedException("Permission denied."));
+      }
+
       commonCheckPermissionThrowing(getEffectivePermissionRules(widget, QContext.getQInstance()), PrivatePermissionSubType.HAS_ACCESS, widget.getName());
    }
 

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/processes/CancelProcessAction.java

@@ -25,6 +25,7 @@ package com.kingsrook.qqq.backend.core.actions.processes;
 import java.util.Optional;
 import java.util.UUID;
 import com.kingsrook.qqq.backend.core.actions.ActionHelper;
+import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.exceptions.QBadRequestException;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.logging.QLogger;
@@ -54,7 +55,7 @@ public class CancelProcessAction extends RunProcessAction
    {
       ActionHelper.validateSession(runProcessInput);
 
-      QProcessMetaData process = runProcessInput.getInstance().getProcess(runProcessInput.getProcessName());
+      QProcessMetaData process = QContext.getQInstance().getProcess(runProcessInput.getProcessName());
       if(process == null)
       {
          throw new QBadRequestException("Process [" + runProcessInput.getProcessName() + "] is not defined in this instance.");

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/processes/RunBackendStepAction.java

@@ -30,6 +30,7 @@ import java.util.Objects;
 import java.util.stream.Collectors;
 import com.kingsrook.qqq.backend.core.actions.ActionHelper;
 import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
+import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.exceptions.QUserFacingException;
 import com.kingsrook.qqq.backend.core.logging.QLogger;
@@ -64,7 +65,7 @@ public class RunBackendStepAction
    {
       ActionHelper.validateSession(runBackendStepInput);
 
-      QProcessMetaData process = runBackendStepInput.getInstance().getProcess(runBackendStepInput.getProcessName());
+      QProcessMetaData process = QContext.getQInstance().getProcess(runBackendStepInput.getProcessName());
       if(process == null)
       {
          throw new QException("Process [" + runBackendStepInput.getProcessName() + "] is not defined in this instance.");

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/processes/RunProcessAction.java

@@ -79,6 +79,7 @@ public class RunProcessAction
 {
    private static final QLogger LOG = QLogger.getLogger(RunProcessAction.class);
 
+   public static final String BASEPULL_KEY_VALUE        = "basepullKeyValue";
    public static final String BASEPULL_THIS_RUNTIME_KEY = "basepullThisRuntimeKey";
    public static final String BASEPULL_LAST_RUNTIME_KEY = "basepullLastRuntimeKey";
    public static final String BASEPULL_TIMESTAMP_FIELD  = "basepullTimestampField";
@@ -99,7 +100,7 @@ public class RunProcessAction
    {
       ActionHelper.validateSession(runProcessInput);
 
-      QProcessMetaData process = runProcessInput.getInstance().getProcess(runProcessInput.getProcessName());
+      QProcessMetaData process = QContext.getQInstance().getProcess(runProcessInput.getProcessName());
       if(process == null)
       {
          throw new QException("Process [" + runProcessInput.getProcessName() + "] is not defined in this instance.");
@@ -517,9 +518,13 @@ public class RunProcessAction
    /*******************************************************************************
     **
     *******************************************************************************/
-   protected String determineBasepullKeyValue(QProcessMetaData process, BasepullConfiguration basepullConfiguration) throws QException
+   protected String determineBasepullKeyValue(QProcessMetaData process, RunProcessInput runProcessInput, BasepullConfiguration basepullConfiguration) throws QException
    {
       String basepullKeyValue = (basepullConfiguration.getKeyValue() != null) ? basepullConfiguration.getKeyValue() : process.getName();
+      if(runProcessInput.getValueString(BASEPULL_KEY_VALUE) != null)
+      {
+         basepullKeyValue = runProcessInput.getValueString(BASEPULL_KEY_VALUE);
+      }
 
       //////////////////////////////////////////////////////////////////////////////////////////////////////////////////
       // if process specifies that it uses variants, look for that data in the session and append to our basepull key //
@@ -551,7 +556,7 @@ public class RunProcessAction
       String basepullTableName            = basepullConfiguration.getTableName();
       String basepullKeyFieldName         = basepullConfiguration.getKeyField();
       String basepullLastRunTimeFieldName = basepullConfiguration.getLastRunTimeFieldName();
-      String basepullKeyValue             = determineBasepullKeyValue(process, basepullConfiguration);
+      String basepullKeyValue             = determineBasepullKeyValue(process, runProcessInput, basepullConfiguration);
 
       ///////////////////////////////////////
       // get the stored basepull timestamp //
@@ -631,7 +636,7 @@ public class RunProcessAction
       String  basepullKeyFieldName                 = basepullConfiguration.getKeyField();
       String  basepullLastRunTimeFieldName         = basepullConfiguration.getLastRunTimeFieldName();
       Integer basepullHoursBackForInitialTimestamp = basepullConfiguration.getHoursBackForInitialTimestamp();
-      String  basepullKeyValue                     = determineBasepullKeyValue(process, basepullConfiguration);
+      String  basepullKeyValue                     = determineBasepullKeyValue(process, runProcessInput, basepullConfiguration);
 
       ///////////////////////////////////////
       // get the stored basepull timestamp //

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/ExportsFullInstanceVerifier.java

@@ -0,0 +1,202 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.actions.reporting;
+
+
+import java.io.ByteArrayOutputStream;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import com.kingsrook.qqq.backend.core.context.QContext;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.logging.QLogger;
+import com.kingsrook.qqq.backend.core.model.actions.reporting.ExportInput;
+import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportDestination;
+import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportFormat;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.Capability;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.ExposedJoin;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
+import com.kingsrook.qqq.backend.core.utils.Pair;
+import com.kingsrook.qqq.backend.core.utils.StringUtils;
+import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
+
+
+/*******************************************************************************
+ ** Utility for verifying that the ExportAction works for all tables, and all
+ ** exposed joins.
+ **
+ ** Meant for use within a unit test, or maybe as part of an instance's boot-up/
+ ** validation.
+ *******************************************************************************/
+public class ExportsFullInstanceVerifier
+{
+   private static final QLogger LOG = QLogger.getLogger(ExportsFullInstanceVerifier.class);
+
+   private boolean filterForAtMostOneRowPerExport = true;
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public void verify(Collection<QTableMetaData> tables) throws QException
+   {
+      Map<Pair<String, String>, Exception> caughtExceptions = new LinkedHashMap<>();
+      for(QTableMetaData table : tables)
+      {
+         if(table.isCapabilityEnabled(QContext.getQInstance().getBackendForTable(table.getName()), Capability.TABLE_QUERY))
+         {
+            LOG.info("Verifying Exports on table", logPair("tableName", table.getName()));
+
+            //////////////////////////////////////////////
+            // run the table by itself (no join fields) //
+            //////////////////////////////////////////////
+            runExport(table.getName(), Collections.emptyList(), "main-table-only", caughtExceptions);
+
+            ///////////////////////////////////////////////////
+            // run once w/ the fields from each exposed join //
+            ///////////////////////////////////////////////////
+            for(ExposedJoin exposedJoin : CollectionUtils.nonNullList(table.getExposedJoins()))
+            {
+               runExport(table.getName(), List.of(exposedJoin), "join-" + exposedJoin.getLabel(), caughtExceptions);
+            }
+
+            /////////////////////////////////////////////////
+            // run w/ all exposed joins (if there are any) //
+            /////////////////////////////////////////////////
+            if(CollectionUtils.nullSafeHasContents(table.getExposedJoins()))
+            {
+               runExport(table.getName(), table.getExposedJoins(), "all-joins", caughtExceptions);
+            }
+         }
+      }
+
+      //////////////////////////////////
+      // log out an exceptions caught //
+      //////////////////////////////////
+      if(!caughtExceptions.isEmpty())
+      {
+         for(Map.Entry<Pair<String, String>, Exception> entry : caughtExceptions.entrySet())
+         {
+            LOG.info("Caught an exception verifying reports", entry.getValue(), logPair("tableName", entry.getKey().getA()), logPair("fieldName", entry.getKey().getB()));
+         }
+         throw (new QException("Reports Verification failed with " + caughtExceptions.size() + " exception" + StringUtils.plural(caughtExceptions.size())));
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private void runExport(String tableName, List<ExposedJoin> exposedJoinList, String description, Map<Pair<String, String>, Exception> caughtExceptions)
+   {
+      try
+      {
+         ////////////////////////////////////////////////////////////////////////////////////
+         // build the list of fieldNames to export - starting with all fields in the table //
+         ////////////////////////////////////////////////////////////////////////////////////
+         List<String> fieldNames = new ArrayList<>();
+         for(QFieldMetaData field : QContext.getQInstance().getTable(tableName).getFields().values())
+         {
+            fieldNames.add(field.getName());
+         }
+
+         ///////////////////////////////////////////////////
+         // add all fields from all exposed joins as well //
+         ///////////////////////////////////////////////////
+         for(ExposedJoin exposedJoin : CollectionUtils.nonNullList(exposedJoinList))
+         {
+            QTableMetaData joinTable = QContext.getQInstance().getTable(exposedJoin.getJoinTable());
+            for(QFieldMetaData field : joinTable.getFields().values())
+            {
+               fieldNames.add(joinTable.getName() + "." + field.getName());
+            }
+         }
+
+         LOG.info("Verifying export", logPair("description", description), logPair("fieldCount", fieldNames.size()));
+
+         QQueryFilter queryFilter = new QQueryFilter();
+
+         /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+         // if caller is okay with a filter that should limit the report to a small number of rows (could be more than 1 for to-many joins), then do so //
+         /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+         if(filterForAtMostOneRowPerExport)
+         {
+            queryFilter.withCriteria(QContext.getQInstance().getTable(tableName).getPrimaryKeyField(), QCriteriaOperator.EQUALS, 1);
+         }
+
+         ExportInput exportInput = new ExportInput();
+         exportInput.setTableName(tableName);
+         exportInput.setFieldNames(fieldNames);
+         exportInput.setReportDestination(new ReportDestination()
+            .withReportOutputStream(new ByteArrayOutputStream())
+            .withReportFormat(ReportFormat.CSV));
+         exportInput.setQueryFilter(queryFilter);
+         new ExportAction().execute(exportInput);
+      }
+      catch(QException e)
+      {
+         caughtExceptions.put(Pair.of(tableName, description), e);
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for filterForAtMostOneRowPerExport
+    *******************************************************************************/
+   public boolean getFilterForAtMostOneRowPerExport()
+   {
+      return (this.filterForAtMostOneRowPerExport);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for filterForAtMostOneRowPerExport
+    *******************************************************************************/
+   public void setFilterForAtMostOneRowPerExport(boolean filterForAtMostOneRowPerExport)
+   {
+      this.filterForAtMostOneRowPerExport = filterForAtMostOneRowPerExport;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for filterForAtMostOneRowPerExport
+    *******************************************************************************/
+   public ExportsFullInstanceVerifier withFilterForAtMostOneRowPerExport(boolean filterForAtMostOneRowPerExport)
+   {
+      this.filterForAtMostOneRowPerExport = filterForAtMostOneRowPerExport;
+      return (this);
+   }
+
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/GenerateReportAction.java

@@ -42,6 +42,7 @@ import com.kingsrook.qqq.backend.core.actions.AbstractQActionFunction;
 import com.kingsrook.qqq.backend.core.actions.async.AsyncRecordPipeLoop;
 import com.kingsrook.qqq.backend.core.actions.customizers.QCodeLoader;
 import com.kingsrook.qqq.backend.core.actions.reporting.customizers.DataSourceQueryInputCustomizer;
+import com.kingsrook.qqq.backend.core.actions.reporting.customizers.ReportCustomRecordSourceInterface;
 import com.kingsrook.qqq.backend.core.actions.reporting.customizers.ReportViewCustomizer;
 import com.kingsrook.qqq.backend.core.actions.tables.CountAction;
 import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
@@ -62,10 +63,13 @@ import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportOutput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.QueryHint;
 import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountOutput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.CriteriaMissingInputValueBehavior;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.FilterUseCase;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.JoinsContext;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryJoin;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
@@ -301,10 +305,19 @@ public class GenerateReportAction extends AbstractQActionFunction<ReportInput, R
       JoinsContext joinsContext = null;
       if(dataSource != null)
       {
+         ///////////////////////////////////////////////////////////////////////////////////////
+         // count records, if applicable, from the data source - for populating into the      //
+         // countByDataSource map, as well as for checking if too many rows (e.g., for excel) //
+         ///////////////////////////////////////////////////////////////////////////////////////
+         countDataSourceRecords(reportInput, dataSource, reportFormat);
+
+         ///////////////////////////////////////////////////////////////////////////////////////////
+         // if there's a source table, set up a joins context, to use below for looking up fields //
+         ///////////////////////////////////////////////////////////////////////////////////////////
          if(StringUtils.hasContent(dataSource.getSourceTable()))
          {
-            joinsContext = new JoinsContext(exportInput.getInstance(), dataSource.getSourceTable(), dataSource.getQueryJoins(), dataSource.getQueryFilter());
-            countDataSourceRecords(reportInput, dataSource, reportFormat);
+            QQueryFilter queryFilter = dataSource.getQueryFilter() == null ? new QQueryFilter() : dataSource.getQueryFilter().clone();
+            joinsContext = new JoinsContext(QContext.getQInstance(), dataSource.getSourceTable(), dataSource.getQueryJoins(), queryFilter);
          }
       }
 
@@ -328,6 +341,7 @@ public class GenerateReportAction extends AbstractQActionFunction<ReportInput, R
             field.setName(column.getName());
             if(StringUtils.hasContent(column.getLabel()))
             {
+
                field.setLabel(column.getLabel());
             }
             fields.add(field);
@@ -344,6 +358,13 @@ public class GenerateReportAction extends AbstractQActionFunction<ReportInput, R
     **
     *******************************************************************************/
    private void countDataSourceRecords(ReportInput reportInput, QReportDataSource dataSource, ReportFormat reportFormat) throws QException
+   {
+      Integer count = null;
+      if(dataSource.getCustomRecordSource() != null)
+      {
+         // todo - add `count` method to interface?
+      }
+      else if(StringUtils.hasContent(dataSource.getSourceTable()))
       {
          QQueryFilter queryFilter = dataSource.getQueryFilter() == null ? new QQueryFilter() : dataSource.getQueryFilter().clone();
          setInputValuesInQueryFilter(reportInput, queryFilter);
@@ -351,17 +372,20 @@ public class GenerateReportAction extends AbstractQActionFunction<ReportInput, R
          CountInput countInput = new CountInput();
          countInput.setTableName(dataSource.getSourceTable());
          countInput.setFilter(queryFilter);
-      countInput.setQueryJoins(dataSource.getQueryJoins());
+         countInput.setQueryJoins(cloneDataSourceQueryJoins(dataSource));
          CountOutput countOutput = new CountAction().execute(countInput);
 
-      if(countOutput.getCount() != null)
-      {
-         countByDataSource.put(dataSource.getName(), countOutput.getCount());
+         count = countOutput.getCount();
+      }
 
-         if(reportFormat.getMaxRows() != null && countOutput.getCount() > reportFormat.getMaxRows())
+      if(count != null)
+      {
+         countByDataSource.put(dataSource.getName(), count);
+
+         if(reportFormat.getMaxRows() != null && count > reportFormat.getMaxRows())
          {
             throw (new QUserFacingException("The requested report would include more rows ("
-               + String.format("%,d", countOutput.getCount()) + ") than the maximum allowed ("
+               + String.format("%,d", count) + ") than the maximum allowed ("
                + String.format("%,d", reportFormat.getMaxRows()) + ") for the selected file format (" + reportFormat + ")."));
          }
       }
@@ -369,6 +393,26 @@ public class GenerateReportAction extends AbstractQActionFunction<ReportInput, R
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private static List<QueryJoin> cloneDataSourceQueryJoins(QReportDataSource dataSource)
+   {
+      if(dataSource == null || dataSource.getQueryJoins() == null)
+      {
+         return (null);
+      }
+
+      List<QueryJoin> rs = new ArrayList<>();
+      for(QueryJoin queryJoin : dataSource.getQueryJoins())
+      {
+         rs.add(queryJoin.clone());
+      }
+      return (rs);
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/
@@ -402,13 +446,19 @@ public class GenerateReportAction extends AbstractQActionFunction<ReportInput, R
       String        tableLabel    = ObjectUtils.tryElse(() -> QContext.getQInstance().getTable(dataSource.getSourceTable()).getLabel(), Objects.requireNonNullElse(dataSource.getSourceTable(), ""));
       AtomicInteger consumedCount = new AtomicInteger(0);
 
-      /////////////////////////////////////////////////////////////////
-      // run a record pipe loop, over the query for this data source //
-      /////////////////////////////////////////////////////////////////
+      /////////////////////////////////////////////////////////////////////////////////////////////////
+      // run a record pipe loop, over the query (or other data-supplier/source) for this data source //
+      /////////////////////////////////////////////////////////////////////////////////////////////////
       RecordPipe recordPipe = new BufferedRecordPipe(1000);
       new AsyncRecordPipeLoop().run("Report[" + reportInput.getReportName() + "]", null, recordPipe, (callback) ->
       {
-         if(dataSource.getSourceTable() != null)
+         if(dataSource.getCustomRecordSource() != null)
+         {
+            ReportCustomRecordSourceInterface recordSource = QCodeLoader.getAdHoc(ReportCustomRecordSourceInterface.class, dataSource.getCustomRecordSource());
+            recordSource.execute(reportInput, dataSource, recordPipe);
+            return (true);
+         }
+         else if(dataSource.getSourceTable() != null)
          {
             QQueryFilter queryFilter = dataSource.getQueryFilter() == null ? new QQueryFilter() : dataSource.getQueryFilter().clone();
             setInputValuesInQueryFilter(reportInput, queryFilter);
@@ -417,12 +467,12 @@ public class GenerateReportAction extends AbstractQActionFunction<ReportInput, R
             queryInput.setRecordPipe(recordPipe);
             queryInput.setTableName(dataSource.getSourceTable());
             queryInput.setFilter(queryFilter);
-            queryInput.setQueryJoins(dataSource.getQueryJoins());
+            queryInput.setQueryJoins(cloneDataSourceQueryJoins(dataSource));
             queryInput.withQueryHint(QueryHint.POTENTIALLY_LARGE_NUMBER_OF_RESULTS);
             queryInput.withQueryHint(QueryHint.MAY_USE_READ_ONLY_BACKEND);
 
             queryInput.setShouldTranslatePossibleValues(true);
-            queryInput.setFieldsToTranslatePossibleValues(setupFieldsToTranslatePossibleValues(reportInput, dataSource, new JoinsContext(reportInput.getInstance(), dataSource.getSourceTable(), dataSource.getQueryJoins(), queryInput.getFilter())));
+            queryInput.setFieldsToTranslatePossibleValues(setupFieldsToTranslatePossibleValues(reportInput, dataSource));
 
             if(dataSource.getQueryInputCustomizer() != null)
             {
@@ -474,7 +524,7 @@ public class GenerateReportAction extends AbstractQActionFunction<ReportInput, R
          }
          consumedCount.getAndAdd(records.size());
 
-         return (consumeRecords(reportInput, dataSource, records, tableView, summaryViews, variantViews));
+         return (consumeRecords(dataSource, records, tableView, summaryViews, variantViews));
       });
 
       ////////////////////////////////////////////////
@@ -493,7 +543,7 @@ public class GenerateReportAction extends AbstractQActionFunction<ReportInput, R
    /*******************************************************************************
     **
     *******************************************************************************/
-   private Set<String> setupFieldsToTranslatePossibleValues(ReportInput reportInput, QReportDataSource dataSource, JoinsContext joinsContext) throws QException
+   private Set<String> setupFieldsToTranslatePossibleValues(ReportInput reportInput, QReportDataSource dataSource) throws QException
    {
       Set<String> fieldsToTranslatePossibleValues = new HashSet<>();
 
@@ -566,7 +616,56 @@ public class GenerateReportAction extends AbstractQActionFunction<ReportInput, R
          return;
       }
 
-      queryFilter.interpretValues(reportInput.getInputValues());
+      //////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      // for reports defined in meta-data, the established rule is, that missing input variable values are discarded. //
+      // but for non-meta-data reports (e.g., user-saved), we expect an exception for missing values.                 //
+      // so, set those use-cases up.                                                                                  //
+      //////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      FilterUseCase filterUseCase;
+      if(StringUtils.hasContent(reportInput.getReportName()) && QContext.getQInstance().getReport(reportInput.getReportName()) != null)
+      {
+         filterUseCase = new ReportFromMetaDataFilterUseCase();
+      }
+      else
+      {
+         filterUseCase = new ReportNotFromMetaDataFilterUseCase();
+      }
+
+      queryFilter.interpretValues(reportInput.getInputValues(), filterUseCase);
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   private static class ReportFromMetaDataFilterUseCase implements FilterUseCase
+   {
+      /***************************************************************************
+       **
+       ***************************************************************************/
+      @Override
+      public CriteriaMissingInputValueBehavior getDefaultCriteriaMissingInputValueBehavior()
+      {
+         return CriteriaMissingInputValueBehavior.REMOVE_FROM_FILTER;
+      }
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   private static class ReportNotFromMetaDataFilterUseCase implements FilterUseCase
+   {
+      /***************************************************************************
+       **
+       ***************************************************************************/
+      @Override
+      public CriteriaMissingInputValueBehavior getDefaultCriteriaMissingInputValueBehavior()
+      {
+         return CriteriaMissingInputValueBehavior.THROW_EXCEPTION;
+      }
    }
 
 
@@ -574,9 +673,9 @@ public class GenerateReportAction extends AbstractQActionFunction<ReportInput, R
    /*******************************************************************************
     **
     *******************************************************************************/
-   private Integer consumeRecords(ReportInput reportInput, QReportDataSource dataSource, List<QRecord> records, QReportView tableView, List<QReportView> summaryViews, List<QReportView> variantViews) throws QException
+   private Integer consumeRecords(QReportDataSource dataSource, List<QRecord> records, QReportView tableView, List<QReportView> summaryViews, List<QReportView> variantViews) throws QException
    {
-      QTableMetaData table = reportInput.getInstance().getTable(dataSource.getSourceTable());
+      QTableMetaData table = QContext.getQInstance().getTable(dataSource.getSourceTable());
 
       ////////////////////////////////////////////////////////////////////////////
       // if this record goes on a table view, add it to the report streamer now //
@@ -687,7 +786,7 @@ public class GenerateReportAction extends AbstractQActionFunction<ReportInput, R
    /*******************************************************************************
     **
     *******************************************************************************/
-   private void addRecordToSummaryKeyAggregates(QTableMetaData table, QRecord record, Map<SummaryKey, Map<String, AggregatesInterface<?, ?>>> viewAggregates, SummaryKey key) throws QException
+   private void addRecordToSummaryKeyAggregates(QTableMetaData table, QRecord record, Map<SummaryKey, Map<String, AggregatesInterface<?, ?>>> viewAggregates, SummaryKey key)
    {
       Map<String, AggregatesInterface<?, ?>> keyAggregates = viewAggregates.computeIfAbsent(key, (name) -> new HashMap<>());
       addRecordToAggregatesMap(table, record, keyAggregates);
@@ -698,7 +797,7 @@ public class GenerateReportAction extends AbstractQActionFunction<ReportInput, R
    /*******************************************************************************
     **
     *******************************************************************************/
-   private void addRecordToAggregatesMap(QTableMetaData table, QRecord record, Map<String, AggregatesInterface<?, ?>> aggregatesMap) throws QException
+   private void addRecordToAggregatesMap(QTableMetaData table, QRecord record, Map<String, AggregatesInterface<?, ?>> aggregatesMap)
    {
       //////////////////////////////////////////////////////////////////////////////////////
       // todo - an optimization could be, to only compute aggregates that we'll need...   //
@@ -706,7 +805,7 @@ public class GenerateReportAction extends AbstractQActionFunction<ReportInput, R
       //////////////////////////////////////////////////////////////////////////////////////
       for(String fieldName : record.getValues().keySet())
       {
-         QFieldMetaData field = null;
+         QFieldMetaData field;
          try
          {
             //////////////////////////////////////////////////////
@@ -780,7 +879,12 @@ public class GenerateReportAction extends AbstractQActionFunction<ReportInput, R
       for(QReportView view : reportViews)
       {
          QReportDataSource dataSource = getDataSource(view.getDataSourceName());
-         QTableMetaData    table         = reportInput.getInstance().getTable(dataSource.getSourceTable());
+         if(dataSource == null)
+         {
+            throw new QReportingException("Data source for summary view was not found (viewName=" + view.getName() + ", dataSourceName=" + view.getDataSourceName() + ").");
+         }
+
+         QTableMetaData table         = QContext.getQInstance().getTable(dataSource.getSourceTable());
          SummaryOutput  summaryOutput = computeSummaryRowsForView(reportInput, view, table);
 
          ExportInput exportInput = new ExportInput();
@@ -867,9 +971,8 @@ public class GenerateReportAction extends AbstractQActionFunction<ReportInput, R
    /*******************************************************************************
     **
     *******************************************************************************/
-   private SummaryOutput computeSummaryRowsForView(ReportInput reportInput, QReportView view, QTableMetaData table) throws QReportingException, QFormulaException
+   private SummaryOutput computeSummaryRowsForView(ReportInput reportInput, QReportView view, QTableMetaData table) throws QFormulaException
    {
-      QValueFormatter              valueFormatter      = new QValueFormatter();
       QMetaDataVariableInterpreter variableInterpreter = new QMetaDataVariableInterpreter();
       variableInterpreter.addValueMap("input", reportInput.getInputValues());
       variableInterpreter.addValueMap("total", getSummaryValuesForInterpreter(totalAggregates));
@@ -941,10 +1044,7 @@ public class GenerateReportAction extends AbstractQActionFunction<ReportInput, R
       //////////////////////////////////////////////////////////////////////////////////////
       if(CollectionUtils.nullSafeHasContents(view.getOrderByFields()))
       {
-         summaryRows.sort((o1, o2) ->
-         {
-            return summaryRowComparator(view, o1, o2);
-         });
+         summaryRows.sort((o1, o2) -> summaryRowComparator(view, o1, o2));
       }
 
       ////////////////
@@ -979,8 +1079,6 @@ public class GenerateReportAction extends AbstractQActionFunction<ReportInput, R
             Serializable serializable = getValueForColumn(variableInterpreter, column);
             totalRow.setValue(column.getName(), serializable);
             thisRowValues.put(column.getName(), serializable);
-
-            String formatted = valueFormatter.formatValue(column.getDisplayFormat(), serializable);
          }
       }
 
@@ -1003,7 +1101,7 @@ public class GenerateReportAction extends AbstractQActionFunction<ReportInput, R
             titleValues.add(variableInterpreter.interpret(titleField));
          }
 
-         title = new QValueFormatter().formatStringWithValues(view.getTitleFormat(), titleValues);
+         title = QValueFormatter.formatStringWithValues(view.getTitleFormat(), titleValues);
       }
       else if(StringUtils.hasContent(view.getTitleFormat()))
       {

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/customizers/ReportCustomRecordSourceInterface.java

@@ -0,0 +1,43 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.actions.reporting.customizers;
+
+
+import com.kingsrook.qqq.backend.core.actions.reporting.RecordPipe;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportInput;
+import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportDataSource;
+
+
+/*******************************************************************************
+ ** Interface to be implemented to do a custom source of data for a report
+ ** (instead of just a query against a table).
+ *******************************************************************************/
+public interface ReportCustomRecordSourceInterface
+{
+
+   /***************************************************************************
+    ** Given the report input, put records into the pipe, for the report.
+    ***************************************************************************/
+   void execute(ReportInput reportInput, QReportDataSource reportDataSource, RecordPipe recordPipe) throws QException;
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/excel/fastexcel/ExcelFastexcelExportStreamer.java

@@ -124,7 +124,7 @@ public class ExcelFastexcelExportStreamer implements ExportStreamerInterface
          if(workbook == null)
          {
             String    appName = ObjectUtils.tryAndRequireNonNullElse(() -> QContext.getQInstance().getBranding().getAppName(), "QQQ");
-            QInstance instance = exportInput.getInstance();
+            QInstance instance = QContext.getQInstance();
             if(instance != null && instance.getBranding() != null && instance.getBranding().getCompanyName() != null)
             {
                appName = instance.getBranding().getCompanyName();

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/excel/poi/ExcelPoiBasedStreamingExportStreamer.java

@@ -128,6 +128,7 @@ public class ExcelPoiBasedStreamingExportStreamer implements ExportStreamerInter
    private Map<String, List<QFieldMetaData>> fieldsPerView            = new HashMap<>();
    private Map<String, Integer>              rowsPerView              = new HashMap<>();
    private Map<String, String>               labelViewsByName         = new HashMap<>();
+   private Map<String, String>               sheetReferenceByViewName = new HashMap<>();
 
 
 
@@ -180,6 +181,7 @@ public class ExcelPoiBasedStreamingExportStreamer implements ExportStreamerInter
             String    sheetReference = sheet.getPackagePart().getPartName().getName().substring(1);
             sheetMapByExcelReference.put(sheetReference, sheet);
             sheetMapByViewName.put(view.getName(), sheet);
+            sheetReferenceByViewName.put(view.getName(), sheetReference);
             sheetCounter++;
          }
 
@@ -446,7 +448,7 @@ public class ExcelPoiBasedStreamingExportStreamer implements ExportStreamerInter
          // - with a new output stream writer    //
          // - and with a SpreadsheetWriter       //
          //////////////////////////////////////////
-         zipOutputStream.putNextEntry(new ZipEntry("xl/worksheets/sheet" + this.sheetIndex++ + ".xml"));
+         zipOutputStream.putNextEntry(new ZipEntry(sheetReferenceByViewName.get(view.getName())));
          activeSheetWriter = new OutputStreamWriter(zipOutputStream);
          sheetWriter = new StreamedSheetWriter(activeSheetWriter);
 

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/excel/poi/StreamedSheetWriter.java

@@ -161,7 +161,7 @@ public class StreamedSheetWriter
             }
          }
 
-         Map<String, Integer> m = new HashMap();
+         Map<String, Integer> m = new HashMap<>();
          m.computeIfAbsent("s", (s) -> 3);
 
          value = rs.toString();

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/scripts/RecordScriptTestInterface.java

@@ -23,6 +23,7 @@ package com.kingsrook.qqq.backend.core.actions.scripts;
 
 
 import java.io.Serializable;
+import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;
 import com.kingsrook.qqq.backend.core.actions.scripts.logging.BuildScriptLogAndScriptLogLineExecutionLogger;
@@ -97,7 +98,7 @@ public class RecordScriptTestInterface implements TestScriptActionInterface
          }
 
          QueryOutput queryOutput = new QueryAction().execute(new QueryInput(tableName)
-            .withFilter(new QQueryFilter(new QFilterCriteria(table.getPrimaryKeyField(), QCriteriaOperator.IN, recordPrimaryKeyList.split(","))))
+            .withFilter(new QQueryFilter(new QFilterCriteria(table.getPrimaryKeyField(), QCriteriaOperator.IN, Arrays.stream(recordPrimaryKeyList.split(",")).toList())))
             .withIncludeAssociations(true));
          if(CollectionUtils.nullSafeIsEmpty(queryOutput.getRecords()))
          {

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/scripts/RunAdHocRecordScriptAction.java

@@ -154,8 +154,9 @@ public class RunAdHocRecordScriptAction
          Method   qRecordListToApiRecordList = apiScriptUtilsClass.getMethod("qRecordListToApiRecordList", List.class, String.class, String.class, String.class);
          Object   apiRecordList              = qRecordListToApiRecordList.invoke(null, input.getRecordList(), input.getTableName(), scriptRevision.getApiName(), scriptRevision.getApiVersion());
 
-         // noinspection unchecked
-         return (ArrayList<? extends Serializable>) apiRecordList;
+         @SuppressWarnings("unchecked")
+         ArrayList<? extends Serializable> rs = (ArrayList<? extends Serializable>) apiRecordList;
+         return rs;
       }
       catch(ClassNotFoundException e)
       {

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/GetAction.java

@@ -352,7 +352,7 @@ public class GetAction
       {
          if(qPossibleValueTranslator == null)
          {
-            qPossibleValueTranslator = new QPossibleValueTranslator(getInput.getInstance(), getInput.getSession());
+            qPossibleValueTranslator = new QPossibleValueTranslator(QContext.getQInstance(), QContext.getQSession());
          }
          qPossibleValueTranslator.translatePossibleValuesInRecords(getInput.getTable(), List.of(returnRecord));
       }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/InsertAction.java

@@ -227,6 +227,11 @@ public class InsertAction extends AbstractQActionFunction<InsertInput, InsertOut
     *******************************************************************************/
    public void performValidations(InsertInput insertInput, boolean isPreview) throws QException
    {
+      if(CollectionUtils.nullSafeIsEmpty(insertInput.getRecords()))
+      {
+         return;
+      }
+
       QTableMetaData table = insertInput.getTable();
 
       ///////////////////////////////////////////////////////////////////
@@ -241,7 +246,7 @@ public class InsertAction extends AbstractQActionFunction<InsertInput, InsertOut
 
       setDefaultValuesInRecords(table, insertInput.getRecords());
 
-      ValueBehaviorApplier.applyFieldBehaviors(ValueBehaviorApplier.Action.INSERT, insertInput.getInstance(), table, insertInput.getRecords(), null);
+      ValueBehaviorApplier.applyFieldBehaviors(ValueBehaviorApplier.Action.INSERT, QContext.getQInstance(), table, insertInput.getRecords(), null);
 
       runPreInsertCustomizerIfItIsTime(insertInput, isPreview, preInsertCustomizer, AbstractPreInsertCustomizer.WhenToRun.BEFORE_UNIQUE_KEY_CHECKS);
       setErrorsIfUniqueKeyErrors(insertInput, table);

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/QueryAction.java

@@ -26,6 +26,7 @@ import java.io.Serializable;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
+import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
@@ -50,8 +51,10 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperat
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryJoin;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.data.QRecordEntity;
 import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.AdornmentType;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
@@ -64,6 +67,7 @@ import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleDispatcher;
 import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleInterface;
 import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
 import com.kingsrook.qqq.backend.core.utils.ListingHash;
+import com.kingsrook.qqq.backend.core.utils.StringUtils;
 import com.kingsrook.qqq.backend.core.utils.ValueUtils;
 
 
@@ -101,6 +105,8 @@ public class QueryAction
          throw (new QException("A table named [" + queryInput.getTableName() + "] was not found in the active QInstance"));
       }
 
+      validateFieldNamesToInclude(queryInput);
+
       QBackendMetaData backend = queryInput.getBackend();
       postQueryRecordCustomizer = QCodeLoader.getTableCustomizer(table, TableCustomizers.POST_QUERY_RECORD.getRole());
       this.queryInput = queryInput;
@@ -158,6 +164,125 @@ public class QueryAction
 
 
 
+   /***************************************************************************
+    ** if QueryInput contains a set of FieldNamesToInclude, then validate that
+    ** those are known field names in the table being queried, or a selected
+    ** queryJoin.
+    ***************************************************************************/
+   static void validateFieldNamesToInclude(QueryInput queryInput) throws QException
+   {
+      Set<String> fieldNamesToInclude = queryInput.getFieldNamesToInclude();
+      if(fieldNamesToInclude == null)
+      {
+         ////////////////////////////////
+         // null set means select all. //
+         ////////////////////////////////
+         return;
+      }
+
+      if(fieldNamesToInclude.isEmpty())
+      {
+         /////////////////////////////////////
+         // empty set, however, is an error //
+         /////////////////////////////////////
+         throw (new QException("An empty set of fieldNamesToInclude was given as queryInput, which is not allowed."));
+      }
+
+      List<String>                unrecognizedFieldNames = new ArrayList<>();
+      Map<String, QTableMetaData> selectedQueryJoins     = null;
+      for(String fieldName : fieldNamesToInclude)
+      {
+         if(fieldName.contains("."))
+         {
+            ////////////////////////////////////////////////
+            // handle names with dots - fields from joins //
+            ////////////////////////////////////////////////
+            String[] parts = fieldName.split("\\.");
+            if(parts.length != 2)
+            {
+               unrecognizedFieldNames.add(fieldName);
+            }
+            else
+            {
+               String tableOrAlias  = parts[0];
+               String fieldNamePart = parts[1];
+
+               ////////////////////////////////////////////
+               // build map of queryJoins being selected //
+               ////////////////////////////////////////////
+               if(selectedQueryJoins == null)
+               {
+                  selectedQueryJoins = new HashMap<>();
+                  for(QueryJoin queryJoin : CollectionUtils.nonNullList(queryInput.getQueryJoins()))
+                  {
+                     if(queryJoin.getSelect())
+                     {
+                        String         joinTableOrAlias = queryJoin.getJoinTableOrItsAlias();
+                        QTableMetaData joinTable        = QContext.getQInstance().getTable(queryJoin.getJoinTable());
+                        if(joinTable != null)
+                        {
+                           selectedQueryJoins.put(joinTableOrAlias, joinTable);
+                        }
+                     }
+                  }
+               }
+
+               if(!selectedQueryJoins.containsKey(tableOrAlias))
+               {
+                  ///////////////////////////////////////////
+                  // unrecognized tableOrAlias is an error //
+                  ///////////////////////////////////////////
+                  unrecognizedFieldNames.add(fieldName);
+               }
+               else
+               {
+                  QTableMetaData joinTable = selectedQueryJoins.get(tableOrAlias);
+                  if(!joinTable.getFields().containsKey(fieldNamePart))
+                  {
+                     //////////////////////////////////////////////////////////
+                     // unrecognized field within the join table is an error //
+                     //////////////////////////////////////////////////////////
+                     unrecognizedFieldNames.add(fieldName);
+                  }
+               }
+            }
+         }
+         else
+         {
+            ///////////////////////////////////////////////////////////////////////
+            // non-join fields - just ensure field name is in table's fields map //
+            ///////////////////////////////////////////////////////////////////////
+            if(!queryInput.getTable().getFields().containsKey(fieldName))
+            {
+               unrecognizedFieldNames.add(fieldName);
+            }
+         }
+      }
+
+      if(!unrecognizedFieldNames.isEmpty())
+      {
+         throw (new QException("QueryInput contained " + unrecognizedFieldNames.size() + " unrecognized field name" + StringUtils.plural(unrecognizedFieldNames) + ": " + StringUtils.join(",", unrecognizedFieldNames)));
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** shorthand way to call for the most common use-case, when you just want the
+    ** entities to be returned, and you just want to pass in a table name and filter.
+    *******************************************************************************/
+   public static <T extends QRecordEntity> List<T> execute(String tableName, Class<T> entityClass, QQueryFilter filter) throws QException
+   {
+      QueryAction queryAction = new QueryAction();
+      QueryInput  queryInput  = new QueryInput();
+      queryInput.setTableName(tableName);
+      queryInput.setFilter(filter);
+      QueryOutput queryOutput = queryAction.execute(queryInput);
+      return (queryOutput.getRecordEntities(entityClass));
+   }
+
+
+
    /*******************************************************************************
     ** shorthand way to call for the most common use-case, when you just want the
     ** records to be returned, and you just want to pass in a table name and filter.
@@ -297,7 +422,7 @@ public class QueryAction
       {
          if(qPossibleValueTranslator == null)
          {
-            qPossibleValueTranslator = new QPossibleValueTranslator(queryInput.getInstance(), queryInput.getSession());
+            qPossibleValueTranslator = new QPossibleValueTranslator(QContext.getQInstance(), QContext.getQSession());
          }
          qPossibleValueTranslator.translatePossibleValuesInRecords(queryInput.getTable(), records, queryInput.getQueryJoins(), queryInput.getFieldsToTranslatePossibleValues());
       }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/UpdateAction.java

@@ -252,7 +252,7 @@ public class UpdateAction
          behaviorsToOmit = Set.of(DynamicDefaultValueBehavior.MODIFY_DATE);
       }
 
-      ValueBehaviorApplier.applyFieldBehaviors(ValueBehaviorApplier.Action.UPDATE, updateInput.getInstance(), table, updateInput.getRecords(), behaviorsToOmit);
+      ValueBehaviorApplier.applyFieldBehaviors(ValueBehaviorApplier.Action.UPDATE, QContext.getQInstance(), table, updateInput.getRecords(), behaviorsToOmit);
       validatePrimaryKeysAreGiven(updateInput);
 
       if(oldRecordList.isPresent())

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/QValueFormatter.java

@@ -28,7 +28,6 @@ import java.time.LocalDateTime;
 import java.time.LocalTime;
 import java.time.ZonedDateTime;
 import java.time.format.DateTimeFormatter;
-import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -68,7 +67,7 @@ public class QValueFormatter
     *******************************************************************************/
    public static String formatValue(QFieldMetaData field, Serializable value)
    {
-      return (formatValue(field.getDisplayFormat(), field.getName(), value));
+      return (formatValue(field.getDisplayFormat(), field.getType(), field.getName(), value));
    }
 
 
@@ -78,7 +77,7 @@ public class QValueFormatter
     *******************************************************************************/
    public static String formatValue(String displayFormat, Serializable value)
    {
-      return (formatValue(displayFormat, "", value));
+      return (formatValue(displayFormat, null, "", value));
    }
 
 
@@ -87,7 +86,7 @@ public class QValueFormatter
     ** For a display format string, an optional fieldName (only used for logging),
     ** and a value, apply the format.
     *******************************************************************************/
-   private static String formatValue(String displayFormat, String fieldName, Serializable value)
+   private static String formatValue(String displayFormat, QFieldType fieldType, String fieldName, Serializable value)
    {
       //////////////////////////////////
       // null values get null results //
@@ -107,6 +106,11 @@ public class QValueFormatter
             return formatBoolean(b);
          }
 
+         if(QFieldType.BOOLEAN.equals(fieldType))
+         {
+            return formatBoolean(ValueUtils.getValueAsBoolean(value));
+         }
+
          if(value instanceof LocalTime lt)
          {
             return formatLocalTime(lt);
@@ -404,6 +408,7 @@ public class QValueFormatter
    }
 
 
+
    /*******************************************************************************
     ** For a single record, set its display values - where caller (meant to stay private)
     ** can specify if they've already done fieldBehaviors (to avoid re-doing).
@@ -462,7 +467,8 @@ public class QValueFormatter
    {
       for(QFieldMetaData field : table.getFields().values())
       {
-         if(field.getType().equals(QFieldType.BLOB))
+         Optional<FieldAdornment> fileDownloadAdornment = field.getAdornment(AdornmentType.FILE_DOWNLOAD);
+         if(fileDownloadAdornment.isPresent())
          {
             //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
             // file name comes from:                                                                                            //
@@ -472,20 +478,7 @@ public class QValueFormatter
             // - tableLabel primaryKey fieldLabel                                                                               //
             // - and - if the FILE_DOWNLOAD adornment had a DEFAULT_EXTENSION, then it gets added (preceded by a dot)           //
             //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-            Optional<FieldAdornment>  fileDownloadAdornment = field.getAdornment(AdornmentType.FILE_DOWNLOAD);
-            Map<String, Serializable> adornmentValues       = Collections.emptyMap();
-
-            if(fileDownloadAdornment.isPresent())
-            {
-               adornmentValues = fileDownloadAdornment.get().getValues();
-            }
-            else
-            {
-               ///////////////////////////////////////////////////////
-               // don't change blobs unless they are file-downloads //
-               ///////////////////////////////////////////////////////
-               continue;
-            }
+            Map<String, Serializable> adornmentValues = fileDownloadAdornment.get().getValues();
 
             String fileNameField    = ValueUtils.getValueAsString(adornmentValues.get(AdornmentType.FileDownloadValues.FILE_NAME_FIELD));
             String fileNameFormat   = ValueUtils.getValueAsString(adornmentValues.get(AdornmentType.FileDownloadValues.FILE_NAME_FORMAT));
@@ -536,7 +529,13 @@ public class QValueFormatter
                   }
                }
 
+               /////////////////////////////////////////////
+               // if field type is blob, update its value //
+               /////////////////////////////////////////////
+               if(QFieldType.BLOB.equals(field.getType()))
+               {
                   record.setValue(field.getName(), "/data/" + table.getName() + "/" + primaryKey + "/" + field.getName() + "/" + fileName);
+               }
                record.setDisplayValue(field.getName(), fileName);
             }
          }
@@ -563,6 +562,7 @@ public class QValueFormatter
             /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
             // heavy fields that weren't fetched - they should have a backend-detail specifying their length (or null if null) //
             /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+            @SuppressWarnings("unchecked")
             Map<String, Serializable> heavyFieldLengths = (Map<String, Serializable>) record.getBackendDetail(QRecord.BACKEND_DETAILS_TYPE_HEAVY_FIELD_LENGTHS);
             if(heavyFieldLengths != null)
             {

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/SearchPossibleValueSourceAction.java

@@ -30,6 +30,7 @@ import java.util.List;
 import java.util.Objects;
 import com.kingsrook.qqq.backend.core.actions.customizers.QCodeLoader;
 import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
+import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.logging.QLogger;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
@@ -69,14 +70,14 @@ public class SearchPossibleValueSourceAction
     *******************************************************************************/
    public SearchPossibleValueSourceOutput execute(SearchPossibleValueSourceInput input) throws QException
    {
-      QInstance            qInstance           = input.getInstance();
+      QInstance            qInstance           = QContext.getQInstance();
       QPossibleValueSource possibleValueSource = qInstance.getPossibleValueSource(input.getPossibleValueSourceName());
       if(possibleValueSource == null)
       {
          throw new QException("Missing possible value source named [" + input.getPossibleValueSourceName() + "]");
       }
 
-      possibleValueTranslator = new QPossibleValueTranslator(input.getInstance(), input.getSession());
+      possibleValueTranslator = new QPossibleValueTranslator(QContext.getQInstance(), QContext.getQSession());
       SearchPossibleValueSourceOutput output = null;
       if(possibleValueSource.getType().equals(QPossibleValueSourceType.ENUM))
       {
@@ -199,7 +200,7 @@ public class SearchPossibleValueSourceAction
       QueryInput queryInput = new QueryInput();
       queryInput.setTableName(possibleValueSource.getTableName());
 
-      QTableMetaData table = input.getInstance().getTable(possibleValueSource.getTableName());
+      QTableMetaData table = QContext.getQInstance().getTable(possibleValueSource.getTableName());
 
       QQueryFilter queryFilter = new QQueryFilter();
       queryFilter.setBooleanOperator(QQueryFilter.BooleanOperator.OR);
@@ -259,9 +260,6 @@ public class SearchPossibleValueSourceAction
          }
       }
 
-      // todo - skip & limit as params
-      queryFilter.setLimit(250);
-
       ///////////////////////////////////////////////////////////////////////////////////////////////////////
       // if given a default filter, make it the 'top level' filter and the one we just created a subfilter //
       ///////////////////////////////////////////////////////////////////////////////////////////////////////
@@ -271,6 +269,9 @@ public class SearchPossibleValueSourceAction
          queryFilter = input.getDefaultQueryFilter();
       }
 
+      // todo - skip & limit as params
+      queryFilter.setLimit(250);
+
       queryFilter.setOrderBys(possibleValueSource.getOrderByFields());
 
       queryInput.setFilter(queryFilter);
@@ -299,6 +300,7 @@ public class SearchPossibleValueSourceAction
    /*******************************************************************************
     **
     *******************************************************************************/
+   @SuppressWarnings({ "rawtypes", "unchecked" })
    private SearchPossibleValueSourceOutput searchPossibleValueCustom(SearchPossibleValueSourceInput input, QPossibleValueSource possibleValueSource)
    {
       try

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/exceptions/QInstanceValidationException.java

@@ -22,8 +22,8 @@
 package com.kingsrook.qqq.backend.core.exceptions;
 
 
-import java.util.Arrays;
 import java.util.List;
+import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
 import com.kingsrook.qqq.backend.core.utils.StringUtils;
 
 
@@ -55,12 +55,11 @@ public class QInstanceValidationException extends QException
     *******************************************************************************/
    public QInstanceValidationException(List<String> reasons)
    {
-      super(
-         (reasons != null && reasons.size() > 0)
-            ? "Instance validation failed for the following reasons:\n - " + StringUtils.join("\n - ", reasons)
+      super((CollectionUtils.nullSafeHasContents(reasons))
+         ? "Instance validation failed for the following reasons:\n - " + StringUtils.join("\n - ", reasons) + "\n(" + reasons.size() + " Total reason" + StringUtils.plural(reasons) + ")"
          : "Validation failed, but no reasons were provided");
 
-      if(reasons != null && reasons.size() > 0)
+      if(CollectionUtils.nullSafeHasContents(reasons))
       {
          this.reasons = reasons;
       }
@@ -68,25 +67,6 @@ public class QInstanceValidationException extends QException
 
 
 
-   /*******************************************************************************
-    ** Constructor of an array/varargs of reasons.  They feed into the core exception message.
-    **
-    *******************************************************************************/
-   public QInstanceValidationException(String... reasons)
-   {
-      super(
-         (reasons != null && reasons.length > 0)
-            ? "Instance validation failed for the following reasons:  " + StringUtils.joinWithCommasAndAnd(Arrays.stream(reasons).toList())
-            : "Validation failed, but no reasons were provided");
-
-      if(reasons != null && reasons.length > 0)
-      {
-         this.reasons = Arrays.stream(reasons).toList();
-      }
-   }
-
-
-
    /*******************************************************************************
     ** Constructor of message & cause - does not populate reasons!
     **

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidator.java

@@ -44,6 +44,7 @@ import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizers;
 import com.kingsrook.qqq.backend.core.actions.dashboard.widgets.AbstractWidgetRenderer;
 import com.kingsrook.qqq.backend.core.actions.metadata.JoinGraph;
 import com.kingsrook.qqq.backend.core.actions.processes.BackendStep;
+import com.kingsrook.qqq.backend.core.actions.reporting.customizers.ReportCustomRecordSourceInterface;
 import com.kingsrook.qqq.backend.core.actions.scripts.TestScriptActionInterface;
 import com.kingsrook.qqq.backend.core.actions.values.QCustomPossibleValueProvider;
 import com.kingsrook.qqq.backend.core.exceptions.QInstanceValidationException;
@@ -922,7 +923,7 @@ public class QInstanceValidator
    /*******************************************************************************
     **
     *******************************************************************************/
-   private <T extends FieldBehavior<T>> void validateTableField(QInstance qInstance, String tableName, String fieldName, QTableMetaData table, QFieldMetaData field)
+   private void validateTableField(QInstance qInstance, String tableName, String fieldName, QTableMetaData table, QFieldMetaData field)
    {
       assertCondition(Objects.equals(fieldName, field.getName()),
          "Inconsistent naming in table " + tableName + " for field " + fieldName + "/" + field.getName() + ".");
@@ -944,12 +945,13 @@ public class QInstanceValidator
          assertCondition(field.getMaxLength() != null, prefix + "specifies a ValueTooLongBehavior, but not a maxLength.");
       }
 
-      Set<Class<FieldBehavior<T>>> usedFieldBehaviorTypes = new HashSet<>();
+      Set<Class<FieldBehavior<?>>> usedFieldBehaviorTypes = new HashSet<>();
       if(field.getBehaviors() != null)
       {
          for(FieldBehavior<?> fieldBehavior : field.getBehaviors())
          {
-            Class<FieldBehavior<T>> behaviorClass = (Class<FieldBehavior<T>>) fieldBehavior.getClass();
+            @SuppressWarnings("unchecked")
+            Class<FieldBehavior<?>> behaviorClass = (Class<FieldBehavior<?>>) fieldBehavior.getClass();
 
             errors.addAll(fieldBehavior.validateBehaviorConfiguration(table, field));
 
@@ -1659,9 +1661,12 @@ public class QInstanceValidator
 
                   String dataSourceErrorPrefix = "Report " + reportName + " data source " + dataSource.getName() + " ";
 
+                  boolean hasASource = false;
+
                   if(StringUtils.hasContent(dataSource.getSourceTable()))
                   {
-                     assertCondition(dataSource.getStaticDataSupplier() == null, dataSourceErrorPrefix + "has both a sourceTable and a staticDataSupplier (exactly 1 is required).");
+                     hasASource = true;
+                     assertCondition(dataSource.getStaticDataSupplier() == null, dataSourceErrorPrefix + "has both a sourceTable and a staticDataSupplier (not compatible together).");
                      if(assertCondition(qInstance.getTable(dataSource.getSourceTable()) != null, dataSourceErrorPrefix + "source table " + dataSource.getSourceTable() + " is not a table in this instance."))
                      {
                         if(dataSource.getQueryFilter() != null)
@@ -1670,14 +1675,21 @@ public class QInstanceValidator
                         }
                      }
                   }
-                  else if(dataSource.getStaticDataSupplier() != null)
+
+                  if(dataSource.getStaticDataSupplier() != null)
                   {
+                     assertCondition(dataSource.getCustomRecordSource() == null, dataSourceErrorPrefix + "has both a staticDataSupplier and a customRecordSource (not compatible together).");
+                     hasASource = true;
                      validateSimpleCodeReference(dataSourceErrorPrefix, dataSource.getStaticDataSupplier(), Supplier.class);
                   }
-                  else
+
+                  if(dataSource.getCustomRecordSource() != null)
                   {
-                     errors.add(dataSourceErrorPrefix + "does not have a sourceTable or a staticDataSupplier (exactly 1 is required).");
+                     hasASource = true;
+                     validateSimpleCodeReference(dataSourceErrorPrefix, dataSource.getCustomRecordSource(), ReportCustomRecordSourceInterface.class);
                   }
+
+                  assertCondition(hasASource, dataSourceErrorPrefix + "does not have a sourceTable, customRecordSource, or a staticDataSupplier.");
                }
             }
 

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/SecretsManagerUtils.java

@@ -24,6 +24,7 @@ package com.kingsrook.qqq.backend.core.instances;
 
 import java.io.File;
 import java.io.IOException;
+import java.nio.charset.StandardCharsets;
 import java.util.Objects;
 import java.util.Optional;
 import com.amazonaws.auth.AWSStaticCredentialsProvider;
@@ -113,7 +114,7 @@ public class SecretsManagerUtils
             dotEnv.renameTo(new File(".env.backup-" + System.currentTimeMillis()));
          }
 
-         FileUtils.writeStringToFile(dotEnv, fullEnv.toString());
+         FileUtils.writeStringToFile(dotEnv, fullEnv.toString(), StandardCharsets.UTF_8);
       }
       else
       {

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/logging/QLogger.java

@@ -280,6 +280,16 @@ public class QLogger
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public void debug(LogPair... logPairs)
+   {
+      logger.warn(() -> makeJsonString(null, null, logPairs));
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/
@@ -420,6 +430,16 @@ public class QLogger
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public void warn(LogPair... logPairs)
+   {
+      logger.warn(() -> makeJsonString(null, null, logPairs));
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/
@@ -480,6 +500,16 @@ public class QLogger
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public void error(LogPair... logPairs)
+   {
+      logger.warn(() -> makeJsonString(null, null, logPairs));
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/AbstractActionInput.java

@@ -32,7 +32,6 @@ import com.kingsrook.qqq.backend.core.exceptions.QInstanceValidationException;
 import com.kingsrook.qqq.backend.core.instances.QInstanceValidator;
 import com.kingsrook.qqq.backend.core.logging.QLogger;
 import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
-import com.kingsrook.qqq.backend.core.model.metadata.authentication.QAuthenticationMetaData;
 import com.kingsrook.qqq.backend.core.model.session.QSession;
 
 
@@ -93,17 +92,6 @@ public class AbstractActionInput
 
 
 
-   /*******************************************************************************
-    **
-    *******************************************************************************/
-   @JsonIgnore
-   public QAuthenticationMetaData getAuthenticationMetaData()
-   {
-      return (getInstance().getAuthentication());
-   }
-
-
-
    /*******************************************************************************
     ** Getter for instance
     **

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/audits/AuditDetailAccumulator.java

@@ -152,5 +152,8 @@ public class AuditDetailAccumulator implements Serializable
    }
 
 
+   /***************************************************************************
+    **
+    ***************************************************************************/
    private record TableNameAndPrimaryKey(String tableName, Serializable primaryKey) {}
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/aggregate/AggregateInput.java

@@ -59,6 +59,30 @@ public class AggregateInput extends AbstractTableActionInput
 
 
 
+   /*******************************************************************************
+    ** Constructor
+    **
+    *******************************************************************************/
+   public AggregateInput(String tableName)
+   {
+      this();
+      setTableName(tableName);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public AggregateInput withTableName(String tableName)
+   {
+      super.withTableName(tableName);
+      return (this);
+   }
+
+
+
    /*******************************************************************************
     ** Getter for filter
     **

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/CriteriaMissingInputValueBehavior.java

@@ -0,0 +1,66 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.model.actions.tables.query;
+
+
+/***************************************************************************
+ ** Possible behaviors for doing interpretValues on a filter, and a criteria
+ ** has a variable value (either as a string-that-looks-like-a-variable,
+ ** as in ${input.foreignId} for a PVS filter, or a FilterVariableExpression),
+ ** and a value for that variable isn't available.
+ **
+ ** Used in conjunction with FilterUseCase and its implementations, e.g.,
+ ** PossibleValueSearchFilterUseCase.
+ ***************************************************************************/
+public enum CriteriaMissingInputValueBehavior
+{
+   //////////////////////////////////////////////////////////////////////
+   // this was the original behavior, before we added this enum.  but, //
+   // it doesn't ever seem entirely valid, and isn't currently used.   //
+   //////////////////////////////////////////////////////////////////////
+   INTERPRET_AS_NULL_VALUE,
+
+   //////////////////////////////////////////////////////////////////////////
+   // make the criteria behave as though it's not in the filter at all.    //
+   // effectively by changing its operator to TRUE, so it always matches.  //
+   // original intended use is for possible-values on  query screens,      //
+   // where a foreign-id isn't present, so we want to show all PV options. //
+   //////////////////////////////////////////////////////////////////////////
+   REMOVE_FROM_FILTER,
+
+   //////////////////////////////////////////////////////////////////////////////////////
+   // make the criteria such that it makes no rows ever match.                         //
+   // e.g., changes it to a FALSE.  I suppose, within an OR, that might                //
+   // not be powerful enough...  but, it solves the immediate use-case in              //
+   // front of us, which is forms, where a PV field should show no values              //
+   // until a foreign key field has a value.                                           //
+   // Note that this use-case used to have the same end-effect by such                 //
+   // variables being interpreted as nulls - but this approach feels more intentional. //
+   //////////////////////////////////////////////////////////////////////////////////////
+   MAKE_NO_MATCHES,
+
+   ///////////////////////////////////////////////////////////////////////////////////////////
+   // throw an exception if a value isn't available.  This is the overall default,          //
+   // and originally was what we did for FilterVariableExpressions, e.g., for saved reports //
+   ///////////////////////////////////////////////////////////////////////////////////////////
+   THROW_EXCEPTION
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/FilterUseCase.java

@@ -0,0 +1,58 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.model.actions.tables.query;
+
+
+/*******************************************************************************
+ ** Interface where we can associate behaviors with various use cases for
+ ** QQueryFilters - the original being, how to handle (in the interpretValues
+ ** method) how to handle missing input values.
+ **
+ ** Includes a default implementation, with a default behavior - which is to
+ ** throw an exception upon missing criteria variable values.
+ *******************************************************************************/
+public interface FilterUseCase
+{
+   FilterUseCase DEFAULT = new DefaultFilterUseCase();
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   CriteriaMissingInputValueBehavior getDefaultCriteriaMissingInputValueBehavior();
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   class DefaultFilterUseCase implements FilterUseCase
+   {
+
+      /***************************************************************************
+       **
+       ***************************************************************************/
+      @Override
+      public CriteriaMissingInputValueBehavior getDefaultCriteriaMissingInputValueBehavior()
+      {
+         return CriteriaMissingInputValueBehavior.THROW_EXCEPTION;
+      }
+   }
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/JoinsContext.java

@@ -378,7 +378,7 @@ public class JoinsContext
             {
                securityFieldTableAlias = matchedQueryJoin.getJoinTableOrItsAlias();
             }
-            tmpTable = instance.getTable(securityFieldTableAlias);
+            tmpTable = instance.getTable(aliasToTableNameMap.getOrDefault(securityFieldTableAlias, securityFieldTableAlias));
 
             ////////////////////////////////////////////////////////////////////////////////////////
             // set the baseTableOrAlias for the next iteration to be this join's joinTableOrAlias //
@@ -404,6 +404,12 @@ public class JoinsContext
                chainIsInner = false;
             }
 
+            if(hasAllAccessKey(recordSecurityLock))
+            {
+               queryJoin.withType(QueryJoin.Type.LEFT);
+               chainIsInner = false;
+            }
+
             addQueryJoin(queryJoin, "forRecordSecurityLock (non-flipped)", "- ");
             addedQueryJoins.add(queryJoin);
             tmpTable = instance.getTable(join.getRightTable());
@@ -423,6 +429,12 @@ public class JoinsContext
                chainIsInner = false;
             }
 
+            if(hasAllAccessKey(recordSecurityLock))
+            {
+               queryJoin.withType(QueryJoin.Type.LEFT);
+               chainIsInner = false;
+            }
+
             addQueryJoin(queryJoin, "forRecordSecurityLock (flipped)", "- ");
             addedQueryJoins.add(queryJoin);
             tmpTable = instance.getTable(join.getLeftTable());
@@ -456,27 +468,37 @@ public class JoinsContext
 
 
 
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   private boolean hasAllAccessKey(RecordSecurityLock recordSecurityLock)
+   {
+      //////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      // check if the key type has an all-access key, and if so, if it's set to true for the current user/session //
+      //////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      QSecurityKeyType securityKeyType = instance.getSecurityKeyType(recordSecurityLock.getSecurityKeyType());
+      if(StringUtils.hasContent(securityKeyType.getAllAccessKeyName()))
+      {
+         QSession session = QContext.getQSession();
+         if(session.hasSecurityKeyValue(securityKeyType.getAllAccessKeyName(), true, QFieldType.BOOLEAN))
+         {
+            return (true);
+         }
+      }
+
+      return (false);
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/
    private void addSubFilterForRecordSecurityLock(RecordSecurityLock recordSecurityLock, QTableMetaData table, String tableNameOrAlias, boolean isOuter, QueryJoin sourceQueryJoin)
    {
-      QSession session = QContext.getQSession();
-
-      //////////////////////////////////////////////////////////////////////////////////////////////////////////////
-      // check if the key type has an all-access key, and if so, if it's set to true for the current user/session //
-      //////////////////////////////////////////////////////////////////////////////////////////////////////////////
-      QSecurityKeyType securityKeyType = instance.getSecurityKeyType(recordSecurityLock.getSecurityKeyType());
-      boolean haveAllAccessKey = false;
-      if(StringUtils.hasContent(securityKeyType.getAllAccessKeyName()))
+      boolean haveAllAccessKey = hasAllAccessKey(recordSecurityLock);
+      if(haveAllAccessKey)
       {
-         //////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-         // if we have all-access on this key, then we don't need a criterion for it (as long as we're in an AND filter) //
-         //////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-         if(session.hasSecurityKeyValue(securityKeyType.getAllAccessKeyName(), true, QFieldType.BOOLEAN))
-         {
-            haveAllAccessKey = true;
-
          if(sourceQueryJoin != null)
          {
             ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
@@ -495,7 +517,6 @@ public class JoinsContext
             return;
          }
       }
-      }
 
       /////////////////////////////////////////////////////////////////////////////////////////
       // for locks w/o a join chain, the lock fieldName will simply be a field on the table. //
@@ -545,7 +566,7 @@ public class JoinsContext
       }
       else
       {
-         List<Serializable> securityKeyValues = session.getSecurityKeyValues(recordSecurityLock.getSecurityKeyType(), type);
+         List<Serializable> securityKeyValues = QContext.getQSession().getSecurityKeyValues(recordSecurityLock.getSecurityKeyType(), type);
          if(CollectionUtils.nullSafeIsEmpty(securityKeyValues))
          {
             ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
@@ -1118,7 +1139,7 @@ public class JoinsContext
          if(useExposedJoins)
          {
             QTableMetaData mainTable = QContext.getQInstance().getTable(mainTableName);
-            for(ExposedJoin exposedJoin : mainTable.getExposedJoins())
+            for(ExposedJoin exposedJoin : CollectionUtils.nonNullList(mainTable.getExposedJoins()))
             {
                if(exposedJoin.getJoinTable().equals(joinTableName))
                {
@@ -1159,6 +1180,7 @@ public class JoinsContext
    }
 
 
+
    /*******************************************************************************
     **
     *******************************************************************************/

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QQueryFilter.java

@@ -25,6 +25,7 @@ package com.kingsrook.qqq.backend.core.model.actions.tables.query;
 import java.io.Serializable;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Collection;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -237,6 +238,28 @@ public class QQueryFilter implements Serializable, Cloneable
 
 
 
+   /*******************************************************************************
+    ** fluent method to add a new criteria
+    *******************************************************************************/
+   public QQueryFilter withCriteria(String fieldName, QCriteriaOperator operator, Collection<? extends Serializable> values)
+   {
+      addCriteria(new QFilterCriteria(fieldName, operator, values));
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** fluent method to add a new criteria
+    *******************************************************************************/
+   public QQueryFilter withCriteria(String fieldName, QCriteriaOperator operator, Serializable... values)
+   {
+      addCriteria(new QFilterCriteria(fieldName, operator, values));
+      return (this);
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/
@@ -505,8 +528,27 @@ public class QQueryFilter implements Serializable, Cloneable
     ** Note - it may be very important that you call this method on a clone of a
     ** QQueryFilter - e.g., if it's one that defined in metaData, and that we don't
     ** want to be (permanently) changed!!
-    *******************************************************************************/
+    **
+    ** This overload does not take in a FilterUseCase - it uses FilterUseCase.DEFAULT
+    ******************************************************************************/
    public void interpretValues(Map<String, Serializable> inputValues) throws QException
+   {
+      interpretValues(inputValues, FilterUseCase.DEFAULT);
+   }
+
+
+
+   /*******************************************************************************
+    ** Replace any criteria values that look like ${input.XXX} with the value of XXX
+    ** from the supplied inputValues map - where the handling of missing values
+    ** is specified in the inputted FilterUseCase parameter
+    **
+    ** Note - it may be very important that you call this method on a clone of a
+    ** QQueryFilter - e.g., if it's one that defined in metaData, and that we don't
+    ** want to be (permanently) changed!!
+    **
+    *******************************************************************************/
+   public void interpretValues(Map<String, Serializable> inputValues, FilterUseCase useCase) throws QException
    {
       List<Exception> caughtExceptions = new ArrayList<>();
 
@@ -522,6 +564,9 @@ public class QQueryFilter implements Serializable, Cloneable
             {
                try
                {
+                  Serializable interpretedValue = value;
+                  Exception    caughtException  = null;
+
                   if(value instanceof AbstractFilterExpression<?>)
                   {
                      ///////////////////////////////////////////////////////////////////////
@@ -530,17 +575,54 @@ public class QQueryFilter implements Serializable, Cloneable
                      ///////////////////////////////////////////////////////////////////////
                      if(value instanceof FilterVariableExpression filterVariableExpression)
                      {
-                        newValues.add(filterVariableExpression.evaluateInputValues(inputValues));
-                     }
-                     else
+                        try
                         {
-                        newValues.add(value);
+                           interpretedValue = filterVariableExpression.evaluateInputValues(inputValues);
+                        }
+                        catch(Exception e)
+                        {
+                           caughtException = e;
+                           interpretedValue = InputNotFound.instance;
+                        }
                      }
                   }
                   else
                   {
+                     /////////////////////////////////////////////////////////////////////////////////////////////////////////
+                     // for non-expressions, cast the value to a string, and see if it can be resolved a variable.          //
+                     // there are 3 possible cases here:                                                                    //
+                     // 1: it doesn't look like a variable, so it just comes back as a string version of whatever went in.  //
+                     // 2: it was resolved from a variable to a value, e.g., ${input.someVar} => someValue                  //
+                     // 3: it looked like a variable, but no value for that variable was present in the interpreter's value //
+                     // map - so we'll get back the InputNotFound.instance.                                                 //
+                     /////////////////////////////////////////////////////////////////////////////////////////////////////////
                      String valueAsString = ValueUtils.getValueAsString(value);
-                     Serializable interpretedValue = variableInterpreter.interpretForObject(valueAsString);
+                     interpretedValue = variableInterpreter.interpretForObject(valueAsString, InputNotFound.instance);
+                  }
+
+                  //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+                  // if interpreting a value returned the not-found value, or an empty string,                                            //
+                  // then decide how to handle the missing value, based on the use-case input                                             //
+                  // Note: questionable, using "" here, but that's what reality is passing a lot for cases we want to treat as missing... //
+                  //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+                  if(interpretedValue == InputNotFound.instance || "".equals(interpretedValue))
+                  {
+                     CriteriaMissingInputValueBehavior missingInputValueBehavior = getMissingInputValueBehavior(useCase);
+
+                     switch(missingInputValueBehavior)
+                     {
+                        case REMOVE_FROM_FILTER -> criterion.setOperator(QCriteriaOperator.TRUE);
+                        case MAKE_NO_MATCHES -> criterion.setOperator(QCriteriaOperator.FALSE);
+                        case INTERPRET_AS_NULL_VALUE -> newValues.add(null);
+
+                        /////////////////////////////////////////////////
+                        // handle case in the default: THROW_EXCEPTION //
+                        /////////////////////////////////////////////////
+                        default -> throw (Objects.requireNonNullElseGet(caughtException, () -> new QUserFacingException("Missing value for criteria on field: " + criterion.getFieldName())));
+                     }
+                  }
+                  else
+                  {
                      newValues.add(interpretedValue);
                   }
                }
@@ -563,6 +645,44 @@ public class QQueryFilter implements Serializable, Cloneable
 
 
 
+   /***************************************************************************
+    ** Note:  in the original build of this, it felt like we *might* want to be
+    ** able to specify these behaviors at the individual criteria level, where
+    ** the implementation would be to add to QFilterCriteria:
+    ** - Map<FilterUseCase, CriteriaMissingInputValueBehavior> missingInputValueBehaviors;
+    ** - CriteriaMissingInputValueBehavior getMissingInputValueBehaviorForUseCase(FilterUseCase useCase) {}
+    *
+    ** (and maybe do that in a sub-class of QFilterCriteria, so it isn't always
+    ** there?  idk...) and then here we'd call:
+    ** - CriteriaMissingInputValueBehavior missingInputValueBehavior = criterion.getMissingInputValueBehaviorForUseCase(useCase);
+    *
+    ** But, we don't actually have that use-case at hand now, so - let's keep it
+    ** just at the level we need for now.
+    **
+    ***************************************************************************/
+   private CriteriaMissingInputValueBehavior getMissingInputValueBehavior(FilterUseCase useCase)
+   {
+      if(useCase == null)
+      {
+         useCase = FilterUseCase.DEFAULT;
+      }
+
+      CriteriaMissingInputValueBehavior missingInputValueBehavior = useCase.getDefaultCriteriaMissingInputValueBehavior();
+      if(missingInputValueBehavior == null)
+      {
+         missingInputValueBehavior = useCase.getDefaultCriteriaMissingInputValueBehavior();
+      }
+
+      if(missingInputValueBehavior == null)
+      {
+         missingInputValueBehavior = FilterUseCase.DEFAULT.getDefaultCriteriaMissingInputValueBehavior();
+      }
+
+      return (missingInputValueBehavior);
+   }
+
+
+
    /*******************************************************************************
     ** Getter for skip
     *******************************************************************************/
@@ -655,4 +775,28 @@ public class QQueryFilter implements Serializable, Cloneable
    {
       return Objects.hash(criteria, orderBys, booleanOperator, subFilters, skip, limit);
    }
+
+
+
+   /***************************************************************************
+    ** "Token" object to be used as the defaultIfLooksLikeVariableButNotFound
+    ** parameter to variableInterpreter.interpretForObject, so we can be
+    ** very clear that we got this default back (e.g., instead of a null,
+    ** which could maybe mean something else?)
+    ***************************************************************************/
+   private static final class InputNotFound implements Serializable
+   {
+      private static InputNotFound instance = new InputNotFound();
+
+
+
+      /*******************************************************************************
+       ** private singleton constructor
+       *******************************************************************************/
+      private InputNotFound()
+      {
+
+      }
+   }
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QueryInput.java

@@ -66,6 +66,14 @@ public class QueryInput extends AbstractTableActionInput implements QueryOrGetIn
    private List<QueryJoin> queryJoins     = null;
    private boolean         selectDistinct = false;
 
+   /////////////////////////////////////////////////////////////////////////////
+   // if this set is null, then the default (all fields) should be included   //
+   // if it's an empty set, that should throw an error                        //
+   // or if there are any fields in it that aren't valid fields on the table, //
+   // or in a selected queryJoin.                                             //
+   /////////////////////////////////////////////////////////////////////////////
+   private Set<String> fieldNamesToInclude;
+
    ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
    // if you say you want to includeAssociations, you can limit which ones by passing them in associationNamesToInclude. //
    // if you leave it null, you get all associations defined on the table.  if you pass it as empty, you get none.       //
@@ -686,4 +694,35 @@ public class QueryInput extends AbstractTableActionInput implements QueryOrGetIn
       return (queryHints.contains(queryHint));
    }
 
+
+
+   /*******************************************************************************
+    ** Getter for fieldNamesToInclude
+    *******************************************************************************/
+   public Set<String> getFieldNamesToInclude()
+   {
+      return (this.fieldNamesToInclude);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for fieldNamesToInclude
+    *******************************************************************************/
+   public void setFieldNamesToInclude(Set<String> fieldNamesToInclude)
+   {
+      this.fieldNamesToInclude = fieldNamesToInclude;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for fieldNamesToInclude
+    *******************************************************************************/
+   public QueryInput withFieldNamesToInclude(Set<String> fieldNamesToInclude)
+   {
+      this.fieldNamesToInclude = fieldNamesToInclude;
+      return (this);
+   }
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QueryJoin.java

@@ -56,7 +56,7 @@ import com.kingsrook.qqq.backend.core.utils.StringUtils;
  ** JoinsContext is constructed before executing a query, and not meant to be set
  ** by users.
  *******************************************************************************/
-public class QueryJoin
+public class QueryJoin implements Cloneable
 {
    private String        baseTableOrAlias;
    private String        joinTable;
@@ -69,6 +69,40 @@ public class QueryJoin
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public QueryJoin clone()
+   {
+      try
+      {
+         QueryJoin clone = (QueryJoin) super.clone();
+
+         if(joinMetaData != null)
+         {
+            clone.joinMetaData = joinMetaData.clone();
+         }
+
+         if(securityCriteria != null)
+         {
+            clone.securityCriteria = new ArrayList<>();
+            for(QFilterCriteria securityCriterion : securityCriteria)
+            {
+               clone.securityCriteria.add(securityCriterion.clone());
+            }
+         }
+
+         return clone;
+      }
+      catch(CloneNotSupportedException e)
+      {
+         throw new AssertionError();
+      }
+   }
+
+
+
    /*******************************************************************************
     ** define the types of joins - INNER, LEFT, RIGHT, or FULL.
     *******************************************************************************/

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/expressions/AbstractFilterExpression.java

@@ -25,6 +25,7 @@ package com.kingsrook.qqq.backend.core.model.actions.tables.query.expressions;
 import java.io.Serializable;
 import java.util.Map;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
 
 
 /*******************************************************************************
@@ -35,16 +36,19 @@ public abstract class AbstractFilterExpression<T extends Serializable> implement
    /*******************************************************************************
     **
     *******************************************************************************/
-   public abstract T evaluate() throws QException;
+   public abstract T evaluate(QFieldMetaData field) throws QException;
 
 
 
    /*******************************************************************************
+    ** Evaluate the expression, given a map of input values.
     **
+    ** By default, this will defer to the evaluate(void) method - but, a subclass
+    ** (e.g., FilterVariableExpression) may react differently.
     *******************************************************************************/
    public T evaluateInputValues(Map<String, Serializable> inputValues) throws QException
    {
-      return (T) this;
+      return evaluate(null);
    }
 
 

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/expressions/FilterVariableExpression.java

@@ -26,6 +26,7 @@ import java.io.Serializable;
 import java.util.Map;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.exceptions.QUserFacingException;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
 import com.kingsrook.qqq.backend.core.utils.ValueUtils;
 
 
@@ -45,7 +46,7 @@ public class FilterVariableExpression extends AbstractFilterExpression<Serializa
     **
     *******************************************************************************/
    @Override
-   public Serializable evaluate() throws QException
+   public Serializable evaluate(QFieldMetaData field) throws QException
    {
       throw (new QUserFacingException("Missing variable value."));
    }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/expressions/Now.java

@@ -22,23 +22,42 @@
 package com.kingsrook.qqq.backend.core.model.actions.tables.query.expressions;
 
 
+import java.io.Serializable;
 import java.time.Instant;
+import java.time.ZoneId;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
+import com.kingsrook.qqq.backend.core.utils.ValueUtils;
 
 
 /*******************************************************************************
  **
  *******************************************************************************/
-public class Now extends AbstractFilterExpression<Instant>
+public class Now extends AbstractFilterExpression<Serializable>
 {
 
    /*******************************************************************************
     **
     *******************************************************************************/
    @Override
-   public Instant evaluate() throws QException
+   public Serializable evaluate(QFieldMetaData field) throws QException
+   {
+      QFieldType type = field == null ? QFieldType.DATE_TIME : field.getType();
+
+      if(type.equals(QFieldType.DATE_TIME))
       {
          return (Instant.now());
       }
+      else if(type.equals(QFieldType.DATE))
+      {
+         ZoneId zoneId = ValueUtils.getSessionOrInstanceZoneId();
+         return (Instant.now().atZone(zoneId).toLocalDate());
+      }
+      else
+      {
+         throw (new QException("Unsupported field type [" + type + "]"));
+      }
+   }
 
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/expressions/NowWithOffset.java

@@ -22,19 +22,24 @@
 package com.kingsrook.qqq.backend.core.model.actions.tables.query.expressions;
 
 
+import java.io.Serializable;
 import java.time.Instant;
+import java.time.LocalDate;
 import java.time.LocalDateTime;
 import java.time.ZoneId;
 import java.time.ZoneOffset;
 import java.time.temporal.ChronoUnit;
 import java.util.concurrent.TimeUnit;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
+import com.kingsrook.qqq.backend.core.utils.ValueUtils;
 
 
 /*******************************************************************************
  **
  *******************************************************************************/
-public class NowWithOffset extends AbstractFilterExpression<Instant>
+public class NowWithOffset extends AbstractFilterExpression<Serializable>
 {
    private Operator   operator;
    private int        amount;
@@ -42,6 +47,9 @@ public class NowWithOffset extends AbstractFilterExpression<Instant>
 
 
 
+   /***************************************************************************
+    **
+    ***************************************************************************/
    public enum Operator
    {PLUS, MINUS}
 
@@ -120,7 +128,30 @@ public class NowWithOffset extends AbstractFilterExpression<Instant>
     **
     *******************************************************************************/
    @Override
-   public Instant evaluate() throws QException
+   public Serializable evaluate(QFieldMetaData field) throws QException
+   {
+      QFieldType type = field == null ? QFieldType.DATE_TIME : field.getType();
+
+      if(type.equals(QFieldType.DATE_TIME))
+      {
+         return (evaluateForDateTime());
+      }
+      else if(type.equals(QFieldType.DATE))
+      {
+         return (evaluateForDate());
+      }
+      else
+      {
+         throw (new QException("Unsupported field type [" + type + "]"));
+      }
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   private Instant evaluateForDateTime()
    {
       /////////////////////////////////////////////////////////////////////////////
       // Instant doesn't let us plus/minus WEEK, MONTH, or YEAR...               //
@@ -144,6 +175,26 @@ public class NowWithOffset extends AbstractFilterExpression<Instant>
 
 
 
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   private LocalDate evaluateForDate()
+   {
+      ZoneId    zoneId = ValueUtils.getSessionOrInstanceZoneId();
+      LocalDate now    = Instant.now().atZone(zoneId).toLocalDate();
+
+      if(operator.equals(Operator.PLUS))
+      {
+         return (now.plus(amount, timeUnit));
+      }
+      else
+      {
+         return (now.minus(amount, timeUnit));
+      }
+   }
+
+
+
    /*******************************************************************************
     ** Getter for operator
     **

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/expressions/ThisOrLastPeriod.java

@@ -22,27 +22,35 @@
 package com.kingsrook.qqq.backend.core.model.actions.tables.query.expressions;
 
 
+import java.io.Serializable;
 import java.time.DayOfWeek;
 import java.time.Instant;
+import java.time.LocalDate;
 import java.time.LocalDateTime;
 import java.time.ZoneId;
 import java.time.temporal.ChronoField;
 import java.time.temporal.ChronoUnit;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.exceptions.QRuntimeException;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
 import com.kingsrook.qqq.backend.core.utils.ValueUtils;
 
 
 /*******************************************************************************
  **
  *******************************************************************************/
-public class ThisOrLastPeriod extends AbstractFilterExpression<Instant>
+public class ThisOrLastPeriod extends AbstractFilterExpression<Serializable>
 {
    private Operator   operator;
    private ChronoUnit timeUnit;
 
 
 
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
    public enum Operator
    {THIS, LAST}
 
@@ -85,7 +93,7 @@ public class ThisOrLastPeriod extends AbstractFilterExpression<Instant>
     ** Factory
     **
     *******************************************************************************/
-   public static ThisOrLastPeriod last(int amount, ChronoUnit timeUnit)
+   public static ThisOrLastPeriod last(ChronoUnit timeUnit)
    {
       return (new ThisOrLastPeriod(Operator.LAST, timeUnit));
    }
@@ -96,7 +104,31 @@ public class ThisOrLastPeriod extends AbstractFilterExpression<Instant>
     **
     *******************************************************************************/
    @Override
-   public Instant evaluate() throws QException
+   public Serializable evaluate(QFieldMetaData field) throws QException
+   {
+      QFieldType type = field == null ? QFieldType.DATE_TIME : field.getType();
+
+      if(type.equals(QFieldType.DATE_TIME))
+      {
+         return (evaluateForDateTime());
+      }
+      else if(type.equals(QFieldType.DATE))
+      {
+         // return (evaluateForDateTime());
+         return (evaluateForDate());
+      }
+      else
+      {
+         throw (new QException("Unsupported field type [" + type + "]"));
+      }
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   private Instant evaluateForDateTime()
    {
       ZoneId zoneId = ValueUtils.getSessionOrInstanceZoneId();
 
@@ -151,7 +183,57 @@ public class ThisOrLastPeriod extends AbstractFilterExpression<Instant>
 
             return operator.equals(Operator.THIS) ? startOfThisYear : startOfLastYear;
          }
-         default -> throw (new QRuntimeException("Unsupported timeUnit: " + timeUnit));
+         default -> throw (new QRuntimeException("Unsupported unit: " + timeUnit));
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public LocalDate evaluateForDate()
+   {
+      ZoneId    zoneId = ValueUtils.getSessionOrInstanceZoneId();
+      LocalDate today  = Instant.now().atZone(zoneId).toLocalDate();
+
+      switch(timeUnit)
+      {
+         case DAYS ->
+         {
+            return operator.equals(Operator.THIS) ? today : today.minusDays(1);
+         }
+         case WEEKS ->
+         {
+            LocalDate startOfThisWeek = today;
+            while(startOfThisWeek.get(ChronoField.DAY_OF_WEEK) != DayOfWeek.SUNDAY.getValue())
+            {
+               ////////////////////////////////////////
+               // go backwards until sunday is found //
+               ////////////////////////////////////////
+               startOfThisWeek = startOfThisWeek.minusDays(1);
+            }
+            return operator.equals(Operator.THIS) ? startOfThisWeek : startOfThisWeek.minusDays(7);
+         }
+         case MONTHS ->
+         {
+            Instant       startOfThisMonth    = ValueUtils.getStartOfMonthInZoneId(zoneId.getId());
+            LocalDateTime startOfThisMonthLDT = LocalDateTime.ofInstant(startOfThisMonth, ZoneId.of(zoneId.getId()));
+            LocalDateTime startOfLastMonthLDT = startOfThisMonthLDT.minusMonths(1);
+            Instant       startOfLastMonth    = startOfLastMonthLDT.toInstant(ZoneId.of(zoneId.getId()).getRules().getOffset(Instant.now()));
+
+            return (operator.equals(Operator.THIS) ? startOfThisMonth : startOfLastMonth).atZone(zoneId).toLocalDate();
+         }
+         case YEARS ->
+         {
+            Instant       startOfThisYear    = ValueUtils.getStartOfYearInZoneId(zoneId.getId());
+            LocalDateTime startOfThisYearLDT = LocalDateTime.ofInstant(startOfThisYear, zoneId);
+            LocalDateTime startOfLastYearLDT = startOfThisYearLDT.minusYears(1);
+            Instant       startOfLastYear    = startOfLastYearLDT.toInstant(zoneId.getRules().getOffset(Instant.now()));
+
+            return (operator.equals(Operator.THIS) ? startOfThisYear : startOfLastYear).atZone(zoneId).toLocalDate();
+         }
+         default -> throw (new QRuntimeException("Unsupported unit: " + timeUnit));
       }
    }
 

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/serialization/QFilterCriteriaDeserializer.java

@@ -79,6 +79,7 @@ public class QFilterCriteriaDeserializer extends StdDeserializer<QFilterCriteria
       /////////////////////////////////
       // get values out of json node //
       /////////////////////////////////
+      @SuppressWarnings("unchecked")
       List<Serializable> values = objectMapper.treeToValue(node.get("values"), List.class);
       String            fieldName      = objectMapper.treeToValue(node.get("fieldName"), String.class);
       QCriteriaOperator operator       = objectMapper.treeToValue(node.get("operator"), QCriteriaOperator.class);

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/storage/StorageInput.java

@@ -31,6 +31,7 @@ import com.kingsrook.qqq.backend.core.model.actions.AbstractTableActionInput;
 public class StorageInput extends AbstractTableActionInput
 {
    private String reference;
+   private String contentType;
 
 
 
@@ -74,4 +75,35 @@ public class StorageInput extends AbstractTableActionInput
       return (this);
    }
 
+
+
+   /*******************************************************************************
+    ** Getter for contentType
+    *******************************************************************************/
+   public String getContentType()
+   {
+      return (this.contentType);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for contentType
+    *******************************************************************************/
+   public void setContentType(String contentType)
+   {
+      this.contentType = contentType;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for contentType
+    *******************************************************************************/
+   public StorageInput withContentType(String contentType)
+   {
+      this.contentType = contentType;
+      return (this);
+   }
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/audits/AuditsMetaDataProvider.java

@@ -220,7 +220,7 @@ public class AuditsMetaDataProvider
          .withRecordLabelFormat("%s %s")
          .withRecordLabelFields("auditTableId", "recordId")
          .withPrimaryKeyField("id")
-         .withField(new QFieldMetaData("id", QFieldType.INTEGER))
+         .withField(new QFieldMetaData("id", QFieldType.LONG))
          .withField(new QFieldMetaData("auditTableId", QFieldType.INTEGER).withPossibleValueSourceName(TABLE_NAME_AUDIT_TABLE))
          .withField(new QFieldMetaData("auditUserId", QFieldType.INTEGER).withPossibleValueSourceName(TABLE_NAME_AUDIT_USER))
          .withField(new QFieldMetaData("recordId", QFieldType.INTEGER))
@@ -243,8 +243,8 @@ public class AuditsMetaDataProvider
          .withRecordLabelFormat("%s")
          .withRecordLabelFields("id")
          .withPrimaryKeyField("id")
-         .withField(new QFieldMetaData("id", QFieldType.INTEGER))
-         .withField(new QFieldMetaData("auditId", QFieldType.INTEGER).withPossibleValueSourceName(TABLE_NAME_AUDIT))
+         .withField(new QFieldMetaData("id", QFieldType.LONG))
+         .withField(new QFieldMetaData("auditId", QFieldType.LONG).withPossibleValueSourceName(TABLE_NAME_AUDIT))
          .withField(new QFieldMetaData("message", QFieldType.STRING).withMaxLength(250).withBehavior(ValueTooLongBehavior.TRUNCATE_ELLIPSIS))
          .withField(new QFieldMetaData("fieldName", QFieldType.STRING).withMaxLength(100).withBehavior(ValueTooLongBehavior.TRUNCATE_ELLIPSIS))
          .withField(new QFieldMetaData("oldValue", QFieldType.STRING).withMaxLength(250).withBehavior(ValueTooLongBehavior.TRUNCATE_ELLIPSIS))

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/dashboard/widgets/AlertData.java

@@ -28,6 +28,9 @@ package com.kingsrook.qqq.backend.core.model.dashboard.widgets;
  *******************************************************************************/
 public class AlertData extends QWidgetData
 {
+   /***************************************************************************
+    **
+    ***************************************************************************/
    public enum AlertType
    {
       ERROR,

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/dashboard/widgets/CompositeWidgetData.java

@@ -40,9 +40,10 @@ public class CompositeWidgetData extends AbstractBlockWidgetData<CompositeWidget
 {
    private List<AbstractBlockWidgetData<?, ?, ?, ?>> blocks = new ArrayList<>();
 
-   private Map<String, Serializable> styleOverrides = new HashMap<>();
-
    private Layout                    layout;
+   private Map<String, Serializable> styleOverrides        = new HashMap<>();
+   private String                    overlayHtml;
+   private Map<String, Serializable> overlayStyleOverrides = new HashMap<>();
 
 
 
@@ -218,4 +219,91 @@ public class CompositeWidgetData extends AbstractBlockWidgetData<CompositeWidget
       return (this);
    }
 
+
+
+   /*******************************************************************************
+    ** Getter for overlayHtml
+    *******************************************************************************/
+   public String getOverlayHtml()
+   {
+      return (this.overlayHtml);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for overlayHtml
+    *******************************************************************************/
+   public void setOverlayHtml(String overlayHtml)
+   {
+      this.overlayHtml = overlayHtml;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for overlayHtml
+    *******************************************************************************/
+   public CompositeWidgetData withOverlayHtml(String overlayHtml)
+   {
+      this.overlayHtml = overlayHtml;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for overlayStyleOverrides
+    *******************************************************************************/
+   public Map<String, Serializable> getOverlayStyleOverrides()
+   {
+      return (this.overlayStyleOverrides);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for overlayStyleOverrides
+    *******************************************************************************/
+   public void setOverlayStyleOverrides(Map<String, Serializable> overlayStyleOverrides)
+   {
+      this.overlayStyleOverrides = overlayStyleOverrides;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for overlayStyleOverrides
+    *******************************************************************************/
+   public CompositeWidgetData withOverlayStyleOverrides(Map<String, Serializable> overlayStyleOverrides)
+   {
+      this.overlayStyleOverrides = overlayStyleOverrides;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public CompositeWidgetData withOverlayStyleOverride(String key, Serializable value)
+   {
+      addOverlayStyleOverride(key, value);
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public void addOverlayStyleOverride(String key, Serializable value)
+   {
+      if(this.overlayStyleOverrides == null)
+      {
+         this.overlayStyleOverrides = new HashMap<>();
+      }
+      this.overlayStyleOverrides.put(key, value);
+   }
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/dashboard/widgets/blocks/AbstractBlockWidgetData.java

@@ -24,6 +24,7 @@ package com.kingsrook.qqq.backend.core.model.dashboard.widgets.blocks;
 
 import java.util.HashMap;
 import java.util.Map;
+import com.kingsrook.qqq.backend.core.model.dashboard.widgets.CompositeWidgetData;
 import com.kingsrook.qqq.backend.core.model.dashboard.widgets.QWidgetData;
 
 
@@ -74,6 +75,7 @@ public abstract class AbstractBlockWidgetData<
    /*******************************************************************************
     **
     *******************************************************************************/
+   @SuppressWarnings("unchecked")
    public T withTooltip(S key, String value)
    {
       addTooltip(key, value);
@@ -99,6 +101,7 @@ public abstract class AbstractBlockWidgetData<
    /*******************************************************************************
     **
     *******************************************************************************/
+   @SuppressWarnings("unchecked")
    public T withTooltip(S key, BlockTooltip value)
    {
       addTooltip(key, value);
@@ -144,6 +147,7 @@ public abstract class AbstractBlockWidgetData<
    /*******************************************************************************
     ** Fluent setter for tooltipMap
     *******************************************************************************/
+   @SuppressWarnings("unchecked")
    public T withTooltipMap(Map<S, BlockTooltip> tooltipMap)
    {
       this.tooltipMap = tooltipMap;
@@ -178,6 +182,7 @@ public abstract class AbstractBlockWidgetData<
     ** Fluent setter for tooltip
     **
     *******************************************************************************/
+   @SuppressWarnings("unchecked")
    public T withTooltip(String tooltip)
    {
       this.tooltip = new BlockTooltip(tooltip);
@@ -190,6 +195,7 @@ public abstract class AbstractBlockWidgetData<
     ** Fluent setter for tooltip
     **
     *******************************************************************************/
+   @SuppressWarnings("unchecked")
    public T withTooltip(BlockTooltip tooltip)
    {
       this.tooltip = tooltip;
@@ -199,8 +205,22 @@ public abstract class AbstractBlockWidgetData<
 
 
    /*******************************************************************************
+    ** Fluent setter for tooltip
     **
     *******************************************************************************/
+   @SuppressWarnings("unchecked")
+   public T withTooltip(CompositeWidgetData data)
+   {
+      this.tooltip = new BlockTooltip(data);
+      return (T) (this);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @SuppressWarnings("unchecked")
    public T withLink(S key, String value)
    {
       addLink(key, value);
@@ -226,6 +246,7 @@ public abstract class AbstractBlockWidgetData<
    /*******************************************************************************
     **
     *******************************************************************************/
+   @SuppressWarnings("unchecked")
    public T withLink(S key, BlockLink value)
    {
       addLink(key, value);
@@ -271,6 +292,7 @@ public abstract class AbstractBlockWidgetData<
    /*******************************************************************************
     ** Fluent setter for linkMap
     *******************************************************************************/
+   @SuppressWarnings("unchecked")
    public T withLinkMap(Map<S, BlockLink> linkMap)
    {
       this.linkMap = linkMap;
@@ -305,6 +327,7 @@ public abstract class AbstractBlockWidgetData<
     ** Fluent setter for link
     **
     *******************************************************************************/
+   @SuppressWarnings("unchecked")
    public T withLink(String link)
    {
       this.link = new BlockLink(link);
@@ -317,6 +340,7 @@ public abstract class AbstractBlockWidgetData<
     ** Fluent setter for link
     **
     *******************************************************************************/
+   @SuppressWarnings("unchecked")
    public T withLink(BlockLink link)
    {
       this.link = link;
@@ -348,6 +372,7 @@ public abstract class AbstractBlockWidgetData<
    /*******************************************************************************
     ** Fluent setter for values
     *******************************************************************************/
+   @SuppressWarnings("unchecked")
    public T withValues(V values)
    {
       this.values = values;
@@ -379,6 +404,7 @@ public abstract class AbstractBlockWidgetData<
    /*******************************************************************************
     ** Fluent setter for styles
     *******************************************************************************/
+   @SuppressWarnings("unchecked")
    public T withStyles(SX styles)
    {
       this.styles = styles;
@@ -386,6 +412,7 @@ public abstract class AbstractBlockWidgetData<
    }
 
 
+
    /*******************************************************************************
     ** Getter for blockId
     *******************************************************************************/
@@ -409,11 +436,11 @@ public abstract class AbstractBlockWidgetData<
    /*******************************************************************************
     ** Fluent setter for blockId
     *******************************************************************************/
+   @SuppressWarnings("unchecked")
    public T withBlockId(String blockId)
    {
       this.blockId = blockId;
       return (T) this;
    }
 
-
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/dashboard/widgets/blocks/BlockTooltip.java

@@ -22,17 +22,24 @@
 package com.kingsrook.qqq.backend.core.model.dashboard.widgets.blocks;
 
 
+import com.kingsrook.qqq.backend.core.model.dashboard.widgets.CompositeWidgetData;
+
+
 /*******************************************************************************
  ** A tooltip used within a (widget) block.
  **
  *******************************************************************************/
 public class BlockTooltip
 {
+   private CompositeWidgetData blockData;
    private String              title;
    private Placement           placement = Placement.BOTTOM;
 
 
 
+   /***************************************************************************
+    **
+    ***************************************************************************/
    public enum Placement
    {BOTTOM, LEFT, RIGHT, TOP}
 
@@ -59,6 +66,17 @@ public class BlockTooltip
 
 
 
+   /*******************************************************************************
+    ** Constructor
+    **
+    *******************************************************************************/
+   public BlockTooltip(CompositeWidgetData blockData)
+   {
+      this.blockData = blockData;
+   }
+
+
+
    /*******************************************************************************
     ** Getter for title
     *******************************************************************************/
@@ -119,4 +137,35 @@ public class BlockTooltip
       return (this);
    }
 
+
+
+   /*******************************************************************************
+    ** Getter for blockData
+    *******************************************************************************/
+   public CompositeWidgetData getBlockData()
+   {
+      return (this.blockData);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for blockData
+    *******************************************************************************/
+   public void setBlockData(CompositeWidgetData blockData)
+   {
+      this.blockData = blockData;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for blockData
+    *******************************************************************************/
+   public BlockTooltip withBlockData(CompositeWidgetData blockData)
+   {
+      this.blockData = blockData;
+      return (this);
+   }
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/data/QRecordEntity.java

@@ -192,6 +192,7 @@ public abstract class QRecordEntity
 
          for(QRecordEntityAssociation qRecordEntityAssociation : getAssociationList(this.getClass()))
          {
+            @SuppressWarnings("unchecked")
             List<? extends QRecordEntity> associatedEntities = (List<? extends QRecordEntity>) qRecordEntityAssociation.getGetter().invoke(this);
             String                        associationName    = qRecordEntityAssociation.getAssociationAnnotation().name();
 
@@ -245,6 +246,7 @@ public abstract class QRecordEntity
 
          for(QRecordEntityAssociation qRecordEntityAssociation : getAssociationList(this.getClass()))
          {
+            @SuppressWarnings("unchecked")
             List<? extends QRecordEntity> associatedEntities = (List<? extends QRecordEntity>) qRecordEntityAssociation.getGetter().invoke(this);
             String                        associationName    = qRecordEntityAssociation.getAssociationAnnotation().name();
 
@@ -346,6 +348,7 @@ public abstract class QRecordEntity
 
                   if(associationAnnotation.isPresent())
                   {
+                     @SuppressWarnings("unchecked")
                      Class<? extends QRecordEntity> listTypeParam = (Class<? extends QRecordEntity>) getListTypeParam(possibleGetter.getReturnType(), possibleGetter.getAnnotatedReturnType());
                      associationList.add(new QRecordEntityAssociation(fieldName, possibleGetter, setter.get(), listTypeParam, associationAnnotation.orElse(null)));
                   }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/QBackendMetaData.java

@@ -53,6 +53,8 @@ public class QBackendMetaData implements TopLevelMetaDataInterface
    private String  variantOptionsTableUsernameField;
    private String  variantOptionsTablePasswordField;
    private String  variantOptionsTableApiKeyField;
+   private String  variantOptionsTableClientIdField;
+   private String  variantOptionsTableClientSecretField;
    private String  variantOptionsTableName;
 
    // todo - at some point, we may want to apply this to secret properties on subclasses?
@@ -648,4 +650,66 @@ public class QBackendMetaData implements TopLevelMetaDataInterface
    {
       qInstance.addBackend(this);
    }
+
+   /*******************************************************************************
+    ** Getter for variantOptionsTableClientIdField
+    *******************************************************************************/
+   public String getVariantOptionsTableClientIdField()
+   {
+      return (this.variantOptionsTableClientIdField);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for variantOptionsTableClientIdField
+    *******************************************************************************/
+   public void setVariantOptionsTableClientIdField(String variantOptionsTableClientIdField)
+   {
+      this.variantOptionsTableClientIdField = variantOptionsTableClientIdField;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for variantOptionsTableClientIdField
+    *******************************************************************************/
+   public QBackendMetaData withVariantOptionsTableClientIdField(String variantOptionsTableClientIdField)
+   {
+      this.variantOptionsTableClientIdField = variantOptionsTableClientIdField;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for variantOptionsTableClientSecretField
+    *******************************************************************************/
+   public String getVariantOptionsTableClientSecretField()
+   {
+      return (this.variantOptionsTableClientSecretField);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for variantOptionsTableClientSecretField
+    *******************************************************************************/
+   public void setVariantOptionsTableClientSecretField(String variantOptionsTableClientSecretField)
+   {
+      this.variantOptionsTableClientSecretField = variantOptionsTableClientSecretField;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for variantOptionsTableClientSecretField
+    *******************************************************************************/
+   public QBackendMetaData withVariantOptionsTableClientSecretField(String variantOptionsTableClientSecretField)
+   {
+      this.variantOptionsTableClientSecretField = variantOptionsTableClientSecretField;
+      return (this);
+   }
+
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/dashboard/ParentWidgetMetaData.java

@@ -39,6 +39,9 @@ public class ParentWidgetMetaData extends QWidgetMetaData
 
 
 
+   /***************************************************************************
+    **
+    ***************************************************************************/
    public enum LayoutType
    {
       GRID,

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/FieldAdornment.java

@@ -188,7 +188,8 @@ public class FieldAdornment
     ** Fluent setter for values
     **
     *******************************************************************************/
-   public FieldAdornment withValues(Pair<String, Serializable>... values)
+   @SafeVarargs
+   public final FieldAdornment withValues(Pair<String, Serializable>... values)
    {
       for(Pair<String, Serializable> value : values)
       {

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/frontend/QFrontendTableMetaData.java

@@ -168,11 +168,11 @@ public class QFrontendTableMetaData
       editPermission = PermissionsHelper.hasTablePermission(actionInput, tableMetaData.getName(), TablePermissionSubType.EDIT);
       deletePermission = PermissionsHelper.hasTablePermission(actionInput, tableMetaData.getName(), TablePermissionSubType.DELETE);
 
-      QBackendMetaData backend = actionInput.getInstance().getBackend(tableMetaData.getBackendName());
+      QBackendMetaData backend = QContext.getQInstance().getBackend(tableMetaData.getBackendName());
       if(backend != null && backend.getUsesVariants())
       {
          usesVariants = true;
-         variantTableLabel = actionInput.getInstance().getTable(backend.getVariantOptionsTableName()).getLabel();
+         variantTableLabel = QContext.getQInstance().getTable(backend.getVariantOptionsTableName()).getLabel();
       }
 
       this.helpContents = tableMetaData.getHelpContent();

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/joins/JoinOn.java

@@ -26,7 +26,7 @@ package com.kingsrook.qqq.backend.core.model.metadata.joins;
  ** Specification for (at least part of) how two tables join together - e.g.,
  ** leftField = rightField.  Used as part of a list in a QJoinMetaData.
  *******************************************************************************/
-public class JoinOn
+public class JoinOn implements Cloneable
 {
    private String leftField;
    private String rightField;
@@ -131,4 +131,22 @@ public class JoinOn
       return (this);
    }
 
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public JoinOn clone()
+   {
+      try
+      {
+         JoinOn clone = (JoinOn) super.clone();
+         return clone;
+      }
+      catch(CloneNotSupportedException e)
+      {
+         throw new AssertionError();
+      }
+   }
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/joins/QJoinMetaData.java

@@ -33,7 +33,7 @@ import com.kingsrook.qqq.backend.core.utils.StringUtils;
 /*******************************************************************************
  ** Definition of how 2 tables join together within a QQQ Instance.
  *******************************************************************************/
-public class QJoinMetaData implements TopLevelMetaDataInterface
+public class QJoinMetaData implements TopLevelMetaDataInterface, Cloneable
 {
    private String   name;
    private JoinType type;
@@ -62,6 +62,44 @@ public class QJoinMetaData implements TopLevelMetaDataInterface
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public QJoinMetaData clone()
+   {
+      try
+      {
+         QJoinMetaData clone = (QJoinMetaData) super.clone();
+
+         if(joinOns != null)
+         {
+            clone.joinOns = new ArrayList<>();
+            for(JoinOn joinOn : joinOns)
+            {
+               clone.joinOns.add(joinOn.clone());
+            }
+         }
+
+         if(orderBys != null)
+         {
+            clone.orderBys = new ArrayList<>();
+            for(QFilterOrderBy orderBy : orderBys)
+            {
+               clone.orderBys.add(orderBy.clone());
+            }
+         }
+
+         return clone;
+      }
+      catch(CloneNotSupportedException e)
+      {
+         throw new AssertionError();
+      }
+   }
+
+
+
    /*******************************************************************************
     ** Getter for name
     **

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/possiblevalues/PossibleValueSearchFilterUseCase.java

@@ -0,0 +1,72 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.model.metadata.possiblevalues;
+
+
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.CriteriaMissingInputValueBehavior;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.FilterUseCase;
+
+
+/*******************************************************************************
+ ** FilterUseCase implementation for the ways that possible value searches
+ ** are performed, and where we want to have different behaviors for criteria
+ ** that are missing an input value.  That is, either for a:
+ **
+ ** - FORM - e.g., creating a new record, or in a process - where we want a
+ ** missing filter value to basically block you from selecting a value in the
+ ** PVS field - e.g., you must enter some other foreign-key value before choosing
+ ** from this possible value - at least that's the use-case we know of now.
+ **
+ ** - FILTER - e.g., a query screen - where there isn't really quite the same
+ ** scenario of choosing that foreign-key value first - so, such a PVS should
+ ** list all its values (e.g., a criteria missing an input value should be
+ ** removed from the filter).
+ *******************************************************************************/
+public enum PossibleValueSearchFilterUseCase implements FilterUseCase
+{
+   FORM(CriteriaMissingInputValueBehavior.MAKE_NO_MATCHES),
+   FILTER(CriteriaMissingInputValueBehavior.REMOVE_FROM_FILTER);
+
+
+   private final CriteriaMissingInputValueBehavior defaultCriteriaMissingInputValueBehavior;
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   PossibleValueSearchFilterUseCase(CriteriaMissingInputValueBehavior defaultCriteriaMissingInputValueBehavior)
+   {
+      this.defaultCriteriaMissingInputValueBehavior = defaultCriteriaMissingInputValueBehavior;
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for defaultCriteriaMissingInputValueBehavior
+    **
+    *******************************************************************************/
+   public CriteriaMissingInputValueBehavior getDefaultCriteriaMissingInputValueBehavior()
+   {
+      return defaultCriteriaMissingInputValueBehavior;
+   }
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/reporting/QReportDataSource.java

@@ -32,6 +32,13 @@ import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
 /*******************************************************************************
  ** Meta-data definition of a source of data for a report (e.g., a table and query
  ** filter or custom-code reference).
+ **
+ ** Runs in 3 modes:
+ **
+ ** - If a customRecordSource is specified, then that code is executed to get the records.
+ ** - else, if a sourceTable is specified, then the corresponding queryFilter
+ ** (optionally along with queryJoins and queryInputCustomizer) is used.
+ ** - else a staticDataSupplier is used.
  *******************************************************************************/
 public class QReportDataSource
 {
@@ -44,6 +51,7 @@ public class QReportDataSource
 
    private QCodeReference queryInputCustomizer;
    private QCodeReference staticDataSupplier;
+   private QCodeReference customRecordSource;
 
 
 
@@ -265,4 +273,35 @@ public class QReportDataSource
       return (this);
    }
 
+
+   /*******************************************************************************
+    ** Getter for customRecordSource
+    *******************************************************************************/
+   public QCodeReference getCustomRecordSource()
+   {
+      return (this.customRecordSource);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for customRecordSource
+    *******************************************************************************/
+   public void setCustomRecordSource(QCodeReference customRecordSource)
+   {
+      this.customRecordSource = customRecordSource;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for customRecordSource
+    *******************************************************************************/
+   public QReportDataSource withCustomRecordSource(QCodeReference customRecordSource)
+   {
+      this.customRecordSource = customRecordSource;
+      return (this);
+   }
+
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/QTableMetaData.java

@@ -804,7 +804,7 @@ public class QTableMetaData implements QAppChildMetaData, Serializable, MetaData
    {
       if(this.associatedScripts == null)
       {
-         this.associatedScripts = new ArrayList();
+         this.associatedScripts = new ArrayList<>();
       }
       this.associatedScripts.add(associatedScript);
       return (this);

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/cache/CacheUseCase.java

@@ -32,6 +32,9 @@ import com.kingsrook.qqq.backend.core.model.metadata.tables.UniqueKey;
  *******************************************************************************/
 public class CacheUseCase
 {
+   /***************************************************************************
+    **
+    ***************************************************************************/
    public enum Type
    {
       PRIMARY_KEY_TO_PRIMARY_KEY, // e.g., the primary key in the cache table equals the primary key in the source table.

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/authentication/implementations/Auth0AuthenticationModule.java

@@ -247,10 +247,7 @@ public class Auth0AuthenticationModule implements QAuthenticationModuleInterface
             //////////////////////////////////////////////////////////////
             // allow customizer to do custom things here, if so desired //
             //////////////////////////////////////////////////////////////
-            if(getCustomizer() != null)
-            {
-               getCustomizer().finalCustomizeSession(qInstance, qSession);
-            }
+            finalCustomizeSession(qInstance, qSession);
 
             return (qSession);
          }
@@ -311,10 +308,7 @@ public class Auth0AuthenticationModule implements QAuthenticationModuleInterface
          //////////////////////////////////////////////////////////////
          // allow customizer to do custom things here, if so desired //
          //////////////////////////////////////////////////////////////
-         if(getCustomizer() != null)
-         {
-            getCustomizer().finalCustomizeSession(qInstance, qSession);
-         }
+         finalCustomizeSession(qInstance, qSession);
 
          return (qSession);
       }
@@ -360,6 +354,23 @@ public class Auth0AuthenticationModule implements QAuthenticationModuleInterface
 
 
 
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   private void finalCustomizeSession(QInstance qInstance, QSession qSession)
+   {
+      if(getCustomizer() != null)
+      {
+         QContext.withTemporaryContext(QContext.capture(), () ->
+         {
+            QContext.setQSession(getChickenAndEggSession());
+            getCustomizer().finalCustomizeSession(qInstance, qSession);
+         });
+      }
+   }
+
+
+
    /*******************************************************************************
     ** Insert a session as a new record into userSession table
     *******************************************************************************/

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/memory/MemoryRecordStore.java

@@ -58,6 +58,7 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.JoinsContext;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryJoin;
 import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
@@ -170,6 +171,8 @@ public class MemoryRecordStore
       Collection<QRecord> tableData = getTableData(input.getTable()).values();
       List<QRecord>       records   = new ArrayList<>();
 
+      QQueryFilter filter       = clonedOrNewFilter(input.getFilter());
+      JoinsContext joinsContext = new JoinsContext(QContext.getQInstance(), input.getTableName(), input.getQueryJoins(), filter);
       if(CollectionUtils.nullSafeHasContents(input.getQueryJoins()))
       {
          tableData = buildJoinCrossProduct(input);
@@ -185,7 +188,7 @@ public class MemoryRecordStore
             qRecord.setTableName(input.getTableName());
          }
 
-         boolean recordMatches = BackendQueryFilterUtils.doesRecordMatch(input.getFilter(), qRecord);
+         boolean recordMatches = BackendQueryFilterUtils.doesRecordMatch(input.getFilter(), joinsContext, qRecord);
 
          if(recordMatches)
          {
@@ -225,7 +228,6 @@ public class MemoryRecordStore
    private Collection<QRecord> buildJoinCrossProduct(QueryInput input) throws QException
    {
       QInstance qInstance = QContext.getQInstance();
-      JoinsContext joinsContext = new JoinsContext(qInstance, input.getTableName(), input.getQueryJoins(), input.getFilter());
 
       List<QRecord>  crossProduct = new ArrayList<>();
       QTableMetaData leftTable    = input.getTable();
@@ -372,9 +374,16 @@ public class MemoryRecordStore
          // set the next serial in the record if needed //
          /////////////////////////////////////////////////
          if(recordToInsert.getValue(primaryKeyField.getName()) == null && (primaryKeyField.getType().equals(QFieldType.INTEGER) || primaryKeyField.getType().equals(QFieldType.LONG)))
+         {
+            if(primaryKeyField.getType().equals(QFieldType.LONG))
+            {
+               recordToInsert.setValue(primaryKeyField.getName(), (nextSerial++).longValue());
+            }
+            else
             {
                recordToInsert.setValue(primaryKeyField.getName(), nextSerial++);
             }
+         }
 
          ///////////////////////////////////////////////////////////////////////////////////////////////////
          // make sure that if the user supplied a serial, greater than the one we had, that we skip ahead //
@@ -383,7 +392,7 @@ public class MemoryRecordStore
          {
             nextSerial = recordToInsert.getValueInteger(primaryKeyField.getName()) + 1;
          }
-         else if(primaryKeyField.getType().equals(QFieldType.LONG) && recordToInsert.getValueLong(primaryKeyField.getName()) > nextSerial)
+         else if(primaryKeyField.getType().equals(QFieldType.LONG) && recordToInsert.getValueInteger(primaryKeyField.getName()) > nextSerial)
          {
             //////////////////////////////////////
             // todo - mmm, could overflow here? //
@@ -662,7 +671,11 @@ public class MemoryRecordStore
             {
                return (-1);
             }
-            return ((Comparable) a).compareTo(b);
+
+            @SuppressWarnings("unchecked")
+            Comparable<Serializable> comparableSerializableA = (Comparable<Serializable>) a;
+
+            return comparableSerializableA.compareTo(b);
          };
 
          //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
@@ -769,6 +782,7 @@ public class MemoryRecordStore
    /*******************************************************************************
     **
     *******************************************************************************/
+   @SuppressWarnings({ "rawtypes", "unchecked" })
    private static Serializable computeAggregate(List<QRecord> records, Aggregate aggregate, QTableMetaData table)
    {
       String            fieldName = aggregate.getFieldName();
@@ -896,4 +910,21 @@ public class MemoryRecordStore
 
       return ValueUtils.getValueAsFieldType(fieldType, aggregateValue);
    }
+
+
+
+   /*******************************************************************************
+    ** Either clone the input filter (so we can change it safely), or return a new blank filter.
+    *******************************************************************************/
+   protected QQueryFilter clonedOrNewFilter(QQueryFilter filter)
+   {
+      if(filter == null)
+      {
+         return (new QQueryFilter());
+      }
+      else
+      {
+         return (filter.clone());
+      }
+   }
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/utils/BackendQueryFilterUtils.java

@@ -34,14 +34,18 @@ import java.util.Objects;
 import java.util.concurrent.atomic.AtomicBoolean;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.logging.QLogger;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.JoinsContext;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.expressions.AbstractFilterExpression;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
 import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
 import com.kingsrook.qqq.backend.core.utils.ValueUtils;
 import org.apache.commons.lang.NotImplementedException;
+import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
 
 
 /*******************************************************************************
@@ -58,8 +62,22 @@ public class BackendQueryFilterUtils
 
    /*******************************************************************************
     ** Test if record matches filter.
-    *******************************************************************************/
+    ******************************************************************************/
    public static boolean doesRecordMatch(QQueryFilter filter, QRecord qRecord)
+   {
+      return doesRecordMatch(filter, null, qRecord);
+   }
+
+
+
+   /*******************************************************************************
+    ** Test if record matches filter - where we are executing a QueryAction, and
+    ** we have a JoinsContext.  Note, if you don't have one of those, you can call
+    ** the overload of this method that doesn't take one, and everything downstream
+    ** /should/ be tolerant of that being absent...  You just might not have the
+    ** benefit of things like knowing field-meta-data associated with criteria...
+    *******************************************************************************/
+   public static boolean doesRecordMatch(QQueryFilter filter, JoinsContext joinsContext, QRecord qRecord)
    {
       if(filter == null || !filter.hasAnyCriteria())
       {
@@ -97,7 +115,36 @@ public class BackendQueryFilterUtils
             }
          }
 
-         boolean criterionMatches = doesCriteriaMatch(criterion, fieldName, value);
+         ///////////////////////////////////////////////////////////////////////////////////////////////
+         // Test if this criteria(on) matches the record.                                             //
+         // As criteria have become more sophisticated over time, we would like to be able to know    //
+         // what field they are for. In general, we'll try to get that from the query's JoinsContext. //
+         // But, in some scenarios, that isn't available - so - be safe and defer to simpler methods  //
+         // that might not have the full field, when necessary.                                       //
+         ///////////////////////////////////////////////////////////////////////////////////////////////
+         Boolean criterionMatches = null;
+         if(joinsContext != null)
+         {
+            JoinsContext.FieldAndTableNameOrAlias fieldAndTableNameOrAlias = null;
+            try
+            {
+               fieldAndTableNameOrAlias = joinsContext.getFieldAndTableNameOrAlias(criterion.getFieldName());
+            }
+            catch(Exception e)
+            {
+               LOG.debug("Exception getting field from joinsContext", e, logPair("fieldName", criterion.getFieldName()));
+            }
+
+            if(fieldAndTableNameOrAlias != null)
+            {
+               criterionMatches = doesCriteriaMatch(criterion, fieldAndTableNameOrAlias.field(), value);
+            }
+         }
+
+         if(criterionMatches == null)
+         {
+            criterionMatches = doesCriteriaMatch(criterion, criterion.getFieldName(), value);
+         }
 
          ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
          // add this new value to the existing recordMatches value - and if we can short circuit the remaining checks, do so. //
@@ -131,11 +178,24 @@ public class BackendQueryFilterUtils
 
 
 
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   public static boolean doesCriteriaMatch(QFilterCriteria criterion, String fieldName, Serializable value)
+   {
+      QFieldMetaData field = new QFieldMetaData(fieldName, ValueUtils.inferQFieldTypeFromValue(value, QFieldType.STRING));
+      return doesCriteriaMatch(criterion, field, value);
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/
-   public static boolean doesCriteriaMatch(QFilterCriteria criterion, String fieldName, Serializable value)
+   private static boolean doesCriteriaMatch(QFilterCriteria criterion, QFieldMetaData field, Serializable value)
    {
+      String fieldName = field == null ? "__unknownField" : field.getName();
+
       ListIterator<Serializable> valueListIterator = criterion.getValues().listIterator();
       while(valueListIterator.hasNext())
       {
@@ -144,7 +204,7 @@ public class BackendQueryFilterUtils
          {
             try
             {
-               valueListIterator.set(expression.evaluate());
+               valueListIterator.set(expression.evaluate(field));
             }
             catch(QException qe)
             {

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/basepull/ExtractViaBasepullQueryStep.java

@@ -40,6 +40,10 @@ import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwith
  *******************************************************************************/
 public class ExtractViaBasepullQueryStep extends ExtractViaQueryStep implements BasepullExtractStepInterface
 {
+   protected static final String SECONDS_TO_SUBTRACT_FROM_THIS_RUN_TIME_KEY = "secondsToSubtractFromThisRunTimeForTimestampQuery";
+   protected static final String SECONDS_TO_SUBTRACT_FROM_LAST_RUN_TIME_KEY = "secondsToSubtractFromLastRunTimeForTimestampQuery";
+
+
 
    /*******************************************************************************
     **
@@ -125,6 +129,7 @@ public class ExtractViaBasepullQueryStep extends ExtractViaQueryStep implements
    protected String getLastRunTimeString(RunBackendStepInput runBackendStepInput) throws QException
    {
       Instant lastRunTime    = runBackendStepInput.getBasepullLastRunTime();
+      Instant updatedRunTime = lastRunTime;
 
       //////////////////////////////////////////////////////////////////////////////////////////////
       // allow the timestamps to be adjusted by the specified number of seconds.                  //
@@ -135,10 +140,19 @@ public class ExtractViaBasepullQueryStep extends ExtractViaQueryStep implements
       Serializable basepullConfigurationValue = runBackendStepInput.getValue(RunProcessAction.BASEPULL_CONFIGURATION);
       if(basepullConfigurationValue instanceof BasepullConfiguration basepullConfiguration && basepullConfiguration.getSecondsToSubtractFromLastRunTimeForTimestampQuery() != null)
       {
-         lastRunTime = lastRunTime.minusSeconds(basepullConfiguration.getSecondsToSubtractFromLastRunTimeForTimestampQuery());
+         updatedRunTime = lastRunTime.minusSeconds(basepullConfiguration.getSecondsToSubtractFromLastRunTimeForTimestampQuery());
       }
 
-      return (lastRunTime.toString());
+      //////////////////////////////////////////////////////////////
+      // if an override was found in the params, use that instead //
+      //////////////////////////////////////////////////////////////
+      if(runBackendStepInput.getValueString(SECONDS_TO_SUBTRACT_FROM_LAST_RUN_TIME_KEY) != null)
+      {
+         int secondsBack = Integer.parseInt(runBackendStepInput.getValueString(SECONDS_TO_SUBTRACT_FROM_LAST_RUN_TIME_KEY));
+         updatedRunTime = lastRunTime.minusSeconds(secondsBack);
+      }
+
+      return (updatedRunTime.toString());
    }
 
 
@@ -149,13 +163,23 @@ public class ExtractViaBasepullQueryStep extends ExtractViaQueryStep implements
    protected String getThisRunTimeString(RunBackendStepInput runBackendStepInput) throws QException
    {
       Instant thisRunTime    = runBackendStepInput.getValueInstant(RunProcessAction.BASEPULL_THIS_RUNTIME_KEY);
+      Instant updatedRunTime = thisRunTime;
 
       Serializable basepullConfigurationValue = runBackendStepInput.getValue(RunProcessAction.BASEPULL_CONFIGURATION);
       if(basepullConfigurationValue instanceof BasepullConfiguration basepullConfiguration && basepullConfiguration.getSecondsToSubtractFromThisRunTimeForTimestampQuery() != null)
       {
-         thisRunTime = thisRunTime.minusSeconds(basepullConfiguration.getSecondsToSubtractFromThisRunTimeForTimestampQuery());
+         updatedRunTime = thisRunTime.minusSeconds(basepullConfiguration.getSecondsToSubtractFromThisRunTimeForTimestampQuery());
       }
 
-      return (thisRunTime.toString());
+      //////////////////////////////////////////////////////////////
+      // if an override was found in the params, use that instead //
+      //////////////////////////////////////////////////////////////
+      if(runBackendStepInput.getValueString(SECONDS_TO_SUBTRACT_FROM_THIS_RUN_TIME_KEY) != null)
+      {
+         int secondsBack = Integer.parseInt(runBackendStepInput.getValueString(SECONDS_TO_SUBTRACT_FROM_THIS_RUN_TIME_KEY));
+         updatedRunTime = thisRunTime.minusSeconds(secondsBack);
+      }
+
+      return (updatedRunTime.toString());
    }
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/delete/BulkDeleteLoadStep.java

@@ -27,6 +27,7 @@ import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
 import java.util.stream.Collectors;
+import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLine;
 import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLineInterface;
@@ -76,7 +77,7 @@ public class BulkDeleteLoadStep extends LoadViaDeleteStep implements ProcessSumm
    {
       super.preRun(runBackendStepInput, runBackendStepOutput);
 
-      QTableMetaData table = runBackendStepInput.getInstance().getTable(runBackendStepInput.getTableName());
+      QTableMetaData table = QContext.getQInstance().getTable(runBackendStepInput.getTableName());
       if(table != null)
       {
          tableLabel = table.getLabel();
@@ -119,7 +120,7 @@ public class BulkDeleteLoadStep extends LoadViaDeleteStep implements ProcessSumm
       ////////////////////////////
       super.runOnePage(runBackendStepInput, runBackendStepOutput);
 
-      QTableMetaData             table               = runBackendStepInput.getInstance().getTable(runBackendStepInput.getTableName());
+      QTableMetaData             table               = QContext.getQInstance().getTable(runBackendStepInput.getTableName());
       String                     primaryKeyFieldName = table.getPrimaryKeyField();
       Map<Serializable, QRecord> outputRecordMap     = runBackendStepOutput.getRecords().stream().collect(Collectors.toMap(r -> r.getValue(primaryKeyFieldName), r -> r, (a, b) -> a));
 

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/delete/BulkDeleteTransformStep.java

@@ -27,6 +27,7 @@ import java.util.ArrayList;
 import java.util.List;
 import java.util.Optional;
 import com.kingsrook.qqq.backend.core.actions.tables.DeleteAction;
+import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLine;
 import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLineInterface;
@@ -65,7 +66,7 @@ public class BulkDeleteTransformStep extends AbstractTransformStep
       ///////////////////////////////////////////////////////
       // capture the table label - for the process summary //
       ///////////////////////////////////////////////////////
-      QTableMetaData table = runBackendStepInput.getInstance().getTable(runBackendStepInput.getTableName());
+      QTableMetaData table = QContext.getQInstance().getTable(runBackendStepInput.getTableName());
       if(table != null)
       {
          tableLabel = table.getLabel();

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/edit/BulkEditLoadStep.java

@@ -25,6 +25,7 @@ package com.kingsrook.qqq.backend.core.processes.implementations.bulk.edit;
 import java.io.Serializable;
 import java.util.ArrayList;
 import java.util.List;
+import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLine;
 import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLineInterface;
@@ -99,7 +100,7 @@ public class BulkEditLoadStep extends LoadViaUpdateStep implements ProcessSummar
    {
       super.preRun(runBackendStepInput, runBackendStepOutput);
 
-      QTableMetaData table = runBackendStepInput.getInstance().getTable(runBackendStepInput.getTableName());
+      QTableMetaData table = QContext.getQInstance().getTable(runBackendStepInput.getTableName());
       if(table != null)
       {
          tableLabel = table.getLabel();
@@ -124,7 +125,7 @@ public class BulkEditLoadStep extends LoadViaUpdateStep implements ProcessSummar
       ////////////////////////////////////////////////////////
       // roll up results based on output from update action //
       ////////////////////////////////////////////////////////
-      QTableMetaData table = runBackendStepInput.getInstance().getTable(runBackendStepInput.getTableName());
+      QTableMetaData table = QContext.getQInstance().getTable(runBackendStepInput.getTableName());
       for(QRecord record : runBackendStepOutput.getRecords())
       {
          Serializable recordPrimaryKey = record.getValue(table.getPrimaryKeyField());

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/edit/BulkEditTransformStep.java

@@ -31,6 +31,7 @@ import java.util.Optional;
 import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction;
 import com.kingsrook.qqq.backend.core.actions.values.QPossibleValueTranslator;
 import com.kingsrook.qqq.backend.core.actions.values.QValueFormatter;
+import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLine;
 import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLineInterface;
@@ -81,7 +82,7 @@ public class BulkEditTransformStep extends AbstractTransformStep
       ///////////////////////////////////////////////////////
       // capture the table label - for the process summary //
       ///////////////////////////////////////////////////////
-      table = runBackendStepInput.getInstance().getTable(runBackendStepInput.getTableName());
+      table = QContext.getQInstance().getTable(runBackendStepInput.getTableName());
       if(table != null)
       {
          tableLabel = table.getLabel();
@@ -230,7 +231,7 @@ public class BulkEditTransformStep extends AbstractTransformStep
 
             if(field.getPossibleValueSourceName() != null)
             {
-               QPossibleValueTranslator qPossibleValueTranslator = new QPossibleValueTranslator(runBackendStepInput.getInstance(), runBackendStepInput.getSession());
+               QPossibleValueTranslator qPossibleValueTranslator = new QPossibleValueTranslator(QContext.getQInstance(), QContext.getQSession());
                String                   translatedValue          = qPossibleValueTranslator.translatePossibleValue(field, value);
                if(StringUtils.hasContent(translatedValue))
                {

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertExtractStep.java

@@ -27,6 +27,7 @@ import java.util.Locale;
 import java.util.Map;
 import java.util.Optional;
 import com.kingsrook.qqq.backend.core.adapters.CsvToQRecordAdapter;
+import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.exceptions.QUserFacingException;
 import com.kingsrook.qqq.backend.core.model.actions.processes.QUploadedFile;
@@ -83,7 +84,7 @@ public class BulkInsertExtractStep extends AbstractExtractStep
             .withLimit(getLimit())
             .withCsv(new String(bytes))
             .withDoCorrectValueTypes(true)
-            .withTable(runBackendStepInput.getInstance().getTable(tableName))
+            .withTable(QContext.getQInstance().getTable(tableName))
             .withMapping(mapping)
             .withRecordCustomizer((record) ->
             {

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTransformStep.java

@@ -38,6 +38,7 @@ import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizerInterfa
 import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizers;
 import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
 import com.kingsrook.qqq.backend.core.actions.tables.helpers.UniqueKeyHelper;
+import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLine;
 import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLineInterface;
@@ -104,7 +105,7 @@ public class BulkInsertTransformStep extends AbstractTransformStep
    @Override
    public void preRun(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException
    {
-      this.table = runBackendStepInput.getInstance().getTable(runBackendStepInput.getTableName());
+      this.table = QContext.getQInstance().getTable(runBackendStepInput.getTableName());
 
       //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
       // since we're doing a unique key check in this class, we can tell the loadViaInsert step that it (rather, the InsertAction) doesn't need to re-do one. //
@@ -121,7 +122,7 @@ public class BulkInsertTransformStep extends AbstractTransformStep
    public void runOnePage(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException
    {
       int            rowsInThisPage = runBackendStepInput.getRecords().size();
-      QTableMetaData table          = runBackendStepInput.getInstance().getTable(runBackendStepInput.getTableName());
+      QTableMetaData table          = QContext.getQInstance().getTable(runBackendStepInput.getTableName());
 
       /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
       // set up an insert-input, which will be used as input to the pre-customizer as well as for additional validations //

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/columnstats/ColumnStatsFullInstanceVerifier.java

@@ -0,0 +1,126 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.processes.implementations.columnstats;
+
+
+import java.util.Collection;
+import java.util.LinkedHashMap;
+import java.util.Map;
+import com.kingsrook.qqq.backend.core.context.QContext;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.logging.QLogger;
+import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
+import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.Capability;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.ExposedJoin;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
+import com.kingsrook.qqq.backend.core.utils.ExceptionUtils;
+import com.kingsrook.qqq.backend.core.utils.Pair;
+import com.kingsrook.qqq.backend.core.utils.StringUtils;
+import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
+
+
+/*******************************************************************************
+ ** Utility for verifying that the ColumnStats process works for all fields,
+ ** on all tables, and all exposed joins.
+ **
+ ** Meant for use within a unit test, or maybe as part of an instance's boot-up/
+ ** validation.
+ *******************************************************************************/
+public class ColumnStatsFullInstanceVerifier
+{
+   private static final QLogger LOG = QLogger.getLogger(ColumnStatsFullInstanceVerifier.class);
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public void verify(Collection<QTableMetaData> tables) throws QException
+   {
+      Map<Pair<String, String>, Exception> caughtExceptions = new LinkedHashMap<>();
+      for(QTableMetaData table : tables)
+      {
+         if(table.isCapabilityEnabled(QContext.getQInstance().getBackendForTable(table.getName()), Capability.QUERY_STATS))
+         {
+            LOG.info("Verifying ColumnStats on table", logPair("tableName", table.getName()));
+            for(QFieldMetaData field : table.getFields().values())
+            {
+               runColumnStats(table.getName(), field.getName(), caughtExceptions);
+            }
+
+            for(ExposedJoin exposedJoin : CollectionUtils.nonNullList(table.getExposedJoins()))
+            {
+               QTableMetaData joinTable = QContext.getQInstance().getTable(exposedJoin.getJoinTable());
+               for(QFieldMetaData field : joinTable.getFields().values())
+               {
+                  runColumnStats(table.getName(), joinTable.getName() + "." + field.getName(), caughtExceptions);
+               }
+            }
+         }
+      }
+
+      // log out an exceptions caught
+      if(!caughtExceptions.isEmpty())
+      {
+         for(Map.Entry<Pair<String, String>, Exception> entry : caughtExceptions.entrySet())
+         {
+            LOG.info("Caught an exception verifying column stats", entry.getValue(), logPair("tableName", entry.getKey().getA()), logPair("fieldName", entry.getKey().getB()));
+         }
+         throw (new QException("Column Status Verification failed with " + caughtExceptions.size() + " exception" + StringUtils.plural(caughtExceptions.size())));
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private void runColumnStats(String tableName, String fieldName, Map<Pair<String, String>, Exception> caughtExceptions) throws QException
+   {
+      try
+      {
+         RunBackendStepInput input = new RunBackendStepInput();
+         input.addValue("tableName", tableName);
+         input.addValue("fieldName", fieldName);
+         RunBackendStepOutput output = new RunBackendStepOutput();
+         new ColumnStatsStep().run(input, output);
+      }
+      catch(QException e)
+      {
+         Throwable rootException = ExceptionUtils.getRootException(e);
+         if(rootException instanceof QException && rootException.getMessage().contains("not supported for this field's data type"))
+         {
+            ////////////////////////////////////////////////
+            // ignore this exception, it's kinda expected //
+            ////////////////////////////////////////////////
+            LOG.debug("Caught an expected-exception in column stats", e, logPair("tableName", tableName), logPair("fieldName", fieldName));
+         }
+         else
+         {
+            caughtExceptions.put(Pair.of(tableName, fieldName), e);
+         }
+      }
+   }
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/columnstats/ColumnStatsStep.java

@@ -136,29 +136,11 @@ public class ColumnStatsStep implements BackendStep
             filter = new QQueryFilter();
          }
 
-         QueryJoin      queryJoin = null;
          QTableMetaData table = QContext.getQInstance().getTable(tableName);
-         QFieldMetaData field     = null;
-         if(fieldName.contains("."))
-         {
-            String[] parts = fieldName.split("\\.", 2);
-            for(ExposedJoin exposedJoin : CollectionUtils.nonNullList(table.getExposedJoins()))
-            {
-               if(exposedJoin.getJoinTable().equals(parts[0]))
-               {
-                  field = QContext.getQInstance().getTable(exposedJoin.getJoinTable()).getField(parts[1]);
-                  queryJoin = new QueryJoin()
-                     .withJoinTable(exposedJoin.getJoinTable())
-                     .withSelect(true)
-                     .withType(QueryJoin.Type.INNER);
-                  break;
-               }
-            }
-         }
-         else
-         {
-            field = table.getField(fieldName);
-         }
+
+         FieldAndQueryJoin fieldAndQueryJoin = getFieldAndQueryJoin(table, fieldName);
+         QFieldMetaData    field             = fieldAndQueryJoin.field();
+         QueryJoin         queryJoin         = fieldAndQueryJoin.queryJoin();
 
          if(field == null)
          {
@@ -213,7 +195,7 @@ public class ColumnStatsStep implements BackendStep
          filter.withOrderBy(new QFilterOrderByAggregate(aggregate, false));
          filter.withOrderBy(new QFilterOrderByGroupBy(groupBy));
 
-         Integer        limit          = 1000; // too big?
+         Integer        limit          = 1000;
          AggregateInput aggregateInput = new AggregateInput();
          aggregateInput.withAggregate(aggregate);
          aggregateInput.withGroupBy(groupBy);
@@ -223,7 +205,11 @@ public class ColumnStatsStep implements BackendStep
 
          if(queryJoin != null)
          {
-            aggregateInput.withQueryJoin(queryJoin);
+            ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+            // re-construct this queryJoin object - just because, the JoinsContext edits the previous one, so we can make some failing-joins otherwise... //
+            ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+            fieldAndQueryJoin = getFieldAndQueryJoin(table, fieldName);
+            aggregateInput.withQueryJoin(fieldAndQueryJoin.queryJoin());
          }
 
          AggregateOutput aggregateOutput = new AggregateAction().execute(aggregateInput);
@@ -526,4 +512,43 @@ public class ColumnStatsStep implements BackendStep
       return (null);
    }
 
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private FieldAndQueryJoin getFieldAndQueryJoin(QTableMetaData table, String fieldName)
+   {
+      QFieldMetaData field     = null;
+      QueryJoin      queryJoin = null;
+      if(fieldName.contains("."))
+      {
+         String[] parts = fieldName.split("\\.", 2);
+         for(ExposedJoin exposedJoin : CollectionUtils.nonNullList(table.getExposedJoins()))
+         {
+            if(exposedJoin.getJoinTable().equals(parts[0]))
+            {
+               field = QContext.getQInstance().getTable(exposedJoin.getJoinTable()).getField(parts[1]);
+               queryJoin = new QueryJoin()
+                  .withJoinTable(exposedJoin.getJoinTable())
+                  .withSelect(true)
+                  .withType(QueryJoin.Type.INNER);
+               break;
+            }
+         }
+      }
+      else
+      {
+         field = table.getField(fieldName);
+      }
+
+      return (new FieldAndQueryJoin(field, queryJoin));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private record FieldAndQueryJoin(QFieldMetaData field, QueryJoin queryJoin) {}
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/basic/BasicETLTransformFunction.java

@@ -28,6 +28,7 @@ import java.util.ArrayList;
 import java.util.List;
 import com.kingsrook.qqq.backend.core.actions.processes.BackendStep;
 import com.kingsrook.qqq.backend.core.adapters.JsonToQFieldMappingAdapter;
+import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.logging.QLogger;
 import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
@@ -85,7 +86,7 @@ public class BasicETLTransformFunction implements BackendStep
          throw (new QException("Mapping was not a Key-based mapping type.  Was a : " + mapping.getClass().getName()));
       }
 
-      QTableMetaData table         = runBackendStepInput.getInstance().getTable(tableName);
+      QTableMetaData table         = QContext.getQInstance().getTable(tableName);
       List<QRecord>  mappedRecords = applyMapping(runBackendStepInput.getRecords(), table, keyBasedFieldMapping);
 
       //////////////////////////////////////////////////////////////////////////////////////////////////////

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/BaseStreamedETLStep.java

@@ -27,6 +27,7 @@ import java.util.List;
 import com.kingsrook.qqq.backend.core.actions.customizers.QCodeLoader;
 import com.kingsrook.qqq.backend.core.actions.values.QPossibleValueTranslator;
 import com.kingsrook.qqq.backend.core.actions.values.QValueFormatter;
+import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.logging.QLogger;
 import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
 import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
@@ -85,14 +86,14 @@ public class BaseStreamedETLStep
    protected void updateRecordsWithDisplayValuesAndPossibleValues(RunBackendStepInput input, List<QRecord> list)
    {
       String         destinationTable = input.getValueString(StreamedETLWithFrontendProcess.FIELD_DESTINATION_TABLE);
-      QTableMetaData table            = input.getInstance().getTable(destinationTable);
+      QTableMetaData table            = QContext.getQInstance().getTable(destinationTable);
 
       if(table != null && list != null)
       {
          QValueFormatter qValueFormatter = new QValueFormatter();
          qValueFormatter.setDisplayValuesInRecords(table, list);
 
-         QPossibleValueTranslator qPossibleValueTranslator = new QPossibleValueTranslator(input.getInstance(), input.getSession());
+         QPossibleValueTranslator qPossibleValueTranslator = new QPossibleValueTranslator(QContext.getQInstance(), QContext.getQSession());
          qPossibleValueTranslator.translatePossibleValuesInRecords(table, list);
       }
    }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/ExtractViaQueryStep.java

@@ -34,6 +34,7 @@ import com.kingsrook.qqq.backend.core.actions.reporting.DistinctFilteringRecordP
 import com.kingsrook.qqq.backend.core.actions.reporting.RecordPipe;
 import com.kingsrook.qqq.backend.core.actions.tables.CountAction;
 import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
+import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.logging.QLogger;
 import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
@@ -267,7 +268,7 @@ public class ExtractViaQueryStep extends AbstractExtractStep
          //////////////////////////////////////////////////////////////////////
          // else, check for recordIds from a frontend launching of a process //
          //////////////////////////////////////////////////////////////////////
-         QTableMetaData table = runBackendStepInput.getInstance().getTable(runBackendStepInput.getValueString(FIELD_SOURCE_TABLE));
+         QTableMetaData table = QContext.getQInstance().getTable(runBackendStepInput.getValueString(FIELD_SOURCE_TABLE));
          if(table == null)
          {
             throw (new QException("source table name was not set - could not load records by id"));
@@ -319,7 +320,7 @@ public class ExtractViaQueryStep extends AbstractExtractStep
       if(needDistinctPipe)
       {
          String         sourceTableName = runBackendStepInput.getValueString(StreamedETLWithFrontendProcess.FIELD_SOURCE_TABLE);
-         QTableMetaData sourceTable     = runBackendStepInput.getInstance().getTable(sourceTableName);
+         QTableMetaData sourceTable     = QContext.getQInstance().getTable(sourceTableName);
          return (new DistinctFilteringRecordPipe(new UniqueKey(sourceTable.getPrimaryKeyField()), overrideCapacity));
       }
       else

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/LoadViaInsertOrUpdateStep.java

@@ -28,6 +28,7 @@ import java.util.Optional;
 import com.kingsrook.qqq.backend.core.actions.QBackendTransaction;
 import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
 import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction;
+import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
 import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
@@ -86,7 +87,7 @@ public class LoadViaInsertOrUpdateStep extends AbstractLoadStep
     *******************************************************************************/
    public void insertAndUpdateRecords(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException
    {
-      QTableMetaData tableMetaData = runBackendStepInput.getInstance().getTable(runBackendStepInput.getValueString(FIELD_DESTINATION_TABLE));
+      QTableMetaData tableMetaData = QContext.getQInstance().getTable(runBackendStepInput.getValueString(FIELD_DESTINATION_TABLE));
 
       if(CollectionUtils.nullSafeHasContents(recordsToInsert))
       {
@@ -139,7 +140,7 @@ public class LoadViaInsertOrUpdateStep extends AbstractLoadStep
     *******************************************************************************/
    protected void evaluateRecords(RunBackendStepInput runBackendStepInput) throws QException
    {
-      QTableMetaData tableMetaData = runBackendStepInput.getInstance().getTable(runBackendStepInput.getValueString(FIELD_DESTINATION_TABLE));
+      QTableMetaData tableMetaData = QContext.getQInstance().getTable(runBackendStepInput.getValueString(FIELD_DESTINATION_TABLE));
       recordsToInsert = new ArrayList<>();
       recordsToUpdate = new ArrayList<>();
 

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLPreviewStep.java

@@ -162,10 +162,10 @@ public class StreamedETLPreviewStep extends BaseStreamedETLStep implements Backe
    private void countRecords(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput, AbstractExtractStep extractStep) throws QException
    {
       String         sourceTableName = runBackendStepInput.getValueString(StreamedETLWithFrontendProcess.FIELD_SOURCE_TABLE);
-      QTableMetaData sourceTable     = runBackendStepInput.getInstance().getTable(sourceTableName);
+      QTableMetaData sourceTable     = QContext.getQInstance().getTable(sourceTableName);
       if(StringUtils.hasContent(sourceTableName))
       {
-         QBackendMetaData sourceTableBackend = runBackendStepInput.getInstance().getBackendForTable(sourceTableName);
+         QBackendMetaData sourceTableBackend = QContext.getQInstance().getBackendForTable(sourceTableName);
          if(sourceTable.isCapabilityEnabled(sourceTableBackend, Capability.TABLE_COUNT))
          {
             Integer recordCount = extractStep.doCount(runBackendStepInput);

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/mergeduplicates/MergeDuplicatesLoadStep.java

@@ -65,8 +65,13 @@ public class MergeDuplicatesLoadStep extends LoadViaInsertOrUpdateStep
    {
       super.runOnePage(runBackendStepInput, runBackendStepOutput);
 
+      @SuppressWarnings("unchecked")
       ListingHash<String, Serializable> otherTableIdsToDelete = (ListingHash<String, Serializable>) runBackendStepInput.getValue("otherTableIdsToDelete");
+
+      @SuppressWarnings("unchecked")
       ListingHash<String, QQueryFilter> otherTableFiltersToDelete = (ListingHash<String, QQueryFilter>) runBackendStepInput.getValue("otherTableFiltersToDelete");
+
+      @SuppressWarnings("unchecked")
       ListingHash<String, QRecord> otherTableRecordsToStore = (ListingHash<String, QRecord>) runBackendStepInput.getValue("otherTableRecordsToStore");
 
       if(otherTableIdsToDelete != null)

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/reports/BasicRunReportProcess.java

@@ -47,6 +47,7 @@ public class BasicRunReportProcess
    public static final String STEP_NAME_ACCESS  = "accessReport";
 
    public static final String FIELD_REPORT_NAME   = "reportName";
+   public static final String FIELD_REPORT_FORMAT = "reportFormat";
 
 
 

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/reports/ExecuteReportStep.java

@@ -31,7 +31,9 @@ import java.time.format.DateTimeFormatter;
 import java.util.Map;
 import com.kingsrook.qqq.backend.core.actions.processes.BackendStep;
 import com.kingsrook.qqq.backend.core.actions.reporting.GenerateReportAction;
+import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.exceptions.QUserFacingException;
 import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
 import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
 import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportDestination;
@@ -49,6 +51,7 @@ import com.kingsrook.qqq.backend.core.utils.StringUtils;
 public class ExecuteReportStep implements BackendStep
 {
 
+
    /*******************************************************************************
     **
     *******************************************************************************/
@@ -57,9 +60,10 @@ public class ExecuteReportStep implements BackendStep
    {
       try
       {
+         ReportFormat    reportFormat = getReportFormat(runBackendStepInput);
          String          reportName   = runBackendStepInput.getValueString("reportName");
-         QReportMetaData report     = runBackendStepInput.getInstance().getReport(reportName);
-         File            tmpFile    = File.createTempFile(reportName, ".xlsx", new File("/tmp/"));
+         QReportMetaData report       = QContext.getQInstance().getReport(reportName);
+         File            tmpFile      = File.createTempFile(reportName, "." + reportFormat.getExtension());
 
          runBackendStepInput.getAsyncJobCallback().updateStatus("Generating Report");
 
@@ -68,7 +72,7 @@ public class ExecuteReportStep implements BackendStep
             ReportInput reportInput = new ReportInput();
             reportInput.setReportName(reportName);
             reportInput.setReportDestination(new ReportDestination()
-               .withReportFormat(ReportFormat.XLSX) // todo - variable
+               .withReportFormat(reportFormat)
                .withReportOutputStream(reportOutputStream));
 
             Map<String, Serializable> values = runBackendStepInput.getValues();
@@ -78,7 +82,7 @@ public class ExecuteReportStep implements BackendStep
 
             String downloadFileBaseName = getDownloadFileBaseName(runBackendStepInput, report);
 
-            runBackendStepOutput.addValue("downloadFileName", downloadFileBaseName + ".xlsx");
+            runBackendStepOutput.addValue("downloadFileName", downloadFileBaseName + "." + reportFormat.getExtension());
             runBackendStepOutput.addValue("serverFilePath", tmpFile.getCanonicalPath());
          }
       }
@@ -90,6 +94,22 @@ public class ExecuteReportStep implements BackendStep
 
 
 
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   private ReportFormat getReportFormat(RunBackendStepInput runBackendStepInput) throws QUserFacingException
+   {
+      String reportFormatInput = runBackendStepInput.getValueString(BasicRunReportProcess.FIELD_REPORT_FORMAT);
+      if(StringUtils.hasContent(reportFormatInput))
+      {
+         return (ReportFormat.fromString(reportFormatInput));
+      }
+
+      return (ReportFormat.XLSX);
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/reports/PrepareReportForRecordStep.java

@@ -26,6 +26,7 @@ import java.io.Serializable;
 import java.util.ArrayList;
 import java.util.Iterator;
 import com.kingsrook.qqq.backend.core.actions.tables.GetAction;
+import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.exceptions.QUserFacingException;
 import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
@@ -106,15 +107,16 @@ public class PrepareReportForRecordStep extends PrepareReportStep
       }
 
       String          reportName = runBackendStepInput.getValueString("reportName");
-      QReportMetaData report     = runBackendStepInput.getInstance().getReport(reportName);
+      QReportMetaData report     = QContext.getQInstance().getReport(reportName);
       // runBackendStepOutput.addValue("downloadFileBaseName", runBackendStepInput.getTable().getLabel() + " " + record.getRecordLabel());
       runBackendStepOutput.addValue("downloadFileBaseName", report.getLabel() + " - " + record.getRecordLabel());
 
       /////////////////////////////////////////////////////////////////////////////////////
       // if there are no more input fields, then remove the INPUT step from the process. //
       /////////////////////////////////////////////////////////////////////////////////////
-      inputFieldList = (ArrayList<QFieldMetaData>) runBackendStepOutput.getValue("inputFieldList");
-      if(!CollectionUtils.nullSafeHasContents(inputFieldList))
+      @SuppressWarnings("unchecked")
+      ArrayList<QFieldMetaData> updatedInputFieldList = (ArrayList<QFieldMetaData>) runBackendStepOutput.getValue("inputFieldList");
+      if(!CollectionUtils.nullSafeHasContents(updatedInputFieldList))
       {
          removeInputStepFromProcess(runBackendStepOutput);
       }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/reports/PrepareReportStep.java

@@ -25,6 +25,7 @@ package com.kingsrook.qqq.backend.core.processes.implementations.reports;
 import java.util.ArrayList;
 import java.util.List;
 import com.kingsrook.qqq.backend.core.actions.processes.BackendStep;
+import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
 import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
@@ -56,7 +57,7 @@ public class PrepareReportStep implements BackendStep
          throw (new QException("Process value [reportName] was not given."));
       }
 
-      QReportMetaData report = runBackendStepInput.getInstance().getReport(reportName);
+      QReportMetaData report = QContext.getQInstance().getReport(reportName);
       if(report == null)
       {
          throw (new QException("Process named [" + reportName + "] was not found in this instance."));

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/savedreports/RenderSavedReportExecuteStep.java

@@ -129,6 +129,7 @@ public class RenderSavedReportExecuteStep implements BackendStep
             .withRenderedReportStatusId(RenderedReportStatus.RUNNING.getId())
             .withReportFormat(ReportFormatPossibleValueEnum.valueOf(reportFormat.name()).getPossibleValueId())
          )).getRecords().get(0);
+         runBackendStepOutput.addValue("renderedReportId", renderedReportRecord.getValue("id"));
 
          ////////////////////////////////////////////////////////////////////////////////////////////
          // convert the report record to report meta-data, which the GenerateReportAction works on //

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/savedreports/ReportsFullInstanceVerifier.java

@@ -0,0 +1,258 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.processes.implementations.savedreports;
+
+
+import java.util.Collection;
+import java.util.Collections;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import com.kingsrook.qqq.backend.core.actions.tables.DeleteAction;
+import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
+import com.kingsrook.qqq.backend.core.context.QContext;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.logging.QLogger;
+import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
+import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
+import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportFormat;
+import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.Capability;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.ExposedJoin;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+import com.kingsrook.qqq.backend.core.model.savedreports.RenderedReport;
+import com.kingsrook.qqq.backend.core.model.savedreports.ReportColumns;
+import com.kingsrook.qqq.backend.core.model.savedreports.SavedReport;
+import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
+import com.kingsrook.qqq.backend.core.utils.JsonUtils;
+import com.kingsrook.qqq.backend.core.utils.Pair;
+import com.kingsrook.qqq.backend.core.utils.StringUtils;
+import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
+
+
+/*******************************************************************************
+ ** Utility for verifying that the RenderReports process works for all fields,
+ ** on all tables, and all exposed joins.
+ **
+ ** Meant for use within a unit test, or maybe as part of an instance's boot-up/
+ ** validation.
+ *******************************************************************************/
+public class ReportsFullInstanceVerifier
+{
+   private static final QLogger LOG = QLogger.getLogger(ReportsFullInstanceVerifier.class);
+
+   private boolean removeRenderedReports          = true;
+   private boolean filterForAtMostOneRowPerReport = true;
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public void verify(Collection<QTableMetaData> tables, String storageTableName) throws QException
+   {
+      Map<Pair<String, String>, Exception> caughtExceptions = new LinkedHashMap<>();
+      for(QTableMetaData table : tables)
+      {
+         if(table.isCapabilityEnabled(QContext.getQInstance().getBackendForTable(table.getName()), Capability.TABLE_QUERY))
+         {
+            LOG.info("Verifying Reports on table", logPair("tableName", table.getName()));
+
+            //////////////////////////////////////////////
+            // run the table by itself (no join fields) //
+            //////////////////////////////////////////////
+            runReport(table.getName(), Collections.emptyList(), "main-table-only", caughtExceptions, storageTableName);
+
+            ///////////////////////////////////////////////////
+            // run once w/ the fields from each exposed join //
+            ///////////////////////////////////////////////////
+            for(ExposedJoin exposedJoin : CollectionUtils.nonNullList(table.getExposedJoins()))
+            {
+               runReport(table.getName(), List.of(exposedJoin), "join-" + exposedJoin.getLabel(), caughtExceptions, storageTableName);
+            }
+
+            /////////////////////////////////////////////////
+            // run w/ all exposed joins (if there are any) //
+            /////////////////////////////////////////////////
+            if(CollectionUtils.nullSafeHasContents(table.getExposedJoins()))
+            {
+               runReport(table.getName(), table.getExposedJoins(), "all-joins", caughtExceptions, storageTableName);
+            }
+         }
+      }
+
+      //////////////////////////////////
+      // log out an exceptions caught //
+      //////////////////////////////////
+      if(!caughtExceptions.isEmpty())
+      {
+         for(Map.Entry<Pair<String, String>, Exception> entry : caughtExceptions.entrySet())
+         {
+            LOG.info("Caught an exception verifying reports", entry.getValue(), logPair("tableName", entry.getKey().getA()), logPair("fieldName", entry.getKey().getB()));
+         }
+         throw (new QException("Reports Verification failed with " + caughtExceptions.size() + " exception" + StringUtils.plural(caughtExceptions.size())));
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private void runReport(String tableName, List<ExposedJoin> exposedJoinList, String description, Map<Pair<String, String>, Exception> caughtExceptions, String storageTableName)
+   {
+      try
+      {
+         ////////////////////////////////////////////////////////////////////////////////////////////////
+         // build the list of reports to include in the column - starting with all fields in the table //
+         ////////////////////////////////////////////////////////////////////////////////////////////////
+         ReportColumns reportColumns = new ReportColumns();
+         for(QFieldMetaData field : QContext.getQInstance().getTable(tableName).getFields().values())
+         {
+            reportColumns.withColumn(field.getName());
+         }
+
+         ///////////////////////////////////////////////////
+         // add all fields from all exposed joins as well //
+         ///////////////////////////////////////////////////
+         for(ExposedJoin exposedJoin : CollectionUtils.nonNullList(exposedJoinList))
+         {
+            QTableMetaData joinTable = QContext.getQInstance().getTable(exposedJoin.getJoinTable());
+            for(QFieldMetaData field : joinTable.getFields().values())
+            {
+               reportColumns.withColumn(joinTable.getName() + "." + field.getName());
+            }
+         }
+
+         QQueryFilter queryFilter = new QQueryFilter();
+
+         /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+         // if caller is okay with a filter that should limit the report to a small number of rows (could be more than 1 for to-many joins), then do so //
+         /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+         if(filterForAtMostOneRowPerReport)
+         {
+            queryFilter.withCriteria(QContext.getQInstance().getTable(tableName).getPrimaryKeyField(), QCriteriaOperator.EQUALS, 1);
+         }
+
+         //////////////////////////////////
+         // insert a saved report record //
+         //////////////////////////////////
+         SavedReport savedReport = new SavedReport();
+         savedReport.setTableName(tableName);
+         savedReport.setLabel("Test " + tableName + " " + description);
+         savedReport.setColumnsJson(JsonUtils.toJson(reportColumns));
+         savedReport.setQueryFilterJson(JsonUtils.toJson(queryFilter));
+         List<QRecord> reportRecordList = new InsertAction().execute(new InsertInput(SavedReport.TABLE_NAME).withRecordEntity(savedReport)).getRecords();
+
+         ///////////////////////
+         // render the report //
+         ///////////////////////
+         RunBackendStepInput  input  = new RunBackendStepInput();
+         RunBackendStepOutput output = new RunBackendStepOutput();
+
+         input.addValue(RenderSavedReportMetaDataProducer.FIELD_NAME_REPORT_FORMAT, ReportFormat.CSV.name());
+         input.addValue(RenderSavedReportMetaDataProducer.FIELD_NAME_STORAGE_TABLE_NAME, storageTableName);
+         input.setRecords(reportRecordList);
+
+         new RenderSavedReportExecuteStep().run(input, output);
+
+         //////////////////////////////////////////
+         // clean up the report, if so requested //
+         //////////////////////////////////////////
+         if(removeRenderedReports)
+         {
+            new DeleteAction().execute(new DeleteInput(RenderedReport.TABLE_NAME).withPrimaryKey(output.getValue("renderedReportId")));
+         }
+      }
+      catch(QException e)
+      {
+         caughtExceptions.put(Pair.of(tableName, description), e);
+      }
+   }
+
+   /*******************************************************************************
+    ** Getter for removeRenderedReports
+    *******************************************************************************/
+   public boolean getRemoveRenderedReports()
+   {
+      return (this.removeRenderedReports);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for removeRenderedReports
+    *******************************************************************************/
+   public void setRemoveRenderedReports(boolean removeRenderedReports)
+   {
+      this.removeRenderedReports = removeRenderedReports;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for removeRenderedReports
+    *******************************************************************************/
+   public ReportsFullInstanceVerifier withRemoveRenderedReports(boolean removeRenderedReports)
+   {
+      this.removeRenderedReports = removeRenderedReports;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for filterForAtMostOneRowPerReport
+    *******************************************************************************/
+   public boolean getFilterForAtMostOneRowPerReport()
+   {
+      return (this.filterForAtMostOneRowPerReport);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for filterForAtMostOneRowPerReport
+    *******************************************************************************/
+   public void setFilterForAtMostOneRowPerReport(boolean filterForAtMostOneRowPerReport)
+   {
+      this.filterForAtMostOneRowPerReport = filterForAtMostOneRowPerReport;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for filterForAtMostOneRowPerReport
+    *******************************************************************************/
+   public ReportsFullInstanceVerifier withFilterForAtMostOneRowPerReport(boolean filterForAtMostOneRowPerReport)
+   {
+      this.filterForAtMostOneRowPerReport = filterForAtMostOneRowPerReport;
+      return (this);
+   }
+
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/savedreports/SavedReportsTableFullVerifier.java

@@ -0,0 +1,151 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.processes.implementations.savedreports;
+
+
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import com.kingsrook.qqq.backend.core.actions.tables.DeleteAction;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.logging.QLogger;
+import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
+import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
+import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportFormat;
+import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.savedreports.RenderedReport;
+import com.kingsrook.qqq.backend.core.utils.StringUtils;
+import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
+
+
+/*******************************************************************************
+ ** Utility for verifying that the RenderReports process works for all report
+ ** records stored in the saved reports table.
+ **
+ ** Meant for use within a unit test, or maybe as part of an instance's boot-up/
+ ** validation.
+ *******************************************************************************/
+public class SavedReportsTableFullVerifier
+{
+   private static final QLogger LOG = QLogger.getLogger(SavedReportsTableFullVerifier.class);
+
+   private boolean removeRenderedReports = true;
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public void verify(List<QRecord> savedReportRecordList, String storageTableName) throws QException
+   {
+      Map<Integer, Exception> caughtExceptions = new LinkedHashMap<>();
+      for(QRecord report : savedReportRecordList)
+      {
+         runReport(report, caughtExceptions, storageTableName);
+      }
+
+      //////////////////////////////////
+      // log out an exceptions caught //
+      //////////////////////////////////
+      if(!caughtExceptions.isEmpty())
+      {
+         for(Map.Entry<Integer, Exception> entry : caughtExceptions.entrySet())
+         {
+            LOG.info("Caught an exception verifying saved reports", entry.getValue(), logPair("savdReportId", entry.getKey()));
+         }
+         throw (new QException("Saved Reports Verification failed with " + caughtExceptions.size() + " exception" + StringUtils.plural(caughtExceptions.size())));
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private void runReport(QRecord savedReport, Map<Integer, Exception> caughtExceptions, String storageTableName)
+   {
+      try
+      {
+         ///////////////////////
+         // render the report //
+         ///////////////////////
+         RunBackendStepInput  input  = new RunBackendStepInput();
+         RunBackendStepOutput output = new RunBackendStepOutput();
+
+         input.addValue(RenderSavedReportMetaDataProducer.FIELD_NAME_REPORT_FORMAT, ReportFormat.XLSX.name());
+         input.addValue(RenderSavedReportMetaDataProducer.FIELD_NAME_STORAGE_TABLE_NAME, storageTableName);
+         input.setRecords(List.of(savedReport));
+
+         new RenderSavedReportExecuteStep().run(input, output);
+         Exception exception = output.getException();
+         if(exception != null)
+         {
+            throw (exception);
+         }
+
+         //////////////////////////////////////////
+         // clean up the report, if so requested //
+         //////////////////////////////////////////
+         if(removeRenderedReports)
+         {
+            new DeleteAction().execute(new DeleteInput(RenderedReport.TABLE_NAME).withPrimaryKey(output.getValue("renderedReportId")));
+         }
+      }
+      catch(Exception e)
+      {
+         caughtExceptions.put(savedReport.getValueInteger("id"), e);
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for removeRenderedReports
+    *******************************************************************************/
+   public boolean getRemoveRenderedReports()
+   {
+      return (this.removeRenderedReports);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for removeRenderedReports
+    *******************************************************************************/
+   public void setRemoveRenderedReports(boolean removeRenderedReports)
+   {
+      this.removeRenderedReports = removeRenderedReports;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for removeRenderedReports
+    *******************************************************************************/
+   public SavedReportsTableFullVerifier withRemoveRenderedReports(boolean removeRenderedReports)
+   {
+      this.removeRenderedReports = removeRenderedReports;
+      return (this);
+   }
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/savedviews/QuerySavedViewProcess.java

@@ -29,6 +29,7 @@ import com.kingsrook.qqq.backend.core.actions.processes.BackendStep;
 import com.kingsrook.qqq.backend.core.actions.tables.GetAction;
 import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.exceptions.QNotFoundException;
 import com.kingsrook.qqq.backend.core.logging.QLogger;
 import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
 import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
@@ -44,6 +45,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
 import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
 import com.kingsrook.qqq.backend.core.model.savedviews.SavedView;
+import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
 
 
 /*******************************************************************************
@@ -78,10 +80,10 @@ public class QuerySavedViewProcess implements BackendStep
    public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException
    {
       ActionHelper.validateSession(runBackendStepInput);
+      Integer savedViewId = runBackendStepInput.getValueInteger("id");
 
       try
       {
-         Integer savedViewId = runBackendStepInput.getValueInteger("id");
          if(savedViewId != null)
          {
             GetInput input = new GetInput();
@@ -89,6 +91,11 @@ public class QuerySavedViewProcess implements BackendStep
             input.setPrimaryKey(savedViewId);
 
             GetOutput output = new GetAction().execute(input);
+            if(output.getRecord() == null)
+            {
+               throw (new QNotFoundException("The requested view was not found."));
+            }
+
             runBackendStepOutput.addRecord(output.getRecord());
             runBackendStepOutput.addValue("savedView", output.getRecord());
             runBackendStepOutput.addValue("savedViewList", (Serializable) List.of(output.getRecord()));
@@ -108,6 +115,11 @@ public class QuerySavedViewProcess implements BackendStep
             runBackendStepOutput.addValue("savedViewList", (Serializable) output.getRecords());
          }
       }
+      catch(QNotFoundException qnfe)
+      {
+         LOG.info("View not found", logPair("savedViewId", savedViewId));
+         throw (qnfe);
+      }
       catch(Exception e)
       {
          LOG.warn("Error querying for saved views", e);

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/scripts/StoreScriptRevisionProcessStep.java

@@ -145,7 +145,7 @@ public class StoreScriptRevisionProcessStep implements BackendStep
 
          try
          {
-            scriptRevision.setValue("author", input.getSession().getUser().getFullName());
+            scriptRevision.setValue("author", QContext.getQSession().getUser().getFullName());
          }
          catch(Exception e)
          {

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/scripts/TestScriptProcessStep.java

@@ -139,7 +139,7 @@ public class TestScriptProcessStep implements BackendStep
          //////////////////////////////////
          // send script outputs back out //
          //////////////////////////////////
-         output.addValue("scriptLogLines", CollectionUtils.useOrWrap(testScriptOutput.getScriptLogLines(), TypeToken.get(ArrayList.class)));
+         output.addValue("scriptLogLines", CollectionUtils.useOrWrap(testScriptOutput.getScriptLogLines(), new TypeToken<ArrayList<QRecord>>() {}));
          output.addValue("outputObject", testScriptOutput.getOutputObject());
 
          if(testScriptOutput.getException() != null)

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/tablesync/AbstractTableSyncTransformStep.java

@@ -23,6 +23,9 @@ package com.kingsrook.qqq.backend.core.processes.implementations.tablesync;
 
 
 import java.io.Serializable;
+import java.time.Instant;
+import java.time.ZoneId;
+import java.time.ZonedDateTime;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
@@ -35,6 +38,7 @@ import java.util.Set;
 import java.util.stream.Collectors;
 import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
 import com.kingsrook.qqq.backend.core.actions.values.QPossibleValueTranslator;
+import com.kingsrook.qqq.backend.core.actions.values.QValueFormatter;
 import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.logging.QLogger;
@@ -53,6 +57,7 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryJoin;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
+import com.kingsrook.qqq.backend.core.model.session.QSession;
 import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.AbstractTransformStep;
 import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess;
 import com.kingsrook.qqq.backend.core.processes.implementations.general.StandardProcessSummaryLineProducer;
@@ -72,33 +77,33 @@ import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
  *******************************************************************************/
 public abstract class AbstractTableSyncTransformStep extends AbstractTransformStep
 {
-   private static final QLogger LOG = QLogger.getLogger(AbstractTableSyncTransformStep.class);
+   protected static final QLogger LOG = QLogger.getLogger(AbstractTableSyncTransformStep.class);
 
-   private ProcessSummaryLine okToInsert = StandardProcessSummaryLineProducer.getOkToInsertLine();
-   private ProcessSummaryLine okToUpdate = StandardProcessSummaryLineProducer.getOkToUpdateLine();
+   protected ProcessSummaryLine okToInsert = StandardProcessSummaryLineProducer.getOkToInsertLine();
+   protected ProcessSummaryLine okToUpdate = StandardProcessSummaryLineProducer.getOkToUpdateLine();
 
-   private ProcessSummaryLine willNotInsert = new ProcessSummaryLine(Status.INFO)
+   protected ProcessSummaryLine willNotInsert = new ProcessSummaryLine(Status.INFO)
       .withMessageSuffix("because this process is not configured to insert records.")
       .withSingularFutureMessage("will not be inserted ")
       .withPluralFutureMessage("will not be inserted ")
       .withSingularPastMessage("was not inserted ")
       .withPluralPastMessage("were not inserted ");
 
-   private ProcessSummaryLine willNotUpdate = new ProcessSummaryLine(Status.INFO)
+   protected ProcessSummaryLine willNotUpdate = new ProcessSummaryLine(Status.INFO)
       .withMessageSuffix("because this process is not configured to update records.")
       .withSingularFutureMessage("will not be updated ")
       .withPluralFutureMessage("will not be updated ")
       .withSingularPastMessage("was not updated ")
       .withPluralPastMessage("were not updated ");
 
-   private ProcessSummaryLine errorMissingKeyField = new ProcessSummaryLine(Status.ERROR)
+   protected ProcessSummaryLine errorMissingKeyField = new ProcessSummaryLine(Status.ERROR)
       .withMessageSuffix("missing a value for the key field.")
       .withSingularFutureMessage("will not be synced, because it is ")
       .withPluralFutureMessage("will not be synced, because they are ")
       .withSingularPastMessage("was not synced, because it is ")
       .withPluralPastMessage("were not synced, because they are ");
 
-   private ProcessSummaryLine unspecifiedError = new ProcessSummaryLine(Status.ERROR)
+   protected ProcessSummaryLine unspecifiedError = new ProcessSummaryLine(Status.ERROR)
       .withMessageSuffix("of an unexpected error: ")
       .withSingularFutureMessage("will not be synced, ")
       .withPluralFutureMessage("will not be synced, ")
@@ -109,7 +114,11 @@ public abstract class AbstractTableSyncTransformStep extends AbstractTransformSt
    protected RunBackendStepOutput runBackendStepOutput = null;
    protected RecordLookupHelper   recordLookupHelper   = null;
 
-   private QPossibleValueTranslator possibleValueTranslator;
+   protected QPossibleValueTranslator possibleValueTranslator;
+
+   protected static final String SYNC_TABLE_PERFORM_INSERTS_KEY = "syncTablePerformInsertsKey";
+   protected static final String SYNC_TABLE_PERFORM_UPDATES_KEY = "syncTablePerformUpdatesKey";
+   protected static final String LOG_TRANSFORM_RESULTS          = "logTransformResults";
 
 
 
@@ -214,6 +223,7 @@ public abstract class AbstractTableSyncTransformStep extends AbstractTransformSt
    {
       if(CollectionUtils.nullSafeIsEmpty(runBackendStepInput.getRecords()))
       {
+         LOG.info("No input records were found.");
          return;
       }
 
@@ -222,6 +232,19 @@ public abstract class AbstractTableSyncTransformStep extends AbstractTransformSt
 
       SyncProcessConfig config = getSyncProcessConfig();
 
+      ////////////////////////////////////////////////////////////
+      // see if these fields have been updated via input fields //
+      ////////////////////////////////////////////////////////////
+      if(runBackendStepInput.getValueString(SYNC_TABLE_PERFORM_INSERTS_KEY) != null)
+      {
+         boolean performInserts = Boolean.parseBoolean(runBackendStepInput.getValueString(SYNC_TABLE_PERFORM_INSERTS_KEY));
+         config = new SyncProcessConfig(config.sourceTable, config.sourceTableKeyField, config.destinationTable, config.destinationTableForeignKey, performInserts, config.performUpdates);
+      }
+      if(runBackendStepInput.getValueString(SYNC_TABLE_PERFORM_UPDATES_KEY) != null)
+      {
+         boolean performUpdates = Boolean.parseBoolean(runBackendStepInput.getValueString(SYNC_TABLE_PERFORM_UPDATES_KEY));
+         config = new SyncProcessConfig(config.sourceTable, config.sourceTableKeyField, config.destinationTable, config.destinationTableForeignKey, config.performUpdates, performUpdates);
+      }
       String sourceTableKeyField             = config.sourceTableKeyField;
       String destinationTableForeignKeyField = config.destinationTableForeignKey;
       String destinationTableName            = config.destinationTable;
@@ -365,15 +388,69 @@ public abstract class AbstractTableSyncTransformStep extends AbstractTransformSt
          {
             if(possibleValueTranslator == null)
             {
-               possibleValueTranslator = new QPossibleValueTranslator(runBackendStepInput.getInstance(), runBackendStepInput.getSession());
+               possibleValueTranslator = new QPossibleValueTranslator(QContext.getQInstance(), QContext.getQSession());
             }
 
-            possibleValueTranslator.translatePossibleValuesInRecords(runBackendStepInput.getInstance().getTable(destinationTableName), runBackendStepOutput.getRecords());
+            possibleValueTranslator.translatePossibleValuesInRecords(QContext.getQInstance().getTable(destinationTableName), runBackendStepOutput.getRecords());
+         }
+      }
+
+      if(Boolean.parseBoolean(runBackendStepInput.getValueString(LOG_TRANSFORM_RESULTS)))
+      {
+         logResults(runBackendStepInput, config);
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** Log results of transformation
+    **
+    *******************************************************************************/
+   protected void logResults(RunBackendStepInput runBackendStepInput, SyncProcessConfig syncProcessConfig)
+   {
+      String timezone = QContext.getQSession().getValue(QSession.VALUE_KEY_USER_TIMEZONE);
+      if(timezone == null)
+      {
+         timezone = QContext.getQInstance().getDefaultTimeZoneId();
+      }
+      Instant lastRunTime = Instant.now();
+      if(runBackendStepInput.getBasepullLastRunTime() != null)
+      {
+         lastRunTime = runBackendStepInput.getBasepullLastRunTime();
+      }
+
+      ZonedDateTime dateTime = lastRunTime.atZone(ZoneId.of(timezone));
+
+      if(syncProcessConfig.performInserts)
+      {
+         if(okToInsert.getCount() == 0)
+         {
+            LOG.info("No Records were found to insert since " + QValueFormatter.formatDateTimeWithZone(dateTime) + ".");
+         }
+         else
+         {
+            String pluralized = okToInsert.getCount() > 1 ? " Records were " : " Record was ";
+            LOG.info(okToInsert.getCount() + pluralized + " found to insert since " + QValueFormatter.formatDateTimeWithZone(dateTime) + ".", logPair("primaryKeys", okToInsert.getPrimaryKeys()));
+         }
+      }
+
+      if(syncProcessConfig.performUpdates)
+      {
+         if(okToUpdate.getCount() == 0)
+         {
+            LOG.info("No Records were found to update since " + QValueFormatter.formatDateTimeWithZone(dateTime) + ".");
+         }
+         else
+         {
+            String pluralized = okToUpdate.getCount() > 1 ? " Records were " : " Record was ";
+            LOG.info(okToUpdate.getCount() + pluralized + " found to update since " + QValueFormatter.formatDateTimeWithZone(dateTime) + ".", logPair("primaryKeys", okToInsert.getPrimaryKeys()));
          }
       }
    }
 
 
+
    /*******************************************************************************
     ** Given a source record, extract what we'll use as its key from it.
     **

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/locks/ProcessLockUtils.java

@@ -433,6 +433,8 @@ public class ProcessLockUtils
          {
             throw (new QException("Error deleting processLock record: " + deleteOutput.getRecordsWithErrors().get(0).getErrorsAsString()));
          }
+
+         LOG.info("Released process lock", logPair("id", processLock.getId()), logPair("key", processLock.getKey()), logPair("typeId", processLock.getProcessLockTypeId()), logPair("details", processLock.getDetails()));
       }
       catch(QException e)
       {

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/utils/GeneralProcessUtils.java

@@ -32,6 +32,7 @@ import java.util.function.Consumer;
 import com.kingsrook.qqq.backend.core.actions.tables.CountAction;
 import com.kingsrook.qqq.backend.core.actions.tables.GetAction;
 import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
+import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.exceptions.QUserFacingException;
 import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
@@ -515,7 +516,7 @@ public class GeneralProcessUtils
     *******************************************************************************/
    public static Integer validateSingleSelectedId(RunBackendStepInput runBackendStepInput, String tableName) throws QException
    {
-      String tableLabel = runBackendStepInput.getInstance().getTable(tableName).getLabel();
+      String tableLabel = QContext.getQInstance().getTable(tableName).getLabel();
 
       ////////////////////////////////////////////////////
       // Get the selected recordId and verify we only 1 //