Merge branch 'release/0.14.0'

# Conflicts:
#	qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidator.java
#	qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidatorTest.java
#	qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinImplementation.java
#	qqq-middleware-javalin/src/test/java/com/kingsrook/qqq/backend/javalin/QJavalinImplementationTest.java
#	qqq-middleware-javalin/src/test/java/com/kingsrook/qqq/backend/javalin/TestUtils.java

.circleci/adjust-pom-version.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+
+if [ -z "$CIRCLE_BRANCH" ] && [ -z "$CIRCLE_TAG" ]; then
+   echo "Error:  env vars CIRCLE_BRANCH and CIRCLE_TAG were not set."
+   exit 1;
+fi
+
+if [ "$CIRCLE_BRANCH" == "dev" ] || [ "$CIRCLE_BRANCH" == "staging" ] || [ "$CIRCLE_BRANCH" == "main" ]; then
+   echo "On a primary branch [$CIRCLE_BRANCH] - will not edit the pom version.";
+   exit 0;
+fi
+
+if [ -n "$CIRCLE_BRANCH" ]; then
+   SLUG=$(echo $CIRCLE_BRANCH | sed 's/[^a-zA-Z0-9]/-/g')
+else
+   SLUG=$(echo $CIRCLE_TAG | sed 's/^snapshot-//g')
+fi
+
+POM=$(dirname $0)/../pom.xml
+
+echo "Updating $POM <revision> to: $SLUG-SNAPSHOT"
+sed -i "s/<revision>.*/<revision>$SLUG-SNAPSHOT<\/revision>/" $POM
+git diff $POM

.circleci/config.yml

@@ -26,6 +26,7 @@ commands:
             sudo rm /etc/alternatives/java
             sudo ln -s /usr/lib/jvm/java-17-openjdk-amd64/bin/java /etc/alternatives/java
       - run:
+          ## used by jacoco uncovered class reporting in pom.xml
           name: Install html2text
           command: |
             sudo apt-get update
@@ -51,10 +52,18 @@ commands:
           module: qqq-backend-module-filesystem
       - store_jacoco_site:
           module: qqq-backend-module-rdbms
+      - store_jacoco_site:
+          module: qqq-backend-module-api
+      - store_jacoco_site:
+          module: qqq-middleware-api
       - store_jacoco_site:
           module: qqq-middleware-javalin
       - store_jacoco_site:
           module: qqq-middleware-picocli
+      - store_jacoco_site:
+          module: qqq-middleware-slack
+      - store_jacoco_site:
+          module: qqq-language-support-javascript
       - store_jacoco_site:
           module: qqq-sample-project
       - run:
@@ -65,11 +74,6 @@ commands:
           when: always
       - store_test_results:
           path: ~/test-results
-      - run:
-          name: Find Un-tested Classes
-          command: |
-            set +o pipefail && for i in */target/site/jacoco/*/index.html; do html2text -width 500 -nobs $i | sed '1,/^Total/d;' | grep -v Created | sed 's/ \+/ /g' | sed 's/ [[:digit:]]$//' | grep -v 0$ | cut -d' ' -f1; done
-          when: always
       - save_cache:
           paths:
             - ~/.m2
@@ -78,6 +82,10 @@ commands:
   mvn_jar_deploy:
     steps:
       - checkout
+      - run:
+          name: Adjust pom version
+          command: |
+            .circleci/adjust-pom-version.sh
       - restore_cache:
           keys:
             - v1-dependencies-{{ checksum "pom.xml" }}

pom.xml

@@ -44,7 +44,7 @@
    </modules>
 
    <properties>
-      <revision>0.13.0</revision>
+      <revision>0.14.0</revision>
 
       <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
       <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
@@ -206,6 +206,63 @@
                <skipUpdateVersion>true</skipUpdateVersion>
             </configuration>
          </plugin>
+         <plugin>
+            <artifactId>exec-maven-plugin</artifactId>
+            <groupId>org.codehaus.mojo</groupId>
+            <version>3.0.0</version>
+            <executions>
+               <execution>
+                  <id>test-coverage-summary</id>
+                  <phase>verify</phase>
+                  <goals>
+                     <goal>exec</goal>
+                  </goals>
+                  <configuration>
+                     <executable>sh</executable>
+                     <arguments>
+                        <argument>-c</argument>
+                        <argument>
+                           <![CDATA[
+if [ ! -e target/site/jacoco/index.html ]; then
+   echo "No jacoco coverage report here.";
+   exit;
+fi
+
+echo
+echo "Jacoco coverage summary report:"
+echo "   See also target/site/jacoco/index.html"
+echo "   and https://www.jacoco.org/jacoco/trunk/doc/counters.html"
+echo "------------------------------------------------------------"
+
+which xpath > /dev/null 2>&1
+if [ "$?" == "0" ]; then
+   echo "Element\nInstructions Missed\nInstruction Coverage\nBranches Missed\nBranch Coverage\nComplexity Missed\nComplexity Hit\nLines Missed\nLines Hit\nMethods Missed\nMethods Hit\nClasses Missed\nClasses Hit\n" > /tmp/$$.headers
+   xpath -n -q -e '/html/body/table/tfoot/tr[1]/td/text()' target/site/jacoco/index.html > /tmp/$$.values
+   paste /tmp/$$.headers /tmp/$$.values | tail +2 | awk -v FS='\t' '{printf("%-20s %s\n",$1,$2)}'
+   rm /tmp/$$.headers /tmp/$$.values
+else
+   echo "xpath is not installed.  Jacoco coverage summary will not be produced here...";
+fi
+
+which xpath > /dev/null 2>&1
+if [ "$?" == "0" ]; then
+   echo "Untested classes, per Jacoco:"
+   echo "-----------------------------"
+   for i in target/site/jacoco/*/index.html; do
+      html2text -width 500 -nobs $i | sed '1,/^Total/d;' | grep -v Created | sed 's/ \+/ /g' | sed 's/ [[:digit:]]$//' | grep -v 0$ | cut -d' ' -f1;
+   done;
+   echo
+else
+   echo "html2text is not installed.  Untested classes from Jacoco will not be printed here...";
+fi
+
+                           ]]>
+                        </argument>
+                     </arguments>
+                  </configuration>
+               </execution>
+            </executions>
+         </plugin>
          <plugin>
             <groupId>org.jacoco</groupId>
             <artifactId>jacoco-maven-plugin</artifactId>
@@ -249,56 +306,14 @@
                </execution>
                <execution>
                   <id>post-unit-test</id>
-                  <phase>verify</phase>
+                  <!-- <phase>verify</phase> -->
+                  <phase>post-integration-test</phase>
                   <goals>
                      <goal>report</goal>
                   </goals>
                </execution>
             </executions>
          </plugin>
-         <plugin>
-            <artifactId>exec-maven-plugin</artifactId>
-            <groupId>org.codehaus.mojo</groupId>
-            <version>3.0.0</version>
-            <executions>
-               <execution>
-                  <id>test-coverage-summary</id>
-                  <phase>verify</phase>
-                  <goals>
-                     <goal>exec</goal>
-                  </goals>
-                  <configuration>
-                     <executable>sh</executable>
-                     <arguments>
-                        <argument>-c</argument>
-                        <argument>
-                           <![CDATA[
-if [ ! -e target/site/jacoco/index.html ]; then
-   echo "No jacoco coverage report here.";
-   exit;
-fi
-
-echo
-echo "Jacoco coverage summary report:"
-echo "   See also target/site/jacoco/index.html"
-echo "   and https://www.jacoco.org/jacoco/trunk/doc/counters.html"
-echo "------------------------------------------------------------"
-which xpath > /dev/null 2>&1
-if [ "$?" == "0" ]; then
-   echo "Element\nInstructions Missed\nInstruction Coverage\nBranches Missed\nBranch Coverage\nComplexity Missed\nComplexity Hit\nLines Missed\nLines Hit\nMethods Missed\nMethods Hit\nClasses Missed\nClasses Hit\n" > /tmp/$$.headers
-   xpath -n -q -e '/html/body/table/tfoot/tr[1]/td/text()' target/site/jacoco/index.html > /tmp/$$.values
-   paste /tmp/$$.headers /tmp/$$.values | tail +2 | awk -v FS='\t' '{printf("%-20s %s\n",$1,$2)}'
-   rm /tmp/$$.headers /tmp/$$.values
-else
-   echo "xpath is not installed.  Jacoco coverage summary will not be produced here..";
-fi
-                           ]]>
-                        </argument>
-                     </arguments>
-                  </configuration>
-               </execution>
-            </executions>
-         </plugin>
       </plugins>
    </build>
 

qqq-backend-core/pom.xml

@@ -56,6 +56,10 @@
          <groupId>software.amazon.awssdk</groupId>
          <artifactId>quicksight</artifactId>
       </dependency>
+      <dependency>
+         <groupId>software.amazon.awssdk</groupId>
+         <artifactId>apigateway</artifactId>
+      </dependency>
       <dependency>
          <groupId>com.amazonaws</groupId>
          <artifactId>aws-java-sdk-secretsmanager</artifactId>
@@ -100,8 +104,18 @@
       </dependency>
       <dependency>
          <groupId>com.auth0</groupId>
-         <artifactId>mvc-auth-commons</artifactId>
-         <version>1.9.2</version>
+         <artifactId>auth0</artifactId>
+         <version>2.1.0</version>
+      </dependency>
+      <dependency>
+         <groupId>com.auth0</groupId>
+         <artifactId>java-jwt</artifactId>
+         <version>4.4.0</version>
+      </dependency>
+      <dependency>
+         <groupId>com.auth0</groupId>
+         <artifactId>jwks-rsa</artifactId>
+         <version>0.22.0</version>
       </dependency>
       <dependency>
          <groupId>io.github.cdimascio</groupId>

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/async/AsyncJobManager.java

@@ -33,11 +33,13 @@ import java.util.concurrent.TimeoutException;
 import com.kingsrook.qqq.backend.core.context.CapturedContext;
 import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.exceptions.QUserFacingException;
 import com.kingsrook.qqq.backend.core.logging.QLogger;
 import com.kingsrook.qqq.backend.core.state.InMemoryStateProvider;
 import com.kingsrook.qqq.backend.core.state.StateProviderInterface;
 import com.kingsrook.qqq.backend.core.state.StateType;
 import com.kingsrook.qqq.backend.core.state.UUIDAndTypeStateKey;
+import org.apache.logging.log4j.Level;
 import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
 
 
@@ -151,7 +153,11 @@ public class AsyncJobManager
          asyncJobStatus.setState(AsyncJobState.ERROR);
          asyncJobStatus.setCaughtException(e);
          getStateProvider().put(uuidAndTypeStateKey, asyncJobStatus);
-         LOG.warn("Job ended with an exception", e, logPair("jobId", uuidAndTypeStateKey.getUuid()));
+
+         //////////////////////////////////////////////////////
+         // if user facing, just log an info, warn otherwise //
+         //////////////////////////////////////////////////////
+         LOG.log((e instanceof QUserFacingException) ? Level.INFO : Level.WARN, "Job ended with an exception", e, logPair("jobId", uuidAndTypeStateKey.getUuid()));
          throw (new CompletionException(e));
       }
       finally

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/audits/AuditAction.java

@@ -215,10 +215,13 @@ public class AuditAction extends AbstractQActionFunction<AuditInput, AuditOutput
                }
             }
 
-            insertInput = new InsertInput();
-            insertInput.setTableName("auditDetail");
-            insertInput.setRecords(auditDetailRecords);
-            new InsertAction().execute(insertInput);
+            if(!auditDetailRecords.isEmpty())
+            {
+               insertInput = new InsertInput();
+               insertInput.setTableName("auditDetail");
+               insertInput.setRecords(auditDetailRecords);
+               new InsertAction().execute(insertInput);
+            }
          }
          catch(Exception e)
          {

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/audits/DMLAuditAction.java

@@ -79,7 +79,6 @@ public class DMLAuditAction extends AbstractQActionFunction<DMLAuditInput, DMLAu
    {
       DMLAuditOutput           output           = new DMLAuditOutput();
       AbstractTableActionInput tableActionInput = input.getTableActionInput();
-      List<QRecord>            recordList       = input.getRecordList();
       List<QRecord>            oldRecordList    = input.getOldRecordList();
       QTableMetaData           table            = tableActionInput.getTable();
       long                     start            = System.currentTimeMillis();
@@ -87,6 +86,9 @@ public class DMLAuditAction extends AbstractQActionFunction<DMLAuditInput, DMLAu
 
       try
       {
+         List<QRecord> recordList = CollectionUtils.nonNullList(input.getRecordList()).stream()
+            .filter(r -> CollectionUtils.nullSafeIsEmpty(r.getErrors())).toList();
+
          AuditLevel auditLevel = getAuditLevel(tableActionInput);
          if(auditLevel == null || auditLevel.equals(AuditLevel.NONE) || CollectionUtils.nullSafeIsEmpty(recordList))
          {
@@ -96,8 +98,13 @@ public class DMLAuditAction extends AbstractQActionFunction<DMLAuditInput, DMLAu
             return (output);
          }
 
-         String                        contextSuffix = "";
-         Optional<AbstractActionInput> actionInput   = QContext.getFirstActionInStack();
+         String contextSuffix = "";
+         if(StringUtils.hasContent(input.getAuditContext()))
+         {
+            contextSuffix = " " + input.getAuditContext();
+         }
+
+         Optional<AbstractActionInput> actionInput = QContext.getFirstActionInStack();
          if(actionInput.isPresent() && actionInput.get() instanceof RunProcessInput runProcessInput)
          {
             String           processName = runProcessInput.getProcessName();
@@ -187,32 +194,57 @@ public class DMLAuditAction extends AbstractQActionFunction<DMLAuditInput, DMLAu
                         continue;
                      }
 
-                     String formattedValue = getFormattedValueForAuditDetail(record, fieldName, field, value);
-                     detailRecord = new QRecord().withValue("message", "Set " + field.getLabel() + " to " + formattedValue);
-                     detailRecord.withValue("newValue", formattedValue);
+                     if(field.getType().equals(QFieldType.BLOB))
+                     {
+                        detailRecord = new QRecord().withValue("message", "Set " + field.getLabel());
+                     }
+                     else
+                     {
+                        String formattedValue = getFormattedValueForAuditDetail(record, fieldName, field, value);
+                        detailRecord = new QRecord().withValue("message", "Set " + field.getLabel() + " to " + formattedValue);
+                        detailRecord.withValue("newValue", formattedValue);
+                     }
                   }
                   else
                   {
                      if(!Objects.equals(oldValue, value))
                      {
-                        String formattedValue    = getFormattedValueForAuditDetail(record, fieldName, field, value);
-                        String formattedOldValue = getFormattedValueForAuditDetail(oldRecord, fieldName, field, oldValue);
-
-                        if(oldValue == null)
+                        if(field.getType().equals(QFieldType.BLOB))
                         {
-                           detailRecord = new QRecord().withValue("message", "Set " + field.getLabel() + " to " + formatFormattedValueForDetailMessage(field, formattedValue));
-                           detailRecord.withValue("newValue", formattedValue);
-                        }
-                        else if(value == null)
-                        {
-                           detailRecord = new QRecord().withValue("message", "Removed " + formatFormattedValueForDetailMessage(field, formattedOldValue) + " from " + field.getLabel());
-                           detailRecord.withValue("oldValue", formattedOldValue);
+                           if(oldValue == null)
+                           {
+                              detailRecord = new QRecord().withValue("message", "Set " + field.getLabel());
+                           }
+                           else if(value == null)
+                           {
+                              detailRecord = new QRecord().withValue("message", "Removed " + field.getLabel());
+                           }
+                           else
+                           {
+                              detailRecord = new QRecord().withValue("message", "Changed " + field.getLabel());
+                           }
                         }
                         else
                         {
-                           detailRecord = new QRecord().withValue("message", "Changed " + field.getLabel() + " from " + formatFormattedValueForDetailMessage(field, formattedOldValue) + " to " + formatFormattedValueForDetailMessage(field, formattedValue));
-                           detailRecord.withValue("oldValue", formattedOldValue);
-                           detailRecord.withValue("newValue", formattedValue);
+                           String formattedValue    = getFormattedValueForAuditDetail(record, fieldName, field, value);
+                           String formattedOldValue = getFormattedValueForAuditDetail(oldRecord, fieldName, field, oldValue);
+
+                           if(oldValue == null)
+                           {
+                              detailRecord = new QRecord().withValue("message", "Set " + field.getLabel() + " to " + formatFormattedValueForDetailMessage(field, formattedValue));
+                              detailRecord.withValue("newValue", formattedValue);
+                           }
+                           else if(value == null)
+                           {
+                              detailRecord = new QRecord().withValue("message", "Removed " + formatFormattedValueForDetailMessage(field, formattedOldValue) + " from " + field.getLabel());
+                              detailRecord.withValue("oldValue", formattedOldValue);
+                           }
+                           else
+                           {
+                              detailRecord = new QRecord().withValue("message", "Changed " + field.getLabel() + " from " + formatFormattedValueForDetailMessage(field, formattedOldValue) + " to " + formatFormattedValueForDetailMessage(field, formattedValue));
+                              detailRecord.withValue("oldValue", formattedOldValue);
+                              detailRecord.withValue("newValue", formattedValue);
+                           }
                         }
                      }
                   }
@@ -239,7 +271,7 @@ public class DMLAuditAction extends AbstractQActionFunction<DMLAuditInput, DMLAu
          // new AuditAction().executeAsync(auditInput); // todo async??? maybe get that from rules???
          new AuditAction().execute(auditInput);
          long end = System.currentTimeMillis();
-         LOG.debug("Audit performance", logPair("auditLevel", String.valueOf(auditLevel)), logPair("recordCount", recordList.size()), logPair("millis", (end - start)));
+         LOG.trace("Audit performance", logPair("auditLevel", String.valueOf(auditLevel)), logPair("recordCount", recordList.size()), logPair("millis", (end - start)));
       }
       catch(Exception e)
       {

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/automation/polling/PollingAutomationPerTableRunner.java

@@ -278,6 +278,7 @@ public class PollingAutomationPerTableRunner implements Runnable
                .withPriority(record.getValueInteger("priority"))
                .withCodeReference(new QCodeReference(RunRecordScriptAutomationHandler.class))
                .withValues(MapBuilder.of("scriptId", record.getValue("scriptId")))
+               .withIncludeRecordAssociations(true)
             );
          }
       }
@@ -364,26 +365,29 @@ public class PollingAutomationPerTableRunner implements Runnable
       QueryInput queryInput = new QueryInput();
       queryInput.setTableName(table.getName());
 
+      ///////////////////////////////////////////////////////////////////////////////////////
+      // set up a filter that is for the primary keys IN the list that we identified above //
+      ///////////////////////////////////////////////////////////////////////////////////////
       QQueryFilter filter = new QQueryFilter();
+      filter.addCriteria(new QFilterCriteria(table.getPrimaryKeyField(), QCriteriaOperator.IN, records.stream().map(r -> r.getValue(table.getPrimaryKeyField())).toList()));
 
-      /////////////////////////////////////////////////////////////////////////////////////////////////////
-      // copy filter criteria from the action's filter to a new filter that we'll run here.              //
-      // Critically - don't modify the filter object on the action!  as that object has a long lifespan. //
-      /////////////////////////////////////////////////////////////////////////////////////////////////////
       if(action.getFilter() != null)
       {
-         if(action.getFilter().getCriteria() != null)
-         {
-            action.getFilter().getCriteria().forEach(filter::addCriteria);
-         }
+         /////////////////////////////////////////////////////////////////////////////////////////////////////
+         // if the action defines a filter of its own, add that to the filter we'll run now as a sub-filter //
+         // not entirely clear if this needs to be a clone, but, it feels safe and cheap enough             //
+         /////////////////////////////////////////////////////////////////////////////////////////////////////
+         filter.addSubFilter(action.getFilter().clone());
+
+         ////////////////////////////////////////////////////////////////////////////////////////////////////////////
+         // we also want to set order-bys from the action into our filter (since they only apply at the top-level) //
+         ////////////////////////////////////////////////////////////////////////////////////////////////////////////
          if(action.getFilter().getOrderBys() != null)
          {
             action.getFilter().getOrderBys().forEach(filter::addOrderBy);
          }
       }
 
-      filter.addCriteria(new QFilterCriteria(table.getPrimaryKeyField(), QCriteriaOperator.IN, records.stream().map(r -> r.getValue(table.getPrimaryKeyField())).toList()));
-
       /////////////////////////////////////////////////////////////////////////////////////////////
       // always add order-by the primary key, to give more predictable/consistent results        //
       // todo - in future - if this becomes a source of slowness, make this a config to opt-out? //
@@ -392,6 +396,8 @@ public class PollingAutomationPerTableRunner implements Runnable
 
       queryInput.setFilter(filter);
 
+      queryInput.setIncludeAssociations(action.getIncludeRecordAssociations());
+
       return (new QueryAction().execute(queryInput).getRecords());
    }
 

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/AbstractPostDeleteCustomizer.java

@@ -0,0 +1,82 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.actions.customizers;
+
+
+import java.util.List;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+
+
+/*******************************************************************************
+ ** Abstract class that a table can specify an implementation of, to provide
+ ** custom actions after a delete takes place.
+ **
+ ** General implementation would be, to iterate over the records (ones which didn't
+ ** have a delete error), and look at their values:
+ ** - possibly adding Errors (`addError`) or Warnings (`addWarning`) to the records?
+ ** - possibly throwing an exception - though doing so won't stop the delete, and instead
+ **   will just set a warning on all of the deleted records...
+ ** - doing "whatever else" you may want to do.
+ ** - returning the list of records (can be the input list) that you want to go back
+ **   to the caller - this is how errors and warnings are propagated .
+ **
+ ** Note that the full deleteInput is available as a field in this class.
+ **
+ ** A future enhancement here may be to take (as fields in this class) the list of
+ ** records that the delete action marked in error - the user might want to do
+ ** something special with them (idk, try some other way to delete them?)
+ *******************************************************************************/
+public abstract class AbstractPostDeleteCustomizer
+{
+   protected DeleteInput deleteInput;
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public abstract List<QRecord> apply(List<QRecord> records) throws QException;
+
+
+
+   /*******************************************************************************
+    ** Getter for deleteInput
+    **
+    *******************************************************************************/
+   public DeleteInput getDeleteInput()
+   {
+      return deleteInput;
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for deleteInput
+    **
+    *******************************************************************************/
+   public void setDeleteInput(DeleteInput deleteInput)
+   {
+      this.deleteInput = deleteInput;
+   }
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/AbstractPostInsertCustomizer.java

@@ -23,12 +23,24 @@ package com.kingsrook.qqq.backend.core.actions.customizers;
 
 
 import java.util.List;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
 
 
 /*******************************************************************************
+ ** Abstract class that a table can specify an implementation of, to provide
+ ** custom actions after an insert takes place.
  **
+ ** General implementation would be, to iterate over the records (the outputs of
+ ** the insert action), and look at their values:
+ ** - possibly adding Errors (`addError`) or Warnings (`addWarning`) to the records
+ ** - possibly throwing an exception - though doing so won't stop the update, and instead
+ **   will just set a warning on all of the updated records...
+ ** - doing "whatever else" you may want to do.
+ ** - returning the list of records (can be the input list) that you want to go back to the caller.
+ **
+ ** Note that the full insertInput is available as a field in this class.
  *******************************************************************************/
 public abstract class AbstractPostInsertCustomizer
 {
@@ -39,7 +51,7 @@ public abstract class AbstractPostInsertCustomizer
    /*******************************************************************************
     **
     *******************************************************************************/
-   public abstract List<QRecord> apply(List<QRecord> records);
+   public abstract List<QRecord> apply(List<QRecord> records) throws QException;
 
 
 

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/AbstractPostUpdateCustomizer.java

@@ -0,0 +1,130 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.actions.customizers;
+
+
+import java.io.Serializable;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+
+
+/*******************************************************************************
+ ** Abstract class that a table can specify an implementation of, to provide
+ ** custom actions after an update takes place.
+ **
+ ** General implementation would be, to iterate over the records (the outputs of
+ ** the update action), and look at their values:
+ ** - possibly adding Errors (`addError`) or Warnings (`addWarning`) to the records?
+ ** - possibly throwing an exception - though doing so won't stop the update, and instead
+ **   will just set a warning on all of the updated records...
+ ** - doing "whatever else" you may want to do.
+ ** - returning the list of records (can be the input list) that you want to go back to the caller.
+ **
+ ** Note that the full updateInput is available as a field in this class, and the
+ ** "old records" (e.g., with values freshly fetched from the backend) will be
+ ** available (if the backend supports it) - both as a list (`getOldRecordList`)
+ ** and as a memoized (by this class) map of primaryKey to record (`getOldRecordMap`).
+ *******************************************************************************/
+public abstract class AbstractPostUpdateCustomizer
+{
+   protected UpdateInput   updateInput;
+   protected List<QRecord> oldRecordList;
+
+   private Map<Serializable, QRecord> oldRecordMap = null;
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public abstract List<QRecord> apply(List<QRecord> records) throws QException;
+
+
+
+   /*******************************************************************************
+    ** Getter for updateInput
+    **
+    *******************************************************************************/
+   public UpdateInput getUpdateInput()
+   {
+      return updateInput;
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for updateInput
+    **
+    *******************************************************************************/
+   public void setUpdateInput(UpdateInput updateInput)
+   {
+      this.updateInput = updateInput;
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public void setOldRecordList(List<QRecord> oldRecordList)
+   {
+      this.oldRecordList = oldRecordList;
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public List<QRecord> getOldRecordList()
+   {
+      return oldRecordList;
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   protected Map<Serializable, QRecord> getOldRecordMap()
+   {
+      if(oldRecordMap == null)
+      {
+         oldRecordMap = new HashMap<>();
+
+         if(oldRecordList != null && updateInput != null)
+         {
+            for(QRecord qRecord : oldRecordList)
+            {
+               oldRecordMap.put(qRecord.getValue(updateInput.getTable().getPrimaryKeyField()), qRecord);
+            }
+         }
+      }
+
+      return (oldRecordMap);
+   }
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/AbstractPreDeleteCustomizer.java

@@ -0,0 +1,119 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.actions.customizers;
+
+
+import java.util.List;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+
+
+/*******************************************************************************
+ ** Abstract class that a table can specify an implementation of, to provide
+ ** custom actions before a delete takes place.
+ **
+ ** It's important for implementations to be aware of the isPreview field, which
+ ** is set to true when the code is running to give users advice, e.g., on a review
+ ** screen - vs. being false when the action is ACTUALLY happening.  So, if you're doing
+ ** things like storing data, you don't want to do that if isPreview is true!!
+ **
+ ** General implementation would be, to iterate over the records (which the DeleteAction
+ ** would look up based on the inputs to the delete action), and look at their values:
+ ** - possibly adding Errors (`addError`) or Warnings (`addWarning`) to the records
+ ** - possibly throwing an exception - if you really don't want the delete operation to continue.
+ ** - doing "whatever else" you may want to do.
+ ** - returning the list of records (can be the input list) - this is how errors
+ **   and warnings are propagated to the DeleteAction.  Note that any records with
+ **   an error will NOT proceed to the backend's delete interface - but those with
+ **   warnings will.
+ **
+ ** Note that the full deleteInput is available as a field in this class.
+ **
+ *******************************************************************************/
+public abstract class AbstractPreDeleteCustomizer
+{
+   protected DeleteInput deleteInput;
+
+   protected boolean isPreview = false;
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public abstract List<QRecord> apply(List<QRecord> records) throws QException;
+
+
+
+   /*******************************************************************************
+    ** Getter for deleteInput
+    **
+    *******************************************************************************/
+   public DeleteInput getDeleteInput()
+   {
+      return deleteInput;
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for deleteInput
+    **
+    *******************************************************************************/
+   public void setDeleteInput(DeleteInput deleteInput)
+   {
+      this.deleteInput = deleteInput;
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for isPreview
+    *******************************************************************************/
+   public boolean getIsPreview()
+   {
+      return (this.isPreview);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for isPreview
+    *******************************************************************************/
+   public void setIsPreview(boolean isPreview)
+   {
+      this.isPreview = isPreview;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for isPreview
+    *******************************************************************************/
+   public AbstractPreDeleteCustomizer withIsPreview(boolean isPreview)
+   {
+      this.isPreview = isPreview;
+      return (this);
+   }
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/AbstractPreInsertCustomizer.java

@@ -0,0 +1,116 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.actions.customizers;
+
+
+import java.util.List;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+
+
+/*******************************************************************************
+ ** Abstract class that a table can specify an implementation of, to provide
+ ** custom actions before an insert takes place.
+ **
+ ** It's important for implementations to be aware of the isPreview field, which
+ ** is set to true when the code is running to give users advice, e.g., on a review
+ ** screen - vs. being false when the action is ACTUALLY happening.  So, if you're doing
+ ** things like storing data, you don't want to do that if isPreview is true!!
+ **
+ ** General implementation would be, to iterate over the records (the inputs to
+ ** the insert action), and look at their values:
+ ** - possibly adding Errors (`addError`) or Warnings (`addWarning`) to the records
+ ** - possibly manipulating values (`setValue`)
+ ** - possibly throwing an exception - if you really don't want the insert operation to continue.
+ ** - doing "whatever else" you may want to do.
+ ** - returning the list of records (can be the input list) that you want to go on to the backend implementation class.
+ **
+ ** Note that the full insertInput is available as a field in this class.
+ *******************************************************************************/
+public abstract class AbstractPreInsertCustomizer
+{
+   protected InsertInput insertInput;
+
+   protected boolean isPreview = false;
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public abstract List<QRecord> apply(List<QRecord> records) throws QException;
+
+
+
+   /*******************************************************************************
+    ** Getter for insertInput
+    **
+    *******************************************************************************/
+   public InsertInput getInsertInput()
+   {
+      return insertInput;
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for insertInput
+    **
+    *******************************************************************************/
+   public void setInsertInput(InsertInput insertInput)
+   {
+      this.insertInput = insertInput;
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for isPreview
+    *******************************************************************************/
+   public boolean getIsPreview()
+   {
+      return (this.isPreview);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for isPreview
+    *******************************************************************************/
+   public void setIsPreview(boolean isPreview)
+   {
+      this.isPreview = isPreview;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for isPreview
+    *******************************************************************************/
+   public AbstractPreInsertCustomizer withIsPreview(boolean isPreview)
+   {
+      this.isPreview = isPreview;
+      return (this);
+   }
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/AbstractPreUpdateCustomizer.java

@@ -0,0 +1,163 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.actions.customizers;
+
+
+import java.io.Serializable;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+
+
+/*******************************************************************************
+ ** Abstract class that a table can specify an implementation of, to provide
+ ** custom actions before an update takes place.
+ **
+ ** It's important for implementations to be aware of the isPreview field, which
+ ** is set to true when the code is running to give users advice, e.g., on a review
+ ** screen - vs. being false when the action is ACTUALLY happening.  So, if you're doing
+ ** things like storing data, you don't want to do that if isPreview is true!!
+ **
+ ** General implementation would be, to iterate over the records (the inputs to
+ ** the update action), and look at their values:
+ ** - possibly adding Errors (`addError`) or Warnings (`addWarning`) to the records
+ ** - possibly manipulating values (`setValue`)
+ ** - possibly throwing an exception - if you really don't want the update operation to continue.
+ ** - doing "whatever else" you may want to do.
+ ** - returning the list of records (can be the input list) that you want to go on to the backend implementation class.
+ **
+ ** Note that the full updateInput is available as a field in this class, and the
+ ** "old records" (e.g., with values freshly fetched from the backend) will be
+ ** available (if the backend supports it) - both as a list (`getOldRecordList`)
+ ** and as a memoized (by this class) map of primaryKey to record (`getOldRecordMap`).
+ *******************************************************************************/
+public abstract class AbstractPreUpdateCustomizer
+{
+   protected UpdateInput   updateInput;
+   protected List<QRecord> oldRecordList;
+   protected boolean       isPreview = false;
+
+   private Map<Serializable, QRecord> oldRecordMap = null;
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public abstract List<QRecord> apply(List<QRecord> records) throws QException;
+
+
+
+   /*******************************************************************************
+    ** Getter for updateInput
+    **
+    *******************************************************************************/
+   public UpdateInput getUpdateInput()
+   {
+      return updateInput;
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for updateInput
+    **
+    *******************************************************************************/
+   public void setUpdateInput(UpdateInput updateInput)
+   {
+      this.updateInput = updateInput;
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public void setOldRecordList(List<QRecord> oldRecordList)
+   {
+      this.oldRecordList = oldRecordList;
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public List<QRecord> getOldRecordList()
+   {
+      return oldRecordList;
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   protected Map<Serializable, QRecord> getOldRecordMap()
+   {
+      if(oldRecordMap == null)
+      {
+         oldRecordMap = new HashMap<>();
+         for(QRecord qRecord : oldRecordList)
+         {
+            oldRecordMap.put(qRecord.getValue(updateInput.getTable().getPrimaryKeyField()), qRecord);
+         }
+      }
+
+      return (oldRecordMap);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for isPreview
+    *******************************************************************************/
+   public boolean getIsPreview()
+   {
+      return (this.isPreview);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for isPreview
+    *******************************************************************************/
+   public void setIsPreview(boolean isPreview)
+   {
+      this.isPreview = isPreview;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for isPreview
+    *******************************************************************************/
+   public AbstractPreUpdateCustomizer withIsPreview(boolean isPreview)
+   {
+      this.isPreview = isPreview;
+      return (this);
+   }
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/ChildInserterPostInsertCustomizer.java

@@ -34,6 +34,9 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertOutput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+import com.kingsrook.qqq.backend.core.model.statusmessages.QStatusMessage;
+import com.kingsrook.qqq.backend.core.model.statusmessages.QWarningMessage;
+import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
 
 
 /*******************************************************************************
@@ -120,6 +123,11 @@ public abstract class ChildInserterPostInsertCustomizer extends AbstractPostInse
          InsertOutput      insertOutput           = new InsertAction().execute(insertInput);
          Iterator<QRecord> insertedRecordIterator = insertOutput.getRecords().iterator();
 
+         /////////////////////////////////////////////////////////////////////////////////
+         // check for any errors when inserting the children, if any errors were found, //
+         // then set a warning in the parent with the details of the problem            //
+         /////////////////////////////////////////////////////////////////////////////////
+
          //////////////////////////////////////////////////////////////////////////////////////////////////////
          // iterate over the original list of records again - for any that need a child (e.g., are missing   //
          // foreign key), set their foreign key to a newly inserted child's key, and add them to be updated. //
@@ -130,7 +138,21 @@ public abstract class ChildInserterPostInsertCustomizer extends AbstractPostInse
             Serializable primaryKey = record.getValue(table.getPrimaryKeyField());
             if(record.getValue(getForeignKeyFieldName()) == null)
             {
-               Serializable foreignKey = insertedRecordIterator.next().getValue(childTable.getPrimaryKeyField());
+               ///////////////////////////////////////////////////////////////////////////////////////////////////
+               // get the corresponding child record, if it has any errors, set that as a warning in the parent //
+               ///////////////////////////////////////////////////////////////////////////////////////////////////
+               QRecord childRecord = insertedRecordIterator.next();
+               if(CollectionUtils.nullSafeHasContents(childRecord.getErrors()))
+               {
+                  for(QStatusMessage error : childRecord.getErrors())
+                  {
+                     record.addWarning(new QWarningMessage("Error creating child " + childTable.getLabel() + " (" + error.toString() + ")"));
+                  }
+                  rs.add(record);
+                  continue;
+               }
+
+               Serializable foreignKey = childRecord.getValue(childTable.getPrimaryKeyField());
                recordsToUpdate.add(new QRecord().withValue(table.getPrimaryKeyField(), primaryKey).withValue(getForeignKeyFieldName(), foreignKey));
                record.setValue(getForeignKeyFieldName(), foreignKey);
                rs.add(record);

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/RecordCustomizerUtilityInterface.java

@@ -0,0 +1,149 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2023.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.actions.customizers;
+
+
+import java.io.Serializable;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Objects;
+import com.kingsrook.qqq.backend.core.context.QContext;
+import com.kingsrook.qqq.backend.core.logging.QLogger;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+import com.kingsrook.qqq.backend.core.model.statusmessages.BadInputStatusMessage;
+import com.kingsrook.qqq.backend.core.utils.StringUtils;
+import com.kingsrook.qqq.backend.core.utils.ValueUtils;
+import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
+
+
+/*******************************************************************************
+ ** Interface with utility methods that pre insert/update/delete customizers
+ ** may want to use.
+ *******************************************************************************/
+public interface RecordCustomizerUtilityInterface
+{
+   QLogger LOG = QLogger.getLogger(RecordCustomizerUtilityInterface.class);
+
+
+
+   /*******************************************************************************
+    ** Container for an old value and a new value.
+    *******************************************************************************/
+   @SuppressWarnings("checkstyle:MethodName")
+   record Change(Serializable oldValue, Serializable newValue)
+   {
+   }
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   default Map<String, Change> getChanges(String tableName, QRecord oldRecord, QRecord newRecord)
+   {
+      Map<String, Change> rs = new HashMap<>();
+
+      QTableMetaData table = QContext.getQInstance().getTable(tableName);
+      for(Map.Entry<String, Serializable> entry : newRecord.getValues().entrySet())
+      {
+         String       fieldName = entry.getKey();
+         Serializable newValue  = entry.getValue();
+         Serializable oldValue  = oldRecord.getValue(fieldName);
+
+         try
+         {
+            QFieldMetaData field         = table.getField(fieldName);
+            Serializable   newTypedValue = ValueUtils.getValueAsFieldType(field.getType(), newValue);
+            Serializable   oldTypedValue = ValueUtils.getValueAsFieldType(field.getType(), oldValue);
+
+            if(!Objects.equals(oldTypedValue, newTypedValue))
+            {
+               rs.put(fieldName, new Change(oldTypedValue, newTypedValue));
+            }
+         }
+         catch(Exception e)
+         {
+            LOG.info("Error getting a value as field's type", e, logPair("fieldName", fieldName), logPair("oldValue", oldValue), logPair("newValue", newValue));
+         }
+      }
+
+      return (rs);
+   }
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   default void errorIfNoValue(Serializable value, QRecord record, String errorMessage)
+   {
+      errorIf(!StringUtils.hasContent(ValueUtils.getValueAsString(value)), record, errorMessage);
+   }
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   default void errorIfEditedValue(QRecord oldRecord, QRecord newRecord, String fieldName, String errorMessage)
+   {
+      if(newRecord.getValues().containsKey(fieldName))
+      {
+         errorIf(isChangedValue(oldRecord.getValue(fieldName), newRecord.getValue(fieldName)), newRecord, errorMessage);
+      }
+   }
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   default boolean isChangedValue(Serializable oldValue, Serializable newValue)
+   {
+      //////////////////////////////////////////////
+      // todo - probably ... some type "coercion" //
+      //////////////////////////////////////////////
+      return (!Objects.equals(oldValue, newValue));
+   }
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   default void errorIfAnyValue(Serializable value, QRecord record, String errorMessage)
+   {
+      if(StringUtils.hasContent(ValueUtils.getValueAsString(value)))
+      {
+         record.addError(new BadInputStatusMessage(errorMessage));
+      }
+   }
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   default void errorIf(boolean condition, QRecord record, String errorMessage)
+   {
+      if(condition)
+      {
+         record.addError(new BadInputStatusMessage(errorMessage));
+      }
+   }
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/TableCustomizers.java

@@ -26,33 +26,30 @@ package com.kingsrook.qqq.backend.core.actions.customizers;
  ** Enum definition of possible table customizers - "roles" for custom code that
  ** can be applied to tables.
  **
- ** Works with TableCustomizer (singular version of this name) objects, during
- ** instance validation, to provide validation of the referenced code (and to
- ** make such validation from sub-backend-modules possible in the future).
- **
- ** The idea of the 3rd argument here is to provide a way that we can enforce
- ** the type-parameters for the custom code.  E.g., if it's a Function - how
- ** can we check at run-time that the type-params are correct?  We couldn't find
- ** how to do this "reflectively", so we can instead try to run the custom code,
- ** passing it objects of the type that this customizer expects, and a validation
- ** error will raise upon ClassCastException...  This maybe could improve!
  *******************************************************************************/
 public enum TableCustomizers
 {
-   POST_QUERY_RECORD(new TableCustomizer("postQueryRecord", AbstractPostQueryCustomizer.class)),
-   POST_INSERT_RECORD(new TableCustomizer("postInsertRecord", AbstractPostInsertCustomizer.class));
+   POST_QUERY_RECORD("postQueryRecord", AbstractPostQueryCustomizer.class),
+   PRE_INSERT_RECORD("preInsertRecord", AbstractPreInsertCustomizer.class),
+   POST_INSERT_RECORD("postInsertRecord", AbstractPostInsertCustomizer.class),
+   PRE_UPDATE_RECORD("preUpdateRecord", AbstractPreUpdateCustomizer.class),
+   POST_UPDATE_RECORD("postUpdateRecord", AbstractPostUpdateCustomizer.class),
+   PRE_DELETE_RECORD("preDeleteRecord", AbstractPreDeleteCustomizer.class),
+   POST_DELETE_RECORD("postDeleteRecord", AbstractPostDeleteCustomizer.class);
 
 
-   private final TableCustomizer tableCustomizer;
+   private final String   role;
+   private final Class<?> expectedType;
 
 
 
    /*******************************************************************************
     **
     *******************************************************************************/
-   TableCustomizers(TableCustomizer tableCustomizer)
+   TableCustomizers(String role, Class<?> expectedType)
    {
-      this.tableCustomizer = tableCustomizer;
+      this.role = role;
+      this.expectedType = expectedType;
    }
 
 
@@ -65,7 +62,7 @@ public enum TableCustomizers
    {
       for(TableCustomizers value : values())
       {
-         if(value.tableCustomizer.getRole().equals(name))
+         if(value.role.equals(name))
          {
             return (value);
          }
@@ -76,24 +73,23 @@ public enum TableCustomizers
 
 
 
-   /*******************************************************************************
-    ** Getter for tableCustomizer
-    **
-    *******************************************************************************/
-   public TableCustomizer getTableCustomizer()
-   {
-      return tableCustomizer;
-   }
-
-
-
    /*******************************************************************************
     ** get the role from the tableCustomizer
     **
     *******************************************************************************/
    public String getRole()
    {
-      return (tableCustomizer.getRole());
+      return (role);
    }
 
+
+
+   /*******************************************************************************
+    ** Getter for expectedType
+    **
+    *******************************************************************************/
+   public Class<?> getExpectedType()
+   {
+      return expectedType;
+   }
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/dashboard/AbstractHTMLWidgetRenderer.java

@@ -216,7 +216,7 @@ public abstract class AbstractHTMLWidgetRenderer extends AbstractWidgetRenderer
    /*******************************************************************************
     **
     *******************************************************************************/
-   public static String linkTableFilter(RenderWidgetInput input, String tableName, QQueryFilter filter) throws QException
+   public static String linkTableFilter(String tableName, QQueryFilter filter) throws QException
    {
       String tablePath = QContext.getQInstance().getTablePath(tableName);
       if(tablePath == null)
@@ -232,9 +232,9 @@ public abstract class AbstractHTMLWidgetRenderer extends AbstractWidgetRenderer
    /*******************************************************************************
     **
     *******************************************************************************/
-   public static String aHrefTableFilterNoOfRecords(RenderWidgetInput input, String tableName, QQueryFilter filter, Integer noOfRecords, String singularLabel, String pluralLabel) throws QException
+   public static String aHrefTableFilterNoOfRecords(String tableName, QQueryFilter filter, Integer noOfRecords, String singularLabel, String pluralLabel) throws QException
    {
-      return (aHrefTableFilterNoOfRecords(input, tableName, filter, noOfRecords, singularLabel, pluralLabel, false));
+      return (aHrefTableFilterNoOfRecords(tableName, filter, noOfRecords, singularLabel, pluralLabel, false));
    }
 
 
@@ -242,7 +242,7 @@ public abstract class AbstractHTMLWidgetRenderer extends AbstractWidgetRenderer
    /*******************************************************************************
     **
     *******************************************************************************/
-   public static String aHrefTableFilterNoOfRecords(RenderWidgetInput input, String tableName, QQueryFilter filter, Integer noOfRecords, String singularLabel, String pluralLabel, boolean onlyLinkCount) throws QException
+   public static String aHrefTableFilterNoOfRecords(String tableName, QQueryFilter filter, Integer noOfRecords, String singularLabel, String pluralLabel, boolean onlyLinkCount) throws QException
    {
       String plural      = StringUtils.plural(noOfRecords, singularLabel, pluralLabel);
       String countString = QValueFormatter.formatValue(DisplayFormat.COMMAS, noOfRecords);
@@ -253,7 +253,7 @@ public abstract class AbstractHTMLWidgetRenderer extends AbstractWidgetRenderer
          return (countString + displayText);
       }
 
-      String href = linkTableFilter(input, tableName, filter);
+      String href = linkTableFilter(tableName, filter);
       if(onlyLinkCount)
       {
          return ("<a href=\"" + href + "\">" + countString + "</a>" + displayText);
@@ -269,7 +269,7 @@ public abstract class AbstractHTMLWidgetRenderer extends AbstractWidgetRenderer
    /*******************************************************************************
     **
     *******************************************************************************/
-   public static String aHrefViewRecord(RenderWidgetInput input, String tableName, Serializable id, String linkText) throws QException
+   public static String aHrefViewRecord(String tableName, Serializable id, String linkText) throws QException
    {
       String tablePath = QContext.getQInstance().getTablePath(tableName);
       if(tablePath == null)
@@ -277,7 +277,7 @@ public abstract class AbstractHTMLWidgetRenderer extends AbstractWidgetRenderer
          return (linkText);
       }
 
-      return ("<a href=\"" + linkRecordView(input, tableName, id) + "\">" + linkText + "</a>");
+      return ("<a href=\"" + linkRecordView(tableName, id) + "\">" + linkText + "</a>");
    }
 
 
@@ -296,7 +296,7 @@ public abstract class AbstractHTMLWidgetRenderer extends AbstractWidgetRenderer
    /*******************************************************************************
     **
     *******************************************************************************/
-   public static String linkRecordView(AbstractActionInput input, String tableName, Serializable recordId) throws QException
+   public static String linkRecordView(String tableName, Serializable recordId) throws QException
    {
       String tablePath = QContext.getQInstance().getTablePath(tableName);
       if(tablePath == null)

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/dashboard/widgets/ChildRecordListRenderer.java

@@ -30,10 +30,13 @@ import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
+import com.kingsrook.qqq.backend.core.actions.tables.CountAction;
 import com.kingsrook.qqq.backend.core.actions.tables.GetAction;
 import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
+import com.kingsrook.qqq.backend.core.actions.values.QValueFormatter;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.exceptions.QNotFoundException;
+import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetOutput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
@@ -72,7 +75,7 @@ public class ChildRecordListRenderer extends AbstractWidgetRenderer
       return (new Builder(new QWidgetMetaData()
          .withName(join.getName())
          .withIsCard(true)
-         .withCodeReference(new QCodeReference(ChildRecordListRenderer.class, null))
+         .withCodeReference(new QCodeReference(ChildRecordListRenderer.class))
          .withType(WidgetType.CHILD_RECORD_LIST.getType())
          .withDefaultValue("joinName", join.getName())));
    }
@@ -158,16 +161,22 @@ public class ChildRecordListRenderer extends AbstractWidgetRenderer
    @Override
    public RenderWidgetOutput render(RenderWidgetInput input) throws QException
    {
-      String        widgetLabel = input.getQueryParams().get("widgetLabel");
-      String        joinName    = input.getQueryParams().get("joinName");
-      QJoinMetaData join        = input.getInstance().getJoin(joinName);
-      String        id          = input.getQueryParams().get("id");
+      String         widgetLabel = input.getQueryParams().get("widgetLabel");
+      String         joinName    = input.getQueryParams().get("joinName");
+      QJoinMetaData  join        = input.getInstance().getJoin(joinName);
+      String         id          = input.getQueryParams().get("id");
+      QTableMetaData leftTable   = input.getInstance().getTable(join.getLeftTable());
+      QTableMetaData rightTable  = input.getInstance().getTable(join.getRightTable());
 
       Integer maxRows = null;
       if(StringUtils.hasContent(input.getQueryParams().get("maxRows")))
       {
          maxRows = ValueUtils.getValueAsInteger(input.getQueryParams().get("maxRows"));
       }
+      else if(input.getWidgetMetaData().getDefaultValues().containsKey("maxRows"))
+      {
+         maxRows = ValueUtils.getValueAsInteger(input.getWidgetMetaData().getDefaultValues().containsKey("maxRows"));
+      }
 
       ////////////////////////////////////////////////////////
       // fetch the record that we're getting children for.  //
@@ -181,8 +190,7 @@ public class ChildRecordListRenderer extends AbstractWidgetRenderer
 
       if(record == null)
       {
-         QTableMetaData table = input.getInstance().getTable(join.getLeftTable());
-         throw (new QNotFoundException("Could not find " + (table == null ? "" : table.getLabel()) + " with primary key " + id));
+         throw (new QNotFoundException("Could not find " + (leftTable == null ? "" : leftTable.getLabel()) + " with primary key " + id));
       }
 
       ////////////////////////////////////////////////////////////////////
@@ -194,20 +202,34 @@ public class ChildRecordListRenderer extends AbstractWidgetRenderer
          filter.addCriteria(new QFilterCriteria(joinOn.getRightField(), QCriteriaOperator.EQUALS, List.of(record.getValue(joinOn.getLeftField()))));
       }
       filter.setOrderBys(join.getOrderBys());
+      filter.setLimit(maxRows);
 
       QueryInput queryInput = new QueryInput();
       queryInput.setTableName(join.getRightTable());
       queryInput.setShouldTranslatePossibleValues(true);
       queryInput.setShouldGenerateDisplayValues(true);
       queryInput.setFilter(filter);
-      queryInput.setLimit(maxRows);
       QueryOutput queryOutput = new QueryAction().execute(queryInput);
 
-      QTableMetaData table       = input.getInstance().getTable(join.getRightTable());
-      String         tablePath   = input.getInstance().getTablePath(table.getName());
-      String         viewAllLink = tablePath == null ? null : (tablePath + "?filter=" + URLEncoder.encode(JsonUtils.toJson(filter), Charset.defaultCharset()));
+      QValueFormatter.setBlobValuesToDownloadUrls(rightTable, queryOutput.getRecords());
 
-      ChildRecordListData widgetData = new ChildRecordListData(widgetLabel, queryOutput, table, tablePath, viewAllLink);
+      int totalRows = queryOutput.getRecords().size();
+      if(maxRows != null && (queryOutput.getRecords().size() == maxRows))
+      {
+         /////////////////////////////////////////////////////////////////////////////////////
+         // if the input said to only do some max, and the # of results we got is that max, //
+         // then do a count query, for displaying 1-n of <count>                            //
+         /////////////////////////////////////////////////////////////////////////////////////
+         CountInput countInput = new CountInput();
+         countInput.setTableName(join.getRightTable());
+         countInput.setFilter(filter);
+         totalRows = new CountAction().execute(countInput).getCount();
+      }
+
+      String tablePath   = input.getInstance().getTablePath(rightTable.getName());
+      String viewAllLink = tablePath == null ? null : (tablePath + "?filter=" + URLEncoder.encode(JsonUtils.toJson(filter), Charset.defaultCharset()));
+
+      ChildRecordListData widgetData = new ChildRecordListData(widgetLabel, queryOutput, rightTable, tablePath, viewAllLink, totalRows);
 
       if(BooleanUtils.isTrue(ValueUtils.getValueAsBoolean(input.getQueryParams().get("canAddChildRecord"))))
       {

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/dashboard/widgets/NoCodeWidgetVelocityUtils.java

@@ -64,14 +64,24 @@ public class NoCodeWidgetVelocityUtils
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public String icon(String iconName, String color)
+   {
+      return String.format("""
+         <span class="material-icons-round notranslate MuiIcon-root MuiIcon-fontSizeInherit" style="color: %s; position: relative; top: 6px;" aria-hidden="true">%s</span>
+         """, color, iconName);
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/
    public String helpIcon()
    {
-      return ("""
-         <span class="material-icons-round notranslate MuiIcon-root MuiIcon-fontSizeInherit" style="color: blue; position: relative; top: 6px;" aria-hidden="true">help_outline</span>
-         """);
+      return (icon("help_outline", "blue"));
    }
 
 
@@ -81,9 +91,7 @@ public class NoCodeWidgetVelocityUtils
     *******************************************************************************/
    public String errorIcon()
    {
-      return ("""
-         <span class="material-icons-round notranslate MuiIcon-root MuiIcon-fontSizeInherit" style="color: red; position: relative; top: 6px;" aria-hidden="true">error_outline</span>
-         """);
+      return (icon("error_outline", "red"));
    }
 
 
@@ -93,9 +101,7 @@ public class NoCodeWidgetVelocityUtils
     *******************************************************************************/
    public String warningIcon()
    {
-      return ("""
-         <span class="material-icons-round notranslate MuiIcon-root MuiIcon-fontSizeInherit" style="color: orange; position: relative; top: 6px;" aria-hidden="true">warning</span>
-         """);
+      return (icon("warning", "orange"));
    }
 
 
@@ -105,9 +111,7 @@ public class NoCodeWidgetVelocityUtils
     *******************************************************************************/
    public String checkIcon()
    {
-      return ("""
-         <span class="material-icons-round notranslate MuiIcon-root MuiIcon-fontSizeInherit" style="color: green; position: relative; top: 6px;" aria-hidden="true">check</span>
-         """);
+      return (icon("check", "green"));
    }
 
 
@@ -117,9 +121,7 @@ public class NoCodeWidgetVelocityUtils
     *******************************************************************************/
    public String pendingIcon()
    {
-      return ("""
-         <span class="material-icons-round notranslate MuiIcon-root MuiIcon-fontSizeInherit" style="color: #0062ff; position: relative; top: 6px;" aria-hidden="true">pending</span>
-         """);
+      return (icon("pending", "#0062ff"));
    }
 
 
@@ -288,7 +290,7 @@ public class NoCodeWidgetVelocityUtils
          WidgetCount  widgetCount = (WidgetCount) context.get(countVariableName + ".source");
          Integer      count       = ValueUtils.getValueAsInteger(context.get(countVariableName));
          QQueryFilter filter      = widgetCount.getEffectiveFilter(input);
-         return (AbstractHTMLWidgetRenderer.aHrefTableFilterNoOfRecords(null, widgetCount.getTableName(), filter, count, singular, plural));
+         return (AbstractHTMLWidgetRenderer.aHrefTableFilterNoOfRecords(widgetCount.getTableName(), filter, count, singular, plural));
       }
       catch(Exception e)
       {

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/interfaces/DeleteInterface.java

@@ -50,4 +50,13 @@ public interface DeleteInterface
       return (false);
    }
 
+   /*******************************************************************************
+    ** Specify whether this particular module's delete action can & should fetch
+    ** records before deleting them, e.g., for audits or "not-found-checks"
+    *******************************************************************************/
+   default boolean supportsPreFetchQuery()
+   {
+      return (true);
+   }
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/interfaces/UpdateInterface.java

@@ -37,4 +37,14 @@ public interface UpdateInterface
     **
     *******************************************************************************/
    UpdateOutput execute(UpdateInput updateInput) throws QException;
+
+   /*******************************************************************************
+    ** Specify whether this particular module's update action can & should fetch
+    ** records before updating them, e.g., for audits or "not-found-checks"
+    *******************************************************************************/
+   default boolean supportsPreFetchQuery()
+   {
+      return (true);
+   }
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/metadata/JoinGraph.java

@@ -0,0 +1,336 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2023.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.actions.metadata;
+
+
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Objects;
+import java.util.Set;
+import java.util.TreeSet;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.joins.QJoinMetaData;
+import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
+import com.kingsrook.qqq.backend.core.utils.StringUtils;
+
+
+/*******************************************************************************
+ ** Object to represent the graph of joins in a QQQ Instance.  e.g., all of the
+ ** connections among tables through joins.
+ *******************************************************************************/
+public class JoinGraph
+{
+
+   private Set<Edge> edges = new HashSet<>();
+
+
+
+   /*******************************************************************************
+    ** Graph edge (no graph nodes needed in here)
+    *******************************************************************************/
+   private record Edge(String joinName, String leftTable, String rightTable)
+   {
+   }
+
+
+
+   /*******************************************************************************
+    ** In this class, we are treating joins as non-directional graph edges - so -
+    ** use this class to "normalize" what may otherwise be duplicated joins in the
+    ** qInstance (e.g., A -> B and B -> A -- in the instance, those are valid, but
+    ** in our graph here, we want to consider those the same).
+    *******************************************************************************/
+   private static class NormalizedJoin
+   {
+      private String tableA;
+      private String tableB;
+      private String joinFieldA;
+      private String joinFieldB;
+
+
+
+      /*******************************************************************************
+       **
+       *******************************************************************************/
+      public NormalizedJoin(QJoinMetaData joinMetaData)
+      {
+         boolean needFlip     = false;
+         int     tableCompare = joinMetaData.getLeftTable().compareTo(joinMetaData.getRightTable());
+         if(tableCompare < 0)
+         {
+            needFlip = true;
+         }
+         else if(tableCompare == 0)
+         {
+            int fieldCompare = joinMetaData.getJoinOns().get(0).getLeftField().compareTo(joinMetaData.getJoinOns().get(0).getRightField());
+            if(fieldCompare < 0)
+            {
+               needFlip = true;
+            }
+         }
+
+         if(needFlip)
+         {
+            joinMetaData = joinMetaData.flip();
+         }
+
+         tableA = joinMetaData.getLeftTable();
+         tableB = joinMetaData.getRightTable();
+         joinFieldA = joinMetaData.getJoinOns().get(0).getLeftField();
+         joinFieldB = joinMetaData.getJoinOns().get(0).getRightField();
+      }
+
+
+
+      /*******************************************************************************
+       **
+       *******************************************************************************/
+      @Override
+      public boolean equals(Object o)
+      {
+         if(this == o)
+         {
+            return true;
+         }
+         if(o == null || getClass() != o.getClass())
+         {
+            return false;
+         }
+         NormalizedJoin that = (NormalizedJoin) o;
+         return Objects.equals(tableA, that.tableA) && Objects.equals(tableB, that.tableB) && Objects.equals(joinFieldA, that.joinFieldA) && Objects.equals(joinFieldB, that.joinFieldB);
+      }
+
+
+
+      /*******************************************************************************
+       **
+       *******************************************************************************/
+      @Override
+      public int hashCode()
+      {
+         return Objects.hash(tableA, tableB, joinFieldA, joinFieldB);
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** Constructor
+    **
+    *******************************************************************************/
+   public JoinGraph(QInstance qInstance)
+   {
+      Set<NormalizedJoin> usedJoins = new HashSet<>();
+      for(QJoinMetaData join : CollectionUtils.nonNullMap(qInstance.getJoins()).values())
+      {
+         NormalizedJoin normalizedJoin = new NormalizedJoin(join);
+         if(usedJoins.contains(normalizedJoin))
+         {
+            continue;
+         }
+
+         usedJoins.add(normalizedJoin);
+         edges.add(new Edge(join.getName(), join.getLeftTable(), join.getRightTable()));
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public record JoinConnection(String joinTable, String viaJoinName) implements Comparable<JoinConnection>
+   {
+
+      /*******************************************************************************
+       **
+       *******************************************************************************/
+      @Override
+      public int compareTo(JoinConnection that)
+      {
+         Comparator<JoinConnection> comparator = Comparator.comparing((JoinConnection jc) -> jc.joinTable())
+            .thenComparing((JoinConnection jc) -> jc.viaJoinName());
+         return (comparator.compare(this, that));
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public record JoinConnectionList(List<JoinConnection> list) implements Comparable<JoinConnectionList>
+   {
+
+      /*******************************************************************************
+       **
+       *******************************************************************************/
+      public JoinConnectionList copy()
+      {
+         return new JoinConnectionList(new ArrayList<>(list));
+      }
+
+
+
+      /*******************************************************************************
+       **
+       *******************************************************************************/
+      public int compareTo(JoinConnectionList that)
+      {
+         if(this.equals(that))
+         {
+            return (0);
+         }
+
+         for(int i = 0; i < Math.min(this.list.size(), that.list.size()); i++)
+         {
+            int comp = this.list.get(i).compareTo(that.list.get(i));
+            if(comp != 0)
+            {
+               return (comp);
+            }
+         }
+
+         return (this.list.size() - that.list.size());
+      }
+
+
+
+      /*******************************************************************************
+       **
+       *******************************************************************************/
+      public boolean matchesJoinPath(List<String> joinPath)
+      {
+         if(list.size() != joinPath.size())
+         {
+            return (false);
+         }
+
+         for(int i = 0; i < list.size(); i++)
+         {
+            if(!list.get(i).viaJoinName().equals(joinPath.get(i)))
+            {
+               return (false);
+            }
+         }
+
+         return (true);
+      }
+
+
+
+      /*******************************************************************************
+       **
+       *******************************************************************************/
+      public String getJoinNamesAsString()
+      {
+         return (StringUtils.join(", ", list().stream().map(jc -> jc.viaJoinName()).toList()));
+      }
+
+
+
+      /*******************************************************************************
+       **
+       *******************************************************************************/
+      public List<String> getJoinNamesAsList()
+      {
+         return (list().stream().map(jc -> jc.viaJoinName()).toList());
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public Set<JoinConnectionList> getJoinConnections(String tableName)
+   {
+      Set<JoinConnectionList> rs = new TreeSet<>();
+      doGetJoinConnections(rs, tableName, new ArrayList<>(), new JoinConnectionList(new ArrayList<>()));
+      return (rs);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private void doGetJoinConnections(Set<JoinConnectionList> joinConnections, String tableName, List<String> path, JoinConnectionList connectionList)
+   {
+      for(Edge edge : edges)
+      {
+         if(edge.leftTable.equals(tableName) || edge.rightTable.equals(tableName))
+         {
+            if(path.contains(edge.joinName))
+            {
+               continue;
+            }
+
+            List<String> newPath = new ArrayList<>(path);
+            newPath.add(edge.joinName);
+            if(!joinConnectionsContain(joinConnections, newPath))
+            {
+               String otherTableName = null;
+               if(!edge.leftTable.equals(tableName))
+               {
+                  otherTableName = edge.leftTable;
+               }
+               else if(!edge.rightTable.equals(tableName))
+               {
+                  otherTableName = edge.rightTable;
+               }
+
+               if(otherTableName != null)
+               {
+
+                  JoinConnectionList newConnectionList = connectionList.copy();
+                  JoinConnection     joinConnection    = new JoinConnection(otherTableName, edge.joinName);
+                  newConnectionList.list.add(joinConnection);
+                  joinConnections.add(newConnectionList);
+                  doGetJoinConnections(joinConnections, otherTableName, new ArrayList<>(newPath), newConnectionList);
+               }
+            }
+         }
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private boolean joinConnectionsContain(Set<JoinConnectionList> joinPaths, List<String> newPath)
+   {
+      for(JoinConnectionList joinConnections : joinPaths)
+      {
+         List<String> joinConnectionJoins = joinConnections.list.stream().map(jc -> jc.viaJoinName).toList();
+         if(joinConnectionJoins.equals(newPath))
+         {
+            return (true);
+         }
+      }
+      return (false);
+   }
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/metadata/MetaDataAction.java

@@ -83,11 +83,14 @@ public class MetaDataAction
          }
 
          QBackendMetaData backendForTable = metaDataInput.getInstance().getBackendForTable(tableName);
-         tables.put(tableName, new QFrontendTableMetaData(metaDataInput, backendForTable, table, false));
+         tables.put(tableName, new QFrontendTableMetaData(metaDataInput, backendForTable, table, false, false));
          treeNodes.put(tableName, new AppTreeNode(table));
       }
       metaDataOutput.setTables(tables);
 
+      // addJoinsToTables(tables);
+      // addJoinedTablesToTables(tables);
+
       ////////////////////////////////////////
       // map processes to frontend metadata //
       ////////////////////////////////////////

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/metadata/TableMetaDataAction.java

@@ -54,7 +54,7 @@ public class TableMetaDataAction
          throw (new QNotFoundException("Table [" + tableMetaDataInput.getTableName() + "] was not found."));
       }
       QBackendMetaData backendForTable = tableMetaDataInput.getInstance().getBackendForTable(table.getName());
-      tableMetaDataOutput.setTable(new QFrontendTableMetaData(tableMetaDataInput, backendForTable, table, true));
+      tableMetaDataOutput.setTable(new QFrontendTableMetaData(tableMetaDataInput, backendForTable, table, true, true));
 
       // todo post-customization - can do whatever w/ the result if you want
 

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/BufferedRecordPipe.java

@@ -24,6 +24,7 @@ package com.kingsrook.qqq.backend.core.actions.reporting;
 
 import java.util.ArrayList;
 import java.util.List;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
 
 
@@ -63,7 +64,7 @@ public class BufferedRecordPipe extends RecordPipe
     **
     *******************************************************************************/
    @Override
-   public void addRecord(QRecord record)
+   public void addRecord(QRecord record) throws QException
    {
       buffer.add(record);
       if(buffer.size() >= bufferSize)
@@ -78,7 +79,7 @@ public class BufferedRecordPipe extends RecordPipe
    /*******************************************************************************
     **
     *******************************************************************************/
-   public void finalFlush()
+   public void finalFlush() throws QException
    {
       if(!buffer.isEmpty())
       {

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/ExportAction.java

@@ -23,8 +23,12 @@ package com.kingsrook.qqq.backend.core.actions.reporting;
 
 
 import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
 import java.util.List;
+import java.util.Map;
 import java.util.Optional;
+import java.util.Set;
 import java.util.concurrent.TimeUnit;
 import com.kingsrook.qqq.backend.core.actions.ActionHelper;
 import com.kingsrook.qqq.backend.core.actions.async.AsyncJobManager;
@@ -32,6 +36,7 @@ import com.kingsrook.qqq.backend.core.actions.async.AsyncJobState;
 import com.kingsrook.qqq.backend.core.actions.async.AsyncJobStatus;
 import com.kingsrook.qqq.backend.core.actions.interfaces.CountInterface;
 import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
+import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.exceptions.QReportingException;
 import com.kingsrook.qqq.backend.core.exceptions.QUserFacingException;
@@ -41,10 +46,13 @@ import com.kingsrook.qqq.backend.core.model.actions.reporting.ExportOutput;
 import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportFormat;
 import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountOutput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryJoin;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.ExposedJoin;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleDispatcher;
 import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleInterface;
@@ -95,15 +103,25 @@ public class ExportAction
       ///////////////////////////////////
       if(CollectionUtils.nullSafeHasContents(exportInput.getFieldNames()))
       {
-         QTableMetaData table         = exportInput.getTable();
-         List<String>   badFieldNames = new ArrayList<>();
+         QTableMetaData              table        = exportInput.getTable();
+         Map<String, QTableMetaData> joinTableMap = getJoinTableMap(table);
+
+         List<String> badFieldNames = new ArrayList<>();
          for(String fieldName : exportInput.getFieldNames())
          {
             try
             {
-               table.getField(fieldName);
+               if(fieldName.contains("."))
+               {
+                  String[] parts = fieldName.split("\\.", 2);
+                  joinTableMap.get(parts[0]).getField(parts[1]);
+               }
+               else
+               {
+                  table.getField(fieldName);
+               }
             }
-            catch(IllegalArgumentException iae)
+            catch(Exception e)
             {
                badFieldNames.add(fieldName);
             }
@@ -128,6 +146,21 @@ public class ExportAction
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private static Map<String, QTableMetaData> getJoinTableMap(QTableMetaData table)
+   {
+      Map<String, QTableMetaData> joinTableMap = new HashMap<>();
+      for(ExposedJoin exposedJoin : CollectionUtils.nonNullList(table.getExposedJoins()))
+      {
+         joinTableMap.put(exposedJoin.getJoinTable(), QContext.getQInstance().getTable(exposedJoin.getJoinTable()));
+      }
+      return joinTableMap;
+   }
+
+
+
    /*******************************************************************************
     ** Run the report.
     *******************************************************************************/
@@ -151,7 +184,33 @@ public class ExportAction
       QueryInput  queryInput  = new QueryInput();
       queryInput.setTableName(exportInput.getTableName());
       queryInput.setFilter(exportInput.getQueryFilter());
-      queryInput.setLimit(exportInput.getLimit());
+
+      List<QueryJoin> queryJoins     = new ArrayList<>();
+      Set<String>     addedJoinNames = new HashSet<>();
+      if(CollectionUtils.nullSafeHasContents(exportInput.getFieldNames()))
+      {
+         for(String fieldName : exportInput.getFieldNames())
+         {
+            if(fieldName.contains("."))
+            {
+               String[] parts         = fieldName.split("\\.", 2);
+               String   joinTableName = parts[0];
+               if(!addedJoinNames.contains(joinTableName))
+               {
+                  queryJoins.add(new QueryJoin(joinTableName).withType(QueryJoin.Type.LEFT).withSelect(true));
+                  addedJoinNames.add(joinTableName);
+               }
+            }
+         }
+      }
+
+      queryInput.setQueryJoins(queryJoins);
+
+      if(queryInput.getFilter() == null)
+      {
+         queryInput.setFilter(new QQueryFilter());
+      }
+      queryInput.getFilter().setLimit(exportInput.getLimit());
       queryInput.setShouldTranslatePossibleValues(true);
 
       /////////////////////////////////////////////////////////////////
@@ -298,24 +357,51 @@ public class ExportAction
     *******************************************************************************/
    private List<QFieldMetaData> getFields(ExportInput exportInput)
    {
+      QTableMetaData              table        = exportInput.getTable();
+      Map<String, QTableMetaData> joinTableMap = getJoinTableMap(table);
+
       List<QFieldMetaData> fieldList;
-      QTableMetaData       table = exportInput.getTable();
       if(exportInput.getFieldNames() != null)
       {
-         fieldList = exportInput.getFieldNames().stream().map(table::getField).toList();
+         fieldList = new ArrayList<>();
+         for(String fieldName : exportInput.getFieldNames())
+         {
+            if(fieldName.contains("."))
+            {
+               String[]       parts     = fieldName.split("\\.", 2);
+               QTableMetaData joinTable = joinTableMap.get(parts[0]);
+               QFieldMetaData field     = joinTable.getField(parts[1]).clone();
+               field.setName(fieldName);
+               field.setLabel(joinTable.getLabel() + ": " + field.getLabel());
+               fieldList.add(field);
+            }
+            else
+            {
+               fieldList.add(table.getField(fieldName));
+            }
+         }
       }
       else
       {
          fieldList = new ArrayList<>(table.getFields().values());
       }
 
-      //////////////////////////////////////////
-      // add fields for possible value labels //
-      //////////////////////////////////////////
       List<QFieldMetaData> returnList = new ArrayList<>();
       for(QFieldMetaData field : fieldList)
       {
+         /////////////////////////////////////////////////////////////////////////////////////////
+         // skip heavy fields.  they aren't fetched, and we generally think we don't want them. //
+         /////////////////////////////////////////////////////////////////////////////////////////
+         if(field.getIsHeavy())
+         {
+            continue;
+         }
+
          returnList.add(field);
+
+         //////////////////////////////////////////
+         // add fields for possible value labels //
+         //////////////////////////////////////////
          if(StringUtils.hasContent(field.getPossibleValueSourceName()))
          {
             returnList.add(new QFieldMetaData(field.getName() + ":possibleValueLabel", QFieldType.STRING).withLabel(field.getLabel() + " Name"));

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/RecordPipe.java

@@ -26,10 +26,11 @@ import java.util.ArrayList;
 import java.util.List;
 import java.util.concurrent.ArrayBlockingQueue;
 import java.util.concurrent.TimeUnit;
-import java.util.function.Consumer;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.logging.QLogger;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
 import com.kingsrook.qqq.backend.core.utils.SleepUtils;
+import com.kingsrook.qqq.backend.core.utils.lambdas.UnsafeConsumer;
 
 
 /*******************************************************************************
@@ -47,7 +48,7 @@ public class RecordPipe
 
    private boolean isTerminated = false;
 
-   private Consumer<List<QRecord>> postRecordActions = null;
+   private UnsafeConsumer<List<QRecord>, QException> postRecordActions = null;
 
    /////////////////////////////////////
    // See usage below for explanation //
@@ -93,7 +94,7 @@ public class RecordPipe
    /*******************************************************************************
     ** Add a record to the pipe.  Will block if the pipe is full.  Will noop if pipe is terminated.
     *******************************************************************************/
-   public void addRecord(QRecord record)
+   public void addRecord(QRecord record) throws QException
    {
       if(isTerminated)
       {
@@ -109,7 +110,7 @@ public class RecordPipe
          // (which we'll create as a field in this class, to avoid always re-constructing) //
          ////////////////////////////////////////////////////////////////////////////////////
          singleRecordListForPostRecordActions.add(record);
-         postRecordActions.accept(singleRecordListForPostRecordActions);
+         postRecordActions.run(singleRecordListForPostRecordActions);
          record = singleRecordListForPostRecordActions.remove(0);
       }
 
@@ -152,11 +153,11 @@ public class RecordPipe
    /*******************************************************************************
     ** Add a list of records to the pipe.  Will block if the pipe is full.  Will noop if pipe is terminated.
     *******************************************************************************/
-   public void addRecords(List<QRecord> records)
+   public void addRecords(List<QRecord> records) throws QException
    {
       if(postRecordActions != null)
       {
-         postRecordActions.accept(records);
+         postRecordActions.run(records);
       }
 
       //////////////////////////////////////////////////////////////////////////////////////////////////
@@ -207,7 +208,7 @@ public class RecordPipe
    /*******************************************************************************
     **
     *******************************************************************************/
-   public void setPostRecordActions(Consumer<List<QRecord>> postRecordActions)
+   public void setPostRecordActions(UnsafeConsumer<List<QRecord>, QException> postRecordActions)
    {
       this.postRecordActions = postRecordActions;
    }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/RecordPipeBufferedWrapper.java

@@ -0,0 +1,79 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.actions.reporting;
+
+
+import java.util.List;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+
+
+/*******************************************************************************
+ ** Subclass of BufferedRecordPipe, which ultimately sends records down to an
+ ** original RecordPipe.
+ **
+ ** Meant to be used where: someone passed in a RecordPipe (so they have a reference
+ ** to it, and they are waiting to read from it), but the producer knows that
+ ** it will be better to buffer the records, so they want to use a buffered pipe
+ ** (but they still need the records to end up in the original pipe - thus -
+ ** it gets wrapped by an object of this class).
+ *******************************************************************************/
+public class RecordPipeBufferedWrapper extends BufferedRecordPipe
+{
+   private RecordPipe wrappedPipe;
+
+
+
+   /*******************************************************************************
+    ** Constructor - uses default buffer size
+    **
+    *******************************************************************************/
+   public RecordPipeBufferedWrapper(RecordPipe wrappedPipe)
+   {
+      this.wrappedPipe = wrappedPipe;
+   }
+
+
+
+   /*******************************************************************************
+    ** Constructor - customize buffer size.
+    **
+    *******************************************************************************/
+   public RecordPipeBufferedWrapper(Integer bufferSize, RecordPipe wrappedPipe)
+   {
+      super(bufferSize);
+      this.wrappedPipe = wrappedPipe;
+   }
+
+
+
+   /*******************************************************************************
+    ** when it's time to actually add records into the pipe, actually add them
+    ** into the wrapped pipe!
+    *******************************************************************************/
+   @Override
+   public void addRecords(List<QRecord> records) throws QException
+   {
+      wrappedPipe.addRecords(records);
+   }
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/scripts/AssociatedScriptContextPrimerInterface.java

@@ -0,0 +1,41 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.actions.scripts;
+
+
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.actions.scripts.ExecuteCodeInput;
+import com.kingsrook.qqq.backend.core.model.scripts.ScriptRevision;
+
+
+/*******************************************************************************
+ **
+ *******************************************************************************/
+public interface AssociatedScriptContextPrimerInterface
+{
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   void primeContext(ExecuteCodeInput executeCodeInput, ScriptRevision scriptRevision) throws QException;
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/scripts/ExecuteCodeAction.java

@@ -25,13 +25,21 @@ package com.kingsrook.qqq.backend.core.actions.scripts;
 import java.io.Serializable;
 import java.util.HashMap;
 import java.util.Map;
+import java.util.Objects;
 import com.kingsrook.qqq.backend.core.actions.scripts.logging.Log4jCodeExecutionLogger;
 import com.kingsrook.qqq.backend.core.actions.scripts.logging.QCodeExecutionLoggerInterface;
+import com.kingsrook.qqq.backend.core.actions.scripts.logging.ScriptExecutionLoggerInterface;
+import com.kingsrook.qqq.backend.core.actions.scripts.logging.StoreScriptLogAndScriptLogLineExecutionLogger;
 import com.kingsrook.qqq.backend.core.exceptions.QCodeException;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.logging.QLogger;
+import com.kingsrook.qqq.backend.core.model.actions.scripts.AbstractRunScriptInput;
 import com.kingsrook.qqq.backend.core.model.actions.scripts.ExecuteCodeInput;
 import com.kingsrook.qqq.backend.core.model.actions.scripts.ExecuteCodeOutput;
 import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
+import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeType;
+import com.kingsrook.qqq.backend.core.model.scripts.ScriptRevision;
+import com.kingsrook.qqq.backend.core.utils.StringUtils;
 
 
 /*******************************************************************************
@@ -49,6 +57,9 @@ import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
  *******************************************************************************/
 public class ExecuteCodeAction
 {
+   private static final QLogger LOG = QLogger.getLogger(ExecuteCodeAction.class);
+
+
 
    /*******************************************************************************
     **
@@ -68,10 +79,10 @@ public class ExecuteCodeAction
       try
       {
          String languageExecutor = switch(codeReference.getCodeType())
-            {
-               case JAVA -> "com.kingsrook.qqq.backend.core.actions.scripts.QJavaExecutor";
-               case JAVA_SCRIPT -> "com.kingsrook.qqq.languages.javascript.QJavaScriptExecutor";
-            };
+         {
+            case JAVA -> "com.kingsrook.qqq.backend.core.actions.scripts.QJavaExecutor";
+            case JAVA_SCRIPT -> "com.kingsrook.qqq.languages.javascript.QJavaScriptExecutor";
+         };
 
          @SuppressWarnings("unchecked")
          Class<? extends QCodeExecutor> executorClass = (Class<? extends QCodeExecutor>) Class.forName(languageExecutor);
@@ -108,6 +119,92 @@ public class ExecuteCodeAction
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public static ExecuteCodeInput setupExecuteCodeInput(AbstractRunScriptInput<?> input, ScriptRevision scriptRevision)
+   {
+      ExecuteCodeInput executeCodeInput = new ExecuteCodeInput();
+      executeCodeInput.setInput(new HashMap<>(Objects.requireNonNullElseGet(input.getInputValues(), HashMap::new)));
+      executeCodeInput.setContext(new HashMap<>());
+
+      Map<String, Serializable> context = executeCodeInput.getContext();
+      if(input.getOutputObject() != null)
+      {
+         context.put("output", input.getOutputObject());
+      }
+
+      if(input.getScriptUtils() != null)
+      {
+         context.put("scriptUtils", input.getScriptUtils());
+      }
+
+      executeCodeInput.setCodeReference(new QCodeReference().withInlineCode(scriptRevision.getContents()).withCodeType(QCodeType.JAVA_SCRIPT)); // todo - code type as attribute of script!!
+
+      ExecuteCodeAction.addApiUtilityToContext(context, scriptRevision);
+      context.put("qqq", new QqqScriptUtils());
+      ExecuteCodeAction.setExecutionLoggerInExecuteCodeInput(input, scriptRevision, executeCodeInput);
+
+      return (executeCodeInput);
+   }
+
+
+
+   /*******************************************************************************
+    ** Try to (dynamically) load the ApiScriptUtils object from the api middleware
+    ** module -- in case the runtime doesn't have that module deployed (e.g, not in
+    ** the project pom).
+    *******************************************************************************/
+   public static void addApiUtilityToContext(Map<String, Serializable> context, ScriptRevision scriptRevision)
+   {
+      if(!StringUtils.hasContent(scriptRevision.getApiName()) || !StringUtils.hasContent(scriptRevision.getApiVersion()))
+      {
+         return;
+      }
+
+      try
+      {
+         Class<?> apiScriptUtilsClass  = Class.forName("com.kingsrook.qqq.api.utils.ApiScriptUtils");
+         Object   apiScriptUtilsObject = apiScriptUtilsClass.getConstructor(String.class, String.class).newInstance(scriptRevision.getApiName(), scriptRevision.getApiVersion());
+         context.put("api", (Serializable) apiScriptUtilsObject);
+      }
+      catch(ClassNotFoundException e)
+      {
+         /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+         // this is the only exception we're kinda expecting here - so catch for it specifically, and just log.trace - others, warn //
+         /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+         LOG.trace("Couldn't load ApiScriptUtils class - qqq-middleware-api not on the classpath?");
+      }
+      catch(Exception e)
+      {
+         LOG.warn("Error adding api utility to script context", e);
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private static void setExecutionLoggerInExecuteCodeInput(AbstractRunScriptInput<?> input, ScriptRevision scriptRevision, ExecuteCodeInput executeCodeInput)
+   {
+      /////////////////////////////////////////////////////////////////////////////////////////////////
+      // let caller supply a logger, or by default use StoreScriptLogAndScriptLogLineExecutionLogger //
+      /////////////////////////////////////////////////////////////////////////////////////////////////
+      QCodeExecutionLoggerInterface executionLogger = Objects.requireNonNullElseGet(input.getLogger(), () -> new StoreScriptLogAndScriptLogLineExecutionLogger(scriptRevision.getScriptId(), scriptRevision.getId()));
+      executeCodeInput.setExecutionLogger(executionLogger);
+      if(executionLogger instanceof ScriptExecutionLoggerInterface scriptExecutionLoggerInterface)
+      {
+         ////////////////////////////////////////////////////////////////////////////////////////////////////
+         // if logger is aware of scripts (as opposed to a generic CodeExecution logger), give it the ids. //
+         ////////////////////////////////////////////////////////////////////////////////////////////////////
+         scriptExecutionLoggerInterface.setScriptId(scriptRevision.getScriptId());
+         scriptExecutionLoggerInterface.setScriptRevisionId(scriptRevision.getId());
+      }
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/scripts/QqqScriptUtils.java

@@ -47,8 +47,17 @@ import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 /*******************************************************************************
  ** Object made available to scripts for access to qqq api (e.g., query, insert,
  ** etc, plus object constructors).
+ **
+ ** Before scripts knew about the API, this class made sense and was used.
+ ** But, when scripts gained knowledge of the API, then it felt like this class could
+ ** be deleted... but, what about, a QQQ deployment without the API module...
+ ** In that case, we might still want this class... think about it.
+ **
+ ** And/Or - it turns out - sometimes using QQQ directly is "better" (?) than using
+ ** an api - so - this object may be available for other use cases (e.g., getting
+ ** a record's backendDetails (e.g., for full json from a source backend api)).
  *******************************************************************************/
-public class ScriptApi implements Serializable
+public class QqqScriptUtils implements Serializable
 {
 
    /*******************************************************************************

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/scripts/RunAdHocRecordScriptAction.java

@@ -22,15 +22,14 @@
 package com.kingsrook.qqq.backend.core.actions.scripts;
 
 
+import java.io.Serializable;
+import java.lang.reflect.Method;
 import java.util.ArrayList;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
-import java.util.Objects;
 import java.util.Optional;
 import com.kingsrook.qqq.backend.core.actions.ActionHelper;
-import com.kingsrook.qqq.backend.core.actions.scripts.logging.QCodeExecutionLoggerInterface;
-import com.kingsrook.qqq.backend.core.actions.scripts.logging.ScriptExecutionLoggerInterface;
-import com.kingsrook.qqq.backend.core.actions.scripts.logging.StoreScriptLogAndScriptLogLineExecutionLogger;
 import com.kingsrook.qqq.backend.core.actions.tables.GetAction;
 import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
 import com.kingsrook.qqq.backend.core.context.QContext;
@@ -49,8 +48,6 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryJoin;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput;
 import com.kingsrook.qqq.backend.core.model.metadata.code.AdHocScriptCodeReference;
-import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
-import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeType;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 import com.kingsrook.qqq.backend.core.model.scripts.Script;
 import com.kingsrook.qqq.backend.core.model.scripts.ScriptRevision;
@@ -96,6 +93,7 @@ public class RunAdHocRecordScriptAction
             QueryInput queryInput = new QueryInput();
             queryInput.setTableName(input.getTableName());
             queryInput.setFilter(new QQueryFilter(new QFilterCriteria(table.getPrimaryKeyField(), QCriteriaOperator.IN, input.getRecordPrimaryKeyList())));
+            queryInput.setIncludeAssociations(true);
             QueryOutput queryOutput = new QueryAction().execute(queryInput);
             input.setRecordList(queryOutput.getRecords());
          }
@@ -112,43 +110,14 @@ public class RunAdHocRecordScriptAction
          /////////////
          // run it! //
          /////////////
-         ExecuteCodeInput executeCodeInput = new ExecuteCodeInput();
-         executeCodeInput.setInput(new HashMap<>(Objects.requireNonNullElseGet(input.getInputValues(), HashMap::new)));
-         executeCodeInput.getInput().put("records", new ArrayList<>(input.getRecordList()));
-         executeCodeInput.setContext(new HashMap<>());
-         if(input.getOutputObject() != null)
-         {
-            executeCodeInput.getContext().put("output", input.getOutputObject());
-         }
-
-         if(input.getScriptUtils() != null)
-         {
-            executeCodeInput.getContext().put("scriptUtils", input.getScriptUtils());
-         }
-
-         executeCodeInput.getContext().put("api", new ScriptApi());
-
-         executeCodeInput.setCodeReference(new QCodeReference().withInlineCode(scriptRevision.getContents()).withCodeType(QCodeType.JAVA_SCRIPT)); // todo - code type as attribute of script!!
-
-         /////////////////////////////////////////////////////////////////////////////////////////////////
-         // let caller supply a logger, or by default use StoreScriptLogAndScriptLogLineExecutionLogger //
-         /////////////////////////////////////////////////////////////////////////////////////////////////
-         QCodeExecutionLoggerInterface executionLogger = Objects.requireNonNullElseGet(input.getLogger(), () -> new StoreScriptLogAndScriptLogLineExecutionLogger(scriptRevision.getScriptId(), scriptRevision.getId()));
-         executeCodeInput.setExecutionLogger(executionLogger);
-         if(executionLogger instanceof ScriptExecutionLoggerInterface scriptExecutionLoggerInterface)
-         {
-            ////////////////////////////////////////////////////////////////////////////////////////////////////
-            // if logger is aware of scripts (as opposed to a generic CodeExecution logger), give it the ids. //
-            ////////////////////////////////////////////////////////////////////////////////////////////////////
-            scriptExecutionLoggerInterface.setScriptId(scriptRevision.getScriptId());
-            scriptExecutionLoggerInterface.setScriptRevisionId(scriptRevision.getId());
-         }
+         ExecuteCodeInput executeCodeInput = ExecuteCodeAction.setupExecuteCodeInput(input, scriptRevision);
+         executeCodeInput.getInput().put("records", getRecordsForScript(input, scriptRevision));
 
          ExecuteCodeOutput executeCodeOutput = new ExecuteCodeOutput();
          new ExecuteCodeAction().run(executeCodeInput, executeCodeOutput);
 
          output.setOutput(executeCodeOutput.getOutput());
-         output.setLogger(executionLogger);
+         output.setLogger(executeCodeInput.getExecutionLogger());
       }
       catch(Exception e)
       {
@@ -158,6 +127,37 @@ public class RunAdHocRecordScriptAction
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private static ArrayList<? extends Serializable> getRecordsForScript(RunAdHocRecordScriptInput input, ScriptRevision scriptRevision)
+   {
+      try
+      {
+         Class<?> apiScriptUtilsClass        = Class.forName("com.kingsrook.qqq.api.utils.ApiScriptUtils");
+         Method   qRecordListToApiRecordList = apiScriptUtilsClass.getMethod("qRecordListToApiRecordList", List.class, String.class, String.class, String.class);
+         Object   apiRecordList              = qRecordListToApiRecordList.invoke(null, input.getRecordList(), input.getTableName(), scriptRevision.getApiName(), scriptRevision.getApiVersion());
+
+         // noinspection unchecked
+         return (ArrayList<? extends Serializable>) apiRecordList;
+      }
+      catch(ClassNotFoundException e)
+      {
+         /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+         // this is the only exception we're kinda expecting here - so catch for it specifically, and just log.trace - others, warn //
+         /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+         LOG.trace("Couldn't load ApiScriptUtils class - qqq-middleware-api not on the classpath?");
+      }
+      catch(Exception e)
+      {
+         LOG.warn("Error converting QRecord list to api record list", e);
+      }
+
+      return (new ArrayList<>(input.getRecordList()));
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/scripts/RunAssociatedScriptAction.java

@@ -25,11 +25,7 @@ package com.kingsrook.qqq.backend.core.actions.scripts;
 import java.io.Serializable;
 import java.util.HashMap;
 import java.util.Map;
-import java.util.Objects;
 import com.kingsrook.qqq.backend.core.actions.ActionHelper;
-import com.kingsrook.qqq.backend.core.actions.scripts.logging.QCodeExecutionLoggerInterface;
-import com.kingsrook.qqq.backend.core.actions.scripts.logging.ScriptExecutionLoggerInterface;
-import com.kingsrook.qqq.backend.core.actions.scripts.logging.StoreScriptLogAndScriptLogLineExecutionLogger;
 import com.kingsrook.qqq.backend.core.actions.tables.GetAction;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.exceptions.QNotFoundException;
@@ -40,8 +36,6 @@ import com.kingsrook.qqq.backend.core.model.actions.scripts.RunAssociatedScriptO
 import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetOutput;
 import com.kingsrook.qqq.backend.core.model.metadata.code.AssociatedScriptCodeReference;
-import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
-import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeType;
 import com.kingsrook.qqq.backend.core.model.scripts.Script;
 import com.kingsrook.qqq.backend.core.model.scripts.ScriptRevision;
 
@@ -54,6 +48,7 @@ public class RunAssociatedScriptAction
    private Map<AssociatedScriptCodeReference, ScriptRevision> scriptRevisionCache = new HashMap<>();
 
 
+
    /*******************************************************************************
     **
     *******************************************************************************/
@@ -61,35 +56,12 @@ public class RunAssociatedScriptAction
    {
       ActionHelper.validateSession(input);
 
-      ScriptRevision scriptRevision = getScriptRevision(input);
+      ScriptRevision   scriptRevision   = getScriptRevision(input);
+      ExecuteCodeInput executeCodeInput = ExecuteCodeAction.setupExecuteCodeInput(input, scriptRevision);
 
-      ExecuteCodeInput executeCodeInput = new ExecuteCodeInput();
-      executeCodeInput.setInput(new HashMap<>(input.getInputValues()));
-      executeCodeInput.setContext(new HashMap<>());
-      if(input.getOutputObject() != null)
+      if(input.getAssociatedScriptContextPrimerInterface() != null)
       {
-         executeCodeInput.getContext().put("output", input.getOutputObject());
-      }
-
-      if(input.getScriptUtils() != null)
-      {
-         executeCodeInput.getContext().put("scriptUtils", input.getScriptUtils());
-      }
-
-      executeCodeInput.setCodeReference(new QCodeReference().withInlineCode(scriptRevision.getContents()).withCodeType(QCodeType.JAVA_SCRIPT)); // todo - code type as attribute of script!!
-
-      /////////////////////////////////////////////////////////////////////////////////////////////////
-      // let caller supply a logger, or by default use StoreScriptLogAndScriptLogLineExecutionLogger //
-      /////////////////////////////////////////////////////////////////////////////////////////////////
-      QCodeExecutionLoggerInterface executionLogger = Objects.requireNonNullElseGet(input.getLogger(), () -> new StoreScriptLogAndScriptLogLineExecutionLogger(scriptRevision.getScriptId(), scriptRevision.getId()));
-      executeCodeInput.setExecutionLogger(executionLogger);
-      if(executionLogger instanceof ScriptExecutionLoggerInterface scriptExecutionLoggerInterface)
-      {
-         ////////////////////////////////////////////////////////////////////////////////////////////////////
-         // if logger is aware of scripts (as opposed to a generic CodeExecution logger), give it the ids. //
-         ////////////////////////////////////////////////////////////////////////////////////////////////////
-         scriptExecutionLoggerInterface.setScriptId(scriptRevision.getScriptId());
-         scriptExecutionLoggerInterface.setScriptRevisionId(scriptRevision.getId());
+         input.getAssociatedScriptContextPrimerInterface().primeContext(executeCodeInput, scriptRevision);
       }
 
       ExecuteCodeOutput executeCodeOutput = new ExecuteCodeOutput();

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/scripts/StoreAssociatedScriptAction.java

@@ -156,8 +156,7 @@ public class StoreAssociatedScriptAction
          queryInput.setFilter(new QQueryFilter()
             .withCriteria(new QFilterCriteria("scriptId", QCriteriaOperator.EQUALS, List.of(script.getValue("id"))))
             .withOrderBy(new QFilterOrderBy("sequenceNo", false))
-         );
-         queryInput.setLimit(1);
+            .withLimit(1));
          QueryOutput queryOutput = new QueryAction().execute(queryInput);
          if(!queryOutput.getRecords().isEmpty())
          {
@@ -184,6 +183,8 @@ public class StoreAssociatedScriptAction
       QRecord scriptRevision = new QRecord()
          .withValue("scriptId", script.getValue("id"))
          .withValue("contents", input.getCode())
+         .withValue("apiName", input.getApiName())
+         .withValue("apiVersion", input.getApiVersion())
          .withValue("commitMessage", commitMessage)
          .withValue("sequenceNo", nextSequenceNo);
 

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/scripts/TestScriptActionInterface.java

@@ -32,6 +32,7 @@ import com.kingsrook.qqq.backend.core.model.actions.scripts.ExecuteCodeOutput;
 import com.kingsrook.qqq.backend.core.model.actions.scripts.TestScriptInput;
 import com.kingsrook.qqq.backend.core.model.actions.scripts.TestScriptOutput;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
+import com.kingsrook.qqq.backend.core.model.scripts.ScriptRevision;
 
 
 /*******************************************************************************
@@ -47,7 +48,7 @@ public interface TestScriptActionInterface
     ** Note - such a method may want or need to put an "output" object into the
     ** executeCodeInput's context map.
     *******************************************************************************/
-   void setupTestScriptInput(TestScriptInput testScriptInput, ExecuteCodeInput executeCodeInput);
+   void setupTestScriptInput(TestScriptInput testScriptInput, ExecuteCodeInput executeCodeInput) throws QException;
 
 
    /*******************************************************************************
@@ -80,6 +81,9 @@ public interface TestScriptActionInterface
     *******************************************************************************/
    default void execute(TestScriptInput input, TestScriptOutput output) throws QException
    {
+      /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      // todo - could this be merged with the various other script runners, to use ExecuteCodeAction.setupExecuteCodeInput?? //
+      /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
       ExecuteCodeInput executeCodeInput = new ExecuteCodeInput();
       executeCodeInput.setContext(new HashMap<>());
 
@@ -87,12 +91,21 @@ public interface TestScriptActionInterface
       BuildScriptLogAndScriptLogLineExecutionLogger executionLogger = new BuildScriptLogAndScriptLogLineExecutionLogger(null, null);
       executeCodeInput.setExecutionLogger(executionLogger);
 
-      setupTestScriptInput(input, executeCodeInput);
-
-      ExecuteCodeOutput executeCodeOutput = new ExecuteCodeOutput();
-
       try
       {
+         setupTestScriptInput(input, executeCodeInput);
+
+         ScriptRevision scriptRevision = new ScriptRevision().withApiName(input.getApiName()).withApiVersion(input.getApiVersion());
+
+         if(this instanceof AssociatedScriptContextPrimerInterface associatedScriptContextPrimerInterface)
+         {
+            associatedScriptContextPrimerInterface.primeContext(executeCodeInput, scriptRevision);
+         }
+
+         ExecuteCodeOutput executeCodeOutput = new ExecuteCodeOutput();
+
+         ExecuteCodeAction.addApiUtilityToContext(executeCodeInput.getContext(), scriptRevision);
+
          new ExecuteCodeAction().run(executeCodeInput, executeCodeOutput);
          output.setOutputObject(processTestScriptOutput(executeCodeOutput));
       }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/DeleteAction.java

@@ -26,11 +26,19 @@ import java.io.Serializable;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.HashSet;
+import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
 import java.util.Set;
+import java.util.stream.Collectors;
 import com.kingsrook.qqq.backend.core.actions.ActionHelper;
 import com.kingsrook.qqq.backend.core.actions.audits.DMLAuditAction;
+import com.kingsrook.qqq.backend.core.actions.customizers.AbstractPostDeleteCustomizer;
+import com.kingsrook.qqq.backend.core.actions.customizers.AbstractPreDeleteCustomizer;
+import com.kingsrook.qqq.backend.core.actions.customizers.QCodeLoader;
+import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizers;
 import com.kingsrook.qqq.backend.core.actions.interfaces.DeleteInterface;
 import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
@@ -44,11 +52,12 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
-import com.kingsrook.qqq.backend.core.model.metadata.audits.AuditLevel;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.joins.QJoinMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.Association;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+import com.kingsrook.qqq.backend.core.model.statusmessages.NotFoundStatusMessage;
+import com.kingsrook.qqq.backend.core.model.statusmessages.QWarningMessage;
 import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleDispatcher;
 import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleInterface;
 import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
@@ -63,8 +72,6 @@ public class DeleteAction
 {
    private static final QLogger LOG = QLogger.getLogger(DeleteAction.class);
 
-   public static final String NOT_FOUND_ERROR_PREFIX = "No record was found to delete";
-
 
 
    /*******************************************************************************
@@ -74,15 +81,43 @@ public class DeleteAction
    {
       ActionHelper.validateSession(deleteInput);
 
-      QBackendModuleDispatcher qBackendModuleDispatcher = new QBackendModuleDispatcher();
-      QBackendModuleInterface  qModule                  = qBackendModuleDispatcher.getQBackendModule(deleteInput.getBackend());
+      QTableMetaData table               = deleteInput.getTable();
+      String         primaryKeyFieldName = table.getPrimaryKeyField();
+      QFieldMetaData primaryKeyField     = table.getField(primaryKeyFieldName);
 
-      if(CollectionUtils.nullSafeHasContents(deleteInput.getPrimaryKeys()) && deleteInput.getQueryFilter() != null)
+      List<Serializable> primaryKeys         = deleteInput.getPrimaryKeys();
+      List<Serializable> originalPrimaryKeys = primaryKeys == null ? null : new ArrayList<>(primaryKeys);
+      if(CollectionUtils.nullSafeHasContents(primaryKeys) && deleteInput.getQueryFilter() != null)
       {
          throw (new QException("A delete request may not contain both a list of primary keys and a query filter."));
       }
 
-      DeleteInterface deleteInterface = qModule.getDeleteInterface();
+      ////////////////////////////////////////////////////////
+      // make sure the primary keys are of the correct type //
+      ////////////////////////////////////////////////////////
+      if(CollectionUtils.nullSafeHasContents(primaryKeys))
+      {
+         for(int i = 0; i < primaryKeys.size(); i++)
+         {
+            Serializable primaryKey       = primaryKeys.get(i);
+            Serializable valueAsFieldType = ValueUtils.getValueAsFieldType(primaryKeyField.getType(), primaryKey);
+            if(!Objects.equals(primaryKey, valueAsFieldType))
+            {
+               primaryKeys.set(i, valueAsFieldType);
+            }
+         }
+      }
+
+      //////////////////////////////////////////////////////
+      // load the backend module and its delete interface //
+      //////////////////////////////////////////////////////
+      QBackendModuleDispatcher qBackendModuleDispatcher = new QBackendModuleDispatcher();
+      QBackendModuleInterface  qModule                  = qBackendModuleDispatcher.getQBackendModule(deleteInput.getBackend());
+      DeleteInterface          deleteInterface          = qModule.getDeleteInterface();
+
+      //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      // if there's a query filter, but the interface doesn't support using a query filter, then do a query for the filter, to get a list of primary keys instead //
+      //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
       if(deleteInput.getQueryFilter() != null && !deleteInterface.supportsQueryFilterInput())
       {
          LOG.info("Querying for primary keys, for backend module " + qModule.getBackendType() + " which does not support queryFilter input for deletes");
@@ -99,32 +134,209 @@ public class DeleteAction
          }
       }
 
-      List<QRecord> recordListForAudit          = getRecordListForAuditIfNeeded(deleteInput);
-      List<QRecord> recordsWithValidationErrors = validateRecordsExistAndCanBeAccessed(deleteInput, recordListForAudit);
+      ////////////////////////////////////////////////////////////////////////////////
+      // fetch the old list of records (if the backend supports it), for audits,    //
+      // for "not-found detection", and for the pre-action to use (if there is one) //
+      ////////////////////////////////////////////////////////////////////////////////
+      Optional<List<QRecord>> oldRecordList = fetchOldRecords(deleteInput, deleteInterface);
 
-      DeleteOutput deleteOutput = deleteInterface.execute(deleteInput);
+      List<QRecord>              customizerResult              = performValidations(deleteInput, oldRecordList, false);
+      List<QRecord>              recordsWithValidationErrors   = new ArrayList<>();
+      Map<Serializable, QRecord> recordsWithValidationWarnings = new LinkedHashMap<>();
 
-      /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-      // merge the backend's output with any validation errors we found (whose ids wouldn't have gotten into the backend delete) //
-      /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-      List<QRecord> outputRecordsWithErrors = deleteOutput.getRecordsWithErrors();
-      if(outputRecordsWithErrors == null)
+      ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      // check if any records got errors in the customizer - if so, remove them from the input list of pkeys to delete //
+      ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      if(customizerResult != null)
       {
-         deleteOutput.setRecordsWithErrors(new ArrayList<>());
-         outputRecordsWithErrors = deleteOutput.getRecordsWithErrors();
+         Set<Serializable> primaryKeysToRemoveFromInput = new HashSet<>();
+         for(QRecord record : customizerResult)
+         {
+            if(CollectionUtils.nullSafeHasContents(record.getErrors()))
+            {
+               recordsWithValidationErrors.add(record);
+               primaryKeysToRemoveFromInput.add(record.getValue(primaryKeyFieldName));
+            }
+            else if(CollectionUtils.nullSafeHasContents(record.getWarnings()))
+            {
+               recordsWithValidationWarnings.put(record.getValue(primaryKeyFieldName), record);
+            }
+         }
+
+         if(!primaryKeysToRemoveFromInput.isEmpty())
+         {
+            primaryKeys.removeAll(primaryKeysToRemoveFromInput);
+         }
       }
 
+      ////////////////////////////////////
+      // have the backend do the delete //
+      ////////////////////////////////////
+      DeleteOutput deleteOutput = deleteInterface.execute(deleteInput);
+
+      /////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      // reset the input's list of primary keys -- callers may use & expect that to be what they had passed in!! //
+      /////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      deleteInput.setPrimaryKeys(originalPrimaryKeys);
+
+      ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      // merge the backend's output with any validation errors we found (whose pkeys wouldn't have gotten into the backend delete) //
+      ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      List<QRecord> outputRecordsWithErrors = Objects.requireNonNullElseGet(deleteOutput.getRecordsWithErrors(), () -> new ArrayList<>());
       outputRecordsWithErrors.addAll(recordsWithValidationErrors);
 
+      ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      // if a record had a validation warning, but then an execution error, remove it from the warning list - so it's only in one of them. //
+      ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      for(QRecord outputRecordWithError : outputRecordsWithErrors)
+      {
+         Serializable pkey = outputRecordWithError.getValue(primaryKeyFieldName);
+         recordsWithValidationWarnings.remove(pkey);
+      }
+
+      ///////////////////////////////////////////////////////////////////////////////////////////
+      // combine the warning list from validation to that from execution - avoiding duplicates //
+      // use a map to manage this list for the rest of this method                             //
+      ///////////////////////////////////////////////////////////////////////////////////////////
+      Map<Serializable, QRecord> outputRecordsWithWarningMap = CollectionUtils.nullSafeIsEmpty(deleteOutput.getRecordsWithWarnings()) ? new LinkedHashMap<>()
+         : deleteOutput.getRecordsWithWarnings().stream().collect(Collectors.toMap(r -> r.getValue(primaryKeyFieldName), r -> r, (a, b) -> a, () -> new LinkedHashMap<>()));
+      for(Map.Entry<Serializable, QRecord> entry : recordsWithValidationWarnings.entrySet())
+      {
+         if(!outputRecordsWithWarningMap.containsKey(entry.getKey()))
+         {
+            outputRecordsWithWarningMap.put(entry.getKey(), entry.getValue());
+         }
+      }
+
+      ////////////////////////////////////////
+      // delete associations, if applicable //
+      ////////////////////////////////////////
       manageAssociations(deleteInput);
 
-      new DMLAuditAction().execute(new DMLAuditInput().withTableActionInput(deleteInput).withRecordList(recordListForAudit));
+      ///////////////////////////////////
+      // do the audit                  //
+      // todo - add input.omitDmlAudit //
+      ///////////////////////////////////
+      DMLAuditInput dmlAuditInput = new DMLAuditInput().withTableActionInput(deleteInput);
+      oldRecordList.ifPresent(l -> dmlAuditInput.setRecordList(l));
+      new DMLAuditAction().execute(dmlAuditInput);
+
+      //////////////////////////////////////////////////////////////
+      // finally, run the post-delete customizer, if there is one //
+      //////////////////////////////////////////////////////////////
+      Optional<AbstractPostDeleteCustomizer> postDeleteCustomizer = QCodeLoader.getTableCustomizer(AbstractPostDeleteCustomizer.class, table, TableCustomizers.POST_DELETE_RECORD.getRole());
+      if(postDeleteCustomizer.isPresent() && oldRecordList.isPresent())
+      {
+         ////////////////////////////////////////////////////////////////////////////
+         // make list of records that are still good - to pass into the customizer //
+         ////////////////////////////////////////////////////////////////////////////
+         List<QRecord> recordsForCustomizer = makeListOfRecordsNotInErrorList(primaryKeyFieldName, oldRecordList.get(), outputRecordsWithErrors);
+
+         try
+         {
+            postDeleteCustomizer.get().setDeleteInput(deleteInput);
+            List<QRecord> postCustomizerResult = postDeleteCustomizer.get().apply(recordsForCustomizer);
+
+            ///////////////////////////////////////////////////////
+            // check if any records got errors in the customizer //
+            ///////////////////////////////////////////////////////
+            for(QRecord record : postCustomizerResult)
+            {
+               Serializable pkey = record.getValue(primaryKeyFieldName);
+               if(CollectionUtils.nullSafeHasContents(record.getErrors()))
+               {
+                  outputRecordsWithErrors.add(record);
+                  outputRecordsWithWarningMap.remove(pkey);
+               }
+               else if(CollectionUtils.nullSafeHasContents(record.getWarnings()))
+               {
+                  outputRecordsWithWarningMap.put(pkey, record);
+               }
+            }
+         }
+         catch(Exception e)
+         {
+            for(QRecord record : recordsForCustomizer)
+            {
+               record.addWarning(new QWarningMessage("An error occurred after the delete: " + e.getMessage()));
+               outputRecordsWithWarningMap.put(record.getValue(primaryKeyFieldName), record);
+            }
+         }
+      }
+
+      deleteOutput.setRecordsWithErrors(outputRecordsWithErrors);
+      deleteOutput.setRecordsWithWarnings(new ArrayList<>(outputRecordsWithWarningMap.values()));
 
       return deleteOutput;
    }
 
 
 
+   /*******************************************************************************
+    ** this method takes in the deleteInput, and the list of old records that matched
+    ** the pkeys in that input.
+    **
+    ** it'll check if any of those pkeys aren't found (in a sub-method) - a record
+    ** with an error message will be added to oldRecordList for any such records.
+    **
+    ** it'll also then call the pre-customizer, if there is one - taking in the
+    ** oldRecordList.  it can add other errors or warnings to records.
+    **
+    ** The return value here is basically oldRecordList - possibly with some new
+    ** entries for the pkey-not-founds, and possibly w/ errors and warnings from the
+    ** customizer.
+    *******************************************************************************/
+   public List<QRecord> performValidations(DeleteInput deleteInput, Optional<List<QRecord>> oldRecordList, boolean isPreview) throws QException
+   {
+      if(oldRecordList.isEmpty())
+      {
+         return (null);
+      }
+
+      QTableMetaData table               = deleteInput.getTable();
+      List<QRecord>  primaryKeysNotFound = validateRecordsExistAndCanBeAccessed(deleteInput, oldRecordList.get());
+
+      ///////////////////////////////////////////////////////////////////////////
+      // after all validations, run the pre-delete customizer, if there is one //
+      ///////////////////////////////////////////////////////////////////////////
+      Optional<AbstractPreDeleteCustomizer> preDeleteCustomizer = QCodeLoader.getTableCustomizer(AbstractPreDeleteCustomizer.class, table, TableCustomizers.PRE_DELETE_RECORD.getRole());
+      List<QRecord>                         customizerResult    = oldRecordList.get();
+      if(preDeleteCustomizer.isPresent())
+      {
+         preDeleteCustomizer.get().setDeleteInput(deleteInput);
+         preDeleteCustomizer.get().setIsPreview(isPreview);
+         customizerResult = preDeleteCustomizer.get().apply(oldRecordList.get());
+      }
+
+      /////////////////////////////////////////////////////////////////////////
+      // add any pkey-not-found records to the front of the customizerResult //
+      /////////////////////////////////////////////////////////////////////////
+      customizerResult.addAll(primaryKeysNotFound);
+
+      return customizerResult;
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private static List<QRecord> makeListOfRecordsNotInErrorList(String primaryKeyField, List<QRecord> oldRecordList, List<QRecord> outputRecordsWithErrors)
+   {
+      Map<Serializable, QRecord> recordsWithErrorsMap = outputRecordsWithErrors.stream().collect(Collectors.toMap(r -> r.getValue(primaryKeyField), r -> r));
+      List<QRecord>              recordsForCustomizer = new ArrayList<>();
+      for(QRecord record : oldRecordList)
+      {
+         if(!recordsWithErrorsMap.containsKey(record.getValue(primaryKeyField)))
+         {
+            recordsForCustomizer.add(record);
+         }
+      }
+      return recordsForCustomizer;
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/
@@ -157,11 +369,14 @@ public class DeleteAction
          QueryOutput        queryOutput    = new QueryAction().execute(queryInput);
          List<Serializable> associatedKeys = queryOutput.getRecords().stream().map(r -> r.getValue(associatedTable.getPrimaryKeyField())).toList();
 
-         DeleteInput nextLevelDeleteInput = new DeleteInput();
-         nextLevelDeleteInput.setTransaction(deleteInput.getTransaction());
-         nextLevelDeleteInput.setTableName(association.getAssociatedTableName());
-         nextLevelDeleteInput.setPrimaryKeys(associatedKeys);
-         DeleteOutput nextLevelDeleteOutput = new DeleteAction().execute(nextLevelDeleteInput);
+         if(CollectionUtils.nullSafeHasContents(associatedKeys))
+         {
+            DeleteInput nextLevelDeleteInput = new DeleteInput();
+            nextLevelDeleteInput.setTransaction(deleteInput.getTransaction());
+            nextLevelDeleteInput.setTableName(association.getAssociatedTableName());
+            nextLevelDeleteInput.setPrimaryKeys(associatedKeys);
+            DeleteOutput nextLevelDeleteOutput = new DeleteAction().execute(nextLevelDeleteInput);
+         }
       }
    }
 
@@ -170,12 +385,9 @@ public class DeleteAction
    /*******************************************************************************
     **
     *******************************************************************************/
-   private static List<QRecord> getRecordListForAuditIfNeeded(DeleteInput deleteInput) throws QException
+   private static Optional<List<QRecord>> fetchOldRecords(DeleteInput deleteInput, DeleteInterface deleteInterface) throws QException
    {
-      List<QRecord> recordListForAudit = null;
-
-      AuditLevel auditLevel = DMLAuditAction.getAuditLevel(deleteInput);
-      if(AuditLevel.RECORD.equals(auditLevel) || AuditLevel.FIELD.equals(auditLevel))
+      if(deleteInterface.supportsPreFetchQuery())
       {
          List<Serializable> primaryKeyList = deleteInput.getPrimaryKeys();
          if(CollectionUtils.nullSafeIsEmpty(deleteInput.getPrimaryKeys()) && deleteInput.getQueryFilter() != null)
@@ -185,19 +397,16 @@ public class DeleteAction
 
          if(CollectionUtils.nullSafeHasContents(primaryKeyList))
          {
-            ////////////////////////////////////////////////////////////////////////////////////
-            // always fetch the records - we'll use them anyway for checking not-exist  below //
-            ////////////////////////////////////////////////////////////////////////////////////
             QueryInput queryInput = new QueryInput();
             queryInput.setTransaction(deleteInput.getTransaction());
             queryInput.setTableName(deleteInput.getTableName());
             queryInput.setFilter(new QQueryFilter(new QFilterCriteria(deleteInput.getTable().getPrimaryKeyField(), QCriteriaOperator.IN, primaryKeyList)));
             QueryOutput queryOutput = new QueryAction().execute(queryInput);
-            recordListForAudit = queryOutput.getRecords();
+            return (Optional.of(queryOutput.getRecords()));
          }
       }
 
-      return (recordListForAudit);
+      return (Optional.empty());
    }
 
 
@@ -207,9 +416,9 @@ public class DeleteAction
     ** records that you can't see because of security - that they won't be found
     ** by the query here, so it's the same to you as if they don't exist at all!
     **
-    ** This method, if it finds any missing records, will:
-    ** - remove those ids from the deleteInput
-    ** - create a QRecord with that id and a not-found error message.
+    ** If this method identifies any missing records (e.g., from PKeys that are
+    ** requested to be deleted, but don't exist (or can't be seen)), then it will
+    ** return those as new QRecords, with error messages.
     *******************************************************************************/
    private List<QRecord> validateRecordsExistAndCanBeAccessed(DeleteInput deleteInput, List<QRecord> oldRecordList) throws QException
    {
@@ -218,64 +427,28 @@ public class DeleteAction
       QTableMetaData table           = deleteInput.getTable();
       QFieldMetaData primaryKeyField = table.getField(table.getPrimaryKeyField());
 
-      Set<Serializable> primaryKeysToRemoveFromInput = new HashSet<>();
-
       List<List<Serializable>> pages = CollectionUtils.getPages(deleteInput.getPrimaryKeys(), 1000);
       for(List<Serializable> page : pages)
       {
-         List<Serializable> primaryKeysToLookup = new ArrayList<>();
-         for(Serializable primaryKeyValue : page)
+         Map<Serializable, QRecord> oldRecordMapByPrimaryKey = new HashMap<>();
+         for(QRecord record : oldRecordList)
          {
-            if(primaryKeyValue != null)
-            {
-               primaryKeysToLookup.add(primaryKeyValue);
-            }
-         }
-
-         Map<Serializable, QRecord> lookedUpRecords = new HashMap<>();
-         if(CollectionUtils.nullSafeHasContents(oldRecordList))
-         {
-            for(QRecord record : oldRecordList)
-            {
-               Serializable primaryKeyValue = record.getValue(table.getPrimaryKeyField());
-               primaryKeyValue = ValueUtils.getValueAsFieldType(primaryKeyField.getType(), primaryKeyValue);
-               lookedUpRecords.put(primaryKeyValue, record);
-            }
-         }
-         else if(!primaryKeysToLookup.isEmpty())
-         {
-            QueryInput queryInput = new QueryInput();
-            queryInput.setTransaction(deleteInput.getTransaction());
-            queryInput.setTableName(table.getName());
-            queryInput.setFilter(new QQueryFilter(new QFilterCriteria(table.getPrimaryKeyField(), QCriteriaOperator.IN, primaryKeysToLookup)));
-            QueryOutput queryOutput = new QueryAction().execute(queryInput);
-            for(QRecord record : queryOutput.getRecords())
-            {
-               lookedUpRecords.put(record.getValue(table.getPrimaryKeyField()), record);
-            }
+            Serializable primaryKeyValue = record.getValue(table.getPrimaryKeyField());
+            primaryKeyValue = ValueUtils.getValueAsFieldType(primaryKeyField.getType(), primaryKeyValue);
+            oldRecordMapByPrimaryKey.put(primaryKeyValue, record);
          }
 
          for(Serializable primaryKeyValue : page)
          {
             primaryKeyValue = ValueUtils.getValueAsFieldType(primaryKeyField.getType(), primaryKeyValue);
-            if(!lookedUpRecords.containsKey(primaryKeyValue))
+            if(!oldRecordMapByPrimaryKey.containsKey(primaryKeyValue))
             {
                QRecord recordWithError = new QRecord();
                recordsWithErrors.add(recordWithError);
                recordWithError.setValue(primaryKeyField.getName(), primaryKeyValue);
-               recordWithError.addError(NOT_FOUND_ERROR_PREFIX + " for " + primaryKeyField.getLabel() + " = " + primaryKeyValue);
-               primaryKeysToRemoveFromInput.add(primaryKeyValue);
+               recordWithError.addError(new NotFoundStatusMessage("No record was found to delete for " + primaryKeyField.getLabel() + " = " + primaryKeyValue));
             }
          }
-
-         /////////////////////////////////////////////////////////////////
-         // do one mass removal of any bad keys from the input key list //
-         /////////////////////////////////////////////////////////////////
-         if(!primaryKeysToRemoveFromInput.isEmpty())
-         {
-            deleteInput.getPrimaryKeys().removeAll(primaryKeysToRemoveFromInput);
-            primaryKeysToRemoveFromInput.clear();
-         }
       }
 
       return (recordsWithErrors);

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/GetAction.java

@@ -35,6 +35,7 @@ import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizers;
 import com.kingsrook.qqq.backend.core.actions.interfaces.GetInterface;
 import com.kingsrook.qqq.backend.core.actions.values.QPossibleValueTranslator;
 import com.kingsrook.qqq.backend.core.actions.values.QValueFormatter;
+import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.logging.LogPair;
 import com.kingsrook.qqq.backend.core.logging.QLogger;
@@ -51,6 +52,8 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateOutput;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.AdornmentType;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.cache.CacheUseCase;
 import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleDispatcher;
@@ -127,29 +130,14 @@ public class GetAction
             ///////////////////////////////////////////////////////////////////////
             // if the record wasn't found, see if we should look in cache-source //
             ///////////////////////////////////////////////////////////////////////
-            QRecord recordFromSource = tryToGetFromCacheSource(getInput, getOutput);
+            QRecord recordFromSource = tryToGetFromCacheSource(getInput);
             if(recordFromSource != null)
             {
+               /////////////////////////////////////////////////////////////////////////////////////////////////
+               // good, we found a record from the source, make sure we should cache it, and if so, do it now //
+               /////////////////////////////////////////////////////////////////////////////////////////////////
                QRecord recordToCache     = mapSourceRecordToCacheRecord(table, recordFromSource);
-               boolean shouldCacheRecord = true;
-
-               ////////////////////////////////////////////////////////////////////////////////
-               // see if there are any exclustions that need to be considered for this table //
-               ////////////////////////////////////////////////////////////////////////////////
-               recordMatchExclusionLoop:
-               for(CacheUseCase useCase : CollectionUtils.nonNullList(table.getCacheOf().getUseCases()))
-               {
-                  for(QQueryFilter filter : CollectionUtils.nonNullList(useCase.getExcludeRecordsMatching()))
-                  {
-                     if(BackendQueryFilterUtils.doesRecordMatch(filter, recordToCache))
-                     {
-                        LOG.info("Not caching record because it matches a use case's filter exclusion", new LogPair("record", recordToCache), new LogPair("filter", filter));
-                        shouldCacheRecord = false;
-                        break recordMatchExclusionLoop;
-                     }
-                  }
-               }
-
+               boolean shouldCacheRecord = shouldCacheRecord(table, recordToCache);
                if(shouldCacheRecord)
                {
                   InsertInput insertInput = new InsertInput();
@@ -182,12 +170,50 @@ public class GetAction
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private boolean shouldCacheRecord(QTableMetaData table, QRecord recordToCache)
+   {
+      boolean shouldCacheRecord = true;
+      recordMatchExclusionLoop:
+      for(CacheUseCase useCase : CollectionUtils.nonNullList(table.getCacheOf().getUseCases()))
+      {
+         for(QQueryFilter filter : CollectionUtils.nonNullList(useCase.getExcludeRecordsMatching()))
+         {
+            if(BackendQueryFilterUtils.doesRecordMatch(filter, recordToCache))
+            {
+               LOG.info("Not caching record because it matches a use case's filter exclusion", new LogPair("record", recordToCache), new LogPair("filter", filter));
+               shouldCacheRecord = false;
+               break recordMatchExclusionLoop;
+            }
+         }
+      }
+
+      return (shouldCacheRecord);
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/
    private static QRecord mapSourceRecordToCacheRecord(QTableMetaData table, QRecord recordFromSource)
    {
       QRecord cacheRecord = new QRecord(recordFromSource);
+
+      //////////////////////////////////////////////////////////////////////////////////////////////
+      // make sure every value in the qRecord is set, because we will possibly be doing an update //
+      // on this record and want to null out any fields not set, not leave them populated         //
+      //////////////////////////////////////////////////////////////////////////////////////////////
+      for(String fieldName : table.getFields().keySet())
+      {
+         if(!cacheRecord.getValues().containsKey(fieldName))
+         {
+            cacheRecord.setValue(fieldName, null);
+         }
+      }
+
       if(StringUtils.hasContent(table.getCacheOf().getCachedDateFieldName()))
       {
          cacheRecord.setValue(table.getCacheOf().getCachedDateFieldName(), Instant.now());
@@ -210,33 +236,54 @@ public class GetAction
          Instant cachedDate   = cachedRecord.getValueInstant(table.getCacheOf().getCachedDateFieldName());
          if(cachedDate == null || cachedDate.isBefore(Instant.now().minus(expirationSeconds, ChronoUnit.SECONDS)))
          {
-            QRecord recordFromSource = tryToGetFromCacheSource(getInput, getOutput);
+            //////////////////////////////////////////////////////////////////////////
+            // keep the serial key from the old record in case we need to delete it //
+            //////////////////////////////////////////////////////////////////////////
+            Serializable oldRecordPrimaryKey      = getOutput.getRecord().getValue(table.getPrimaryKeyField());
+            boolean      shouldDeleteCachedRecord = true;
+
+            ///////////////////////////////////////////
+            // fetch record from original source now //
+            ///////////////////////////////////////////
+            QRecord recordFromSource = tryToGetFromCacheSource(getInput);
             if(recordFromSource != null)
             {
-               ///////////////////////////////////////////////////////////////////
-               // if the record was found in the source, update it in the cache //
-               ///////////////////////////////////////////////////////////////////
+               //////////////////////////////////////////////////////////////////////
+               // if the record was found in the source, put it into the output    //
+               // object so returned back to caller, check that it should actually //
+               // be cached before doing so                                        //
+               //////////////////////////////////////////////////////////////////////
                QRecord recordToCache = mapSourceRecordToCacheRecord(table, recordFromSource);
                recordToCache.setValue(table.getPrimaryKeyField(), cachedRecord.getValue(table.getPrimaryKeyField()));
+               getOutput.setRecord(recordToCache);
 
-               UpdateInput updateInput = new UpdateInput();
-               updateInput.setTableName(getInput.getTableName());
-               updateInput.setRecords(List.of(recordToCache));
-               UpdateOutput updateOutput = new UpdateAction().execute(updateInput);
-
-               getOutput.setRecord(updateOutput.getRecords().get(0));
+               if(shouldCacheRecord(table, recordToCache))
+               {
+                  UpdateInput updateInput = new UpdateInput();
+                  updateInput.setTableName(getInput.getTableName());
+                  updateInput.setRecords(List.of(recordToCache));
+                  UpdateOutput updateOutput = new UpdateAction().execute(updateInput);
+                  getOutput.setRecord(updateOutput.getRecords().get(0));
+                  shouldDeleteCachedRecord = false;
+               }
             }
             else
+            {
+               ///////////////////////////////////////////////////////////////////////////////////////
+               // if we did not get a record back from the source, empty out the getOutput's record //
+               ///////////////////////////////////////////////////////////////////////////////////////
+               getOutput.setRecord(null);
+            }
+
+            if(shouldDeleteCachedRecord)
             {
                /////////////////////////////////////////////////////////////////////////////
                // if the record is no longer in the source, then remove it from the cache //
                /////////////////////////////////////////////////////////////////////////////
                DeleteInput deleteInput = new DeleteInput();
                deleteInput.setTableName(getInput.getTableName());
-               deleteInput.setPrimaryKeys(List.of(getOutput.getRecord().getValue(table.getPrimaryKeyField())));
+               deleteInput.setPrimaryKeys(List.of(oldRecordPrimaryKey));
                new DeleteAction().execute(deleteInput);
-
-               getOutput.setRecord(null);
             }
          }
       }
@@ -247,7 +294,7 @@ public class GetAction
    /*******************************************************************************
     **
     *******************************************************************************/
-   private QRecord tryToGetFromCacheSource(GetInput getInput, GetOutput getOutput) throws QException
+   private QRecord tryToGetFromCacheSource(GetInput getInput) throws QException
    {
       QRecord        recordFromSource = null;
       QTableMetaData table            = getInput.getTable();
@@ -332,6 +379,8 @@ public class GetAction
          queryInput.setIncludeAssociations(getInput.getIncludeAssociations());
          queryInput.setAssociationNamesToInclude(getInput.getAssociationNamesToInclude());
          queryInput.setShouldFetchHeavyFields(getInput.getShouldFetchHeavyFields());
+         queryInput.setShouldMaskPasswords(getInput.getShouldMaskPasswords());
+         queryInput.setShouldOmitHiddenFields(getInput.getShouldOmitHiddenFields());
 
          QueryOutput queryOutput = new QueryAction().execute(queryInput);
 
@@ -372,6 +421,32 @@ public class GetAction
          QValueFormatter.setDisplayValuesInRecords(getInput.getTable(), List.of(returnRecord));
       }
 
+      if(getInput.getShouldOmitHiddenFields() || getInput.getShouldMaskPasswords())
+      {
+         Map<String, QFieldMetaData> fields = QContext.getQInstance().getTable(getInput.getTableName()).getFields();
+         for(String fieldName : fields.keySet())
+         {
+            QFieldMetaData field = fields.get(fieldName);
+            if(getInput.getShouldOmitHiddenFields() && field.getIsHidden())
+            {
+               returnRecord.removeValue(fieldName);
+            }
+            else if(getInput.getShouldMaskPasswords() && field.getType() != null && field.getType().needsMasked() && !field.hasAdornmentType(AdornmentType.REVEAL))
+            {
+               //////////////////////////////////////////////////////////////////////
+               // empty out the value completely first (which will remove from     //
+               // display fields as well) then update display value if flag is set //
+               //////////////////////////////////////////////////////////////////////
+               returnRecord.removeValue(fieldName);
+               returnRecord.setValue(fieldName, "************");
+               if(getInput.getShouldGenerateDisplayValues())
+               {
+                  returnRecord.setDisplayValue(fieldName, record.getValueString(fieldName));
+               }
+            }
+         }
+      }
+
       //////////////////////////////////////////////////////////////////////////////
       // note - shouldFetchHeavyFields should be handled by the underlying action //
       //////////////////////////////////////////////////////////////////////////////

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/InsertAction.java

@@ -38,8 +38,10 @@ import com.kingsrook.qqq.backend.core.actions.audits.DMLAuditAction;
 import com.kingsrook.qqq.backend.core.actions.automation.AutomationStatus;
 import com.kingsrook.qqq.backend.core.actions.automation.RecordAutomationStatusUpdater;
 import com.kingsrook.qqq.backend.core.actions.customizers.AbstractPostInsertCustomizer;
+import com.kingsrook.qqq.backend.core.actions.customizers.AbstractPreInsertCustomizer;
 import com.kingsrook.qqq.backend.core.actions.customizers.QCodeLoader;
 import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizers;
+import com.kingsrook.qqq.backend.core.actions.interfaces.InsertInterface;
 import com.kingsrook.qqq.backend.core.actions.tables.helpers.UniqueKeyHelper;
 import com.kingsrook.qqq.backend.core.actions.tables.helpers.ValidateRecordSecurityLockHelper;
 import com.kingsrook.qqq.backend.core.actions.values.ValueBehaviorApplier;
@@ -51,14 +53,18 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertOutput;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
 import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinOn;
 import com.kingsrook.qqq.backend.core.model.metadata.joins.QJoinMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.Association;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.UniqueKey;
+import com.kingsrook.qqq.backend.core.model.statusmessages.BadInputStatusMessage;
+import com.kingsrook.qqq.backend.core.model.statusmessages.QWarningMessage;
 import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleDispatcher;
 import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleInterface;
 import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
+import com.kingsrook.qqq.backend.core.utils.ValueUtils;
 import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
 
 
@@ -86,28 +92,41 @@ public class InsertAction extends AbstractQActionFunction<InsertInput, InsertOut
          throw (new QException("Error:  Undefined table: " + insertInput.getTableName()));
       }
 
-      Optional<AbstractPostInsertCustomizer> postInsertCustomizer = QCodeLoader.getTableCustomizer(AbstractPostInsertCustomizer.class, table, TableCustomizers.POST_INSERT_RECORD.getRole());
       setAutomationStatusField(insertInput);
 
-      QBackendModuleInterface qModule = getBackendModuleInterface(insertInput);
-      // todo pre-customization - just get to modify the request?
+      //////////////////////////////////////////////////////
+      // load the backend module and its insert interface //
+      //////////////////////////////////////////////////////
+      QBackendModuleInterface qModule         = getBackendModuleInterface(insertInput);
+      InsertInterface         insertInterface = qModule.getInsertInterface();
 
-      ValueBehaviorApplier.applyFieldBehaviors(insertInput.getInstance(), table, insertInput.getRecords());
-      setErrorsIfUniqueKeyErrors(insertInput, table);
-      validateRequiredFields(insertInput);
-      ValidateRecordSecurityLockHelper.validateSecurityFields(insertInput.getTable(), insertInput.getRecords(), ValidateRecordSecurityLockHelper.Action.INSERT);
+      /////////////////////////////
+      // run standard validators //
+      /////////////////////////////
+      performValidations(insertInput, false);
 
-      InsertOutput insertOutput = qModule.getInsertInterface().execute(insertInput);
-      List<String> errors       = insertOutput.getRecords().stream().flatMap(r -> r.getErrors().stream()).toList();
+      ////////////////////////////////////
+      // have the backend do the insert //
+      ////////////////////////////////////
+      InsertOutput insertOutput = insertInterface.execute(insertInput);
+
+      //////////////////////////////
+      // log if there were errors //
+      //////////////////////////////
+      List<String> errors = insertOutput.getRecords().stream().flatMap(r -> r.getErrors().stream().map(Object::toString)).toList();
       if(CollectionUtils.nullSafeHasContents(errors))
       {
-         LOG.warn("Errors in insertAction", logPair("tableName", table.getName()), logPair("errorCount", errors.size()), errors.size() < 10 ? logPair("errors", errors) : logPair("first10Errors", errors.subList(0, 10)));
+         LOG.info("Errors in insertAction", logPair("tableName", table.getName()), logPair("errorCount", errors.size()), errors.size() < 10 ? logPair("errors", errors) : logPair("first10Errors", errors.subList(0, 10)));
       }
 
+      //////////////////////////////////////////////////
+      // insert any associations in the input records //
+      //////////////////////////////////////////////////
       manageAssociations(table, insertOutput.getRecords(), insertInput.getTransaction());
 
-      // todo post-customization - can do whatever w/ the result if you want
-
+      //////////////////
+      // do the audit //
+      //////////////////
       if(insertInput.getOmitDmlAudit())
       {
          LOG.debug("Requested to omit DML audit");
@@ -117,10 +136,24 @@ public class InsertAction extends AbstractQActionFunction<InsertInput, InsertOut
          new DMLAuditAction().execute(new DMLAuditInput().withTableActionInput(insertInput).withRecordList(insertOutput.getRecords()));
       }
 
+      //////////////////////////////////////////////////////////////
+      // finally, run the post-insert customizer, if there is one //
+      //////////////////////////////////////////////////////////////
+      Optional<AbstractPostInsertCustomizer> postInsertCustomizer = QCodeLoader.getTableCustomizer(AbstractPostInsertCustomizer.class, table, TableCustomizers.POST_INSERT_RECORD.getRole());
       if(postInsertCustomizer.isPresent())
       {
-         postInsertCustomizer.get().setInsertInput(insertInput);
-         insertOutput.setRecords(postInsertCustomizer.get().apply(insertOutput.getRecords()));
+         try
+         {
+            postInsertCustomizer.get().setInsertInput(insertInput);
+            insertOutput.setRecords(postInsertCustomizer.get().apply(insertOutput.getRecords()));
+         }
+         catch(Exception e)
+         {
+            for(QRecord record : insertOutput.getRecords())
+            {
+               record.addWarning(new QWarningMessage("An error occurred after the insert: " + e.getMessage()));
+            }
+         }
       }
 
       return insertOutput;
@@ -128,6 +161,37 @@ public class InsertAction extends AbstractQActionFunction<InsertInput, InsertOut
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public void performValidations(InsertInput insertInput, boolean isPreview) throws QException
+   {
+      QTableMetaData table = insertInput.getTable();
+
+      ValueBehaviorApplier.applyFieldBehaviors(insertInput.getInstance(), table, insertInput.getRecords());
+      setErrorsIfUniqueKeyErrors(insertInput, table);
+
+      if(insertInput.getInputSource().shouldValidateRequiredFields())
+      {
+         validateRequiredFields(insertInput);
+      }
+
+      ValidateRecordSecurityLockHelper.validateSecurityFields(insertInput.getTable(), insertInput.getRecords(), ValidateRecordSecurityLockHelper.Action.INSERT);
+
+      ///////////////////////////////////////////////////////////////////////////
+      // after all validations, run the pre-insert customizer, if there is one //
+      ///////////////////////////////////////////////////////////////////////////
+      Optional<AbstractPreInsertCustomizer> preInsertCustomizer = QCodeLoader.getTableCustomizer(AbstractPreInsertCustomizer.class, table, TableCustomizers.PRE_INSERT_RECORD.getRole());
+      if(preInsertCustomizer.isPresent())
+      {
+         preInsertCustomizer.get().setInsertInput(insertInput);
+         preInsertCustomizer.get().setIsPreview(isPreview);
+         insertInput.setRecords(preInsertCustomizer.get().apply(insertInput.getRecords()));
+      }
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/
@@ -146,7 +210,7 @@ public class InsertAction extends AbstractQActionFunction<InsertInput, InsertOut
             {
                if(record.getValue(requiredField.getName()) == null || (requiredField.getType().isStringLike() && record.getValueString(requiredField.getName()).trim().equals("")))
                {
-                  record.addError("Missing value in required field: " + requiredField.getLabel());
+                  record.addError(new BadInputStatusMessage("Missing value in required field: " + requiredField.getLabel()));
                }
             }
          }
@@ -169,24 +233,33 @@ public class InsertAction extends AbstractQActionFunction<InsertInput, InsertOut
          List<QRecord> nextLevelInserts = new ArrayList<>();
          for(QRecord record : insertedRecords)
          {
+            if(CollectionUtils.nullSafeHasContents(record.getErrors()))
+            {
+               continue;
+            }
+
             if(record.getAssociatedRecords() != null && record.getAssociatedRecords().containsKey(association.getName()))
             {
                for(QRecord associatedRecord : CollectionUtils.nonNullList(record.getAssociatedRecords().get(association.getName())))
                {
                   for(JoinOn joinOn : join.getJoinOns())
                   {
-                     associatedRecord.setValue(joinOn.getRightField(), record.getValue(joinOn.getLeftField()));
+                     QFieldType type = table.getField(joinOn.getLeftField()).getType();
+                     associatedRecord.setValue(joinOn.getRightField(), ValueUtils.getValueAsFieldType(type, record.getValue(joinOn.getLeftField())));
                   }
                   nextLevelInserts.add(associatedRecord);
                }
             }
          }
 
-         InsertInput nextLevelInsertInput = new InsertInput();
-         nextLevelInsertInput.setTransaction(transaction);
-         nextLevelInsertInput.setTableName(association.getAssociatedTableName());
-         nextLevelInsertInput.setRecords(nextLevelInserts);
-         InsertOutput nextLevelInsertOutput = new InsertAction().execute(nextLevelInsertInput);
+         if(CollectionUtils.nullSafeHasContents(nextLevelInserts))
+         {
+            InsertInput nextLevelInsertInput = new InsertInput();
+            nextLevelInsertInput.setTransaction(transaction);
+            nextLevelInsertInput.setTableName(association.getAssociatedTableName());
+            nextLevelInsertInput.setRecords(nextLevelInserts);
+            InsertOutput nextLevelInsertOutput = new InsertAction().execute(nextLevelInsertInput);
+         }
       }
    }
 
@@ -232,7 +305,7 @@ public class InsertAction extends AbstractQActionFunction<InsertInput, InsertOut
                Optional<List<Serializable>> keyValues = UniqueKeyHelper.getKeyValues(table, uniqueKey, record);
                if(keyValues.isPresent() && (existingKeys.get(uniqueKey).contains(keyValues.get()) || keysInThisList.get(uniqueKey).contains(keyValues.get())))
                {
-                  record.addError("Another record already exists with this " + uniqueKey.getDescription(table));
+                  record.addError(new BadInputStatusMessage("Another record already exists with this " + uniqueKey.getDescription(table)));
                   foundDupe = true;
                   break;
                }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/QueryAction.java

@@ -27,6 +27,7 @@ import java.util.ArrayList;
 import java.util.Collection;
 import java.util.HashSet;
 import java.util.List;
+import java.util.Map;
 import java.util.Optional;
 import java.util.Set;
 import com.kingsrook.qqq.backend.core.actions.ActionHelper;
@@ -34,6 +35,7 @@ import com.kingsrook.qqq.backend.core.actions.customizers.AbstractPostQueryCusto
 import com.kingsrook.qqq.backend.core.actions.customizers.QCodeLoader;
 import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizers;
 import com.kingsrook.qqq.backend.core.actions.reporting.BufferedRecordPipe;
+import com.kingsrook.qqq.backend.core.actions.reporting.RecordPipeBufferedWrapper;
 import com.kingsrook.qqq.backend.core.actions.values.QPossibleValueTranslator;
 import com.kingsrook.qqq.backend.core.actions.values.QValueFormatter;
 import com.kingsrook.qqq.backend.core.context.QContext;
@@ -45,6 +47,8 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.AdornmentType;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinOn;
 import com.kingsrook.qqq.backend.core.model.metadata.joins.QJoinMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.Association;
@@ -53,6 +57,7 @@ import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleDispatcher;
 import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleInterface;
 import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
 import com.kingsrook.qqq.backend.core.utils.ListingHash;
+import com.kingsrook.qqq.backend.core.utils.ValueUtils;
 
 
 /*******************************************************************************
@@ -77,20 +82,31 @@ public class QueryAction
    {
       ActionHelper.validateSession(queryInput);
 
+      if(queryInput.getTableName() == null)
+      {
+         throw (new QException("Table name was not specified in query input"));
+      }
+
+      if(queryInput.getTable() == null)
+      {
+         throw (new QException("A table named [" + queryInput.getTableName() + "] was not found in the active QInstance"));
+      }
+
       postQueryRecordCustomizer = QCodeLoader.getTableCustomizer(AbstractPostQueryCustomizer.class, queryInput.getTable(), TableCustomizers.POST_QUERY_RECORD.getRole());
       this.queryInput = queryInput;
 
       if(queryInput.getRecordPipe() != null)
       {
          queryInput.getRecordPipe().setPostRecordActions(this::postRecordActions);
-      }
 
-      if(queryInput.getIncludeAssociations() && queryInput.getRecordPipe() != null)
-      {
-         //////////////////////////////////////////////
-         // todo - support this in the future maybe? //
-         //////////////////////////////////////////////
-         throw (new QException("Associations may not be fetched into a RecordPipe."));
+         if(queryInput.getIncludeAssociations())
+         {
+            //////////////////////////////////////////////////////////////////////////////////////////
+            // if the user requested to include associations, it's important that that is buffered, //
+            // (for performance reasons), so, wrap the user's pipe with a buffer                    //
+            //////////////////////////////////////////////////////////////////////////////////////////
+            queryInput.setRecordPipe(new RecordPipeBufferedWrapper(queryInput.getRecordPipe()));
+         }
       }
 
       QBackendModuleDispatcher qBackendModuleDispatcher = new QBackendModuleDispatcher();
@@ -109,11 +125,6 @@ public class QueryAction
          postRecordActions(queryOutput.getRecords());
       }
 
-      if(queryInput.getIncludeAssociations())
-      {
-         manageAssociations(queryInput, queryOutput);
-      }
-
       return queryOutput;
    }
 
@@ -122,7 +133,7 @@ public class QueryAction
    /*******************************************************************************
     **
     *******************************************************************************/
-   private void manageAssociations(QueryInput queryInput, QueryOutput queryOutput) throws QException
+   private void manageAssociations(QueryInput queryInput, List<QRecord> queryOutputRecords) throws QException
    {
       QTableMetaData table = queryInput.getTable();
       for(Association association : CollectionUtils.nonNullList(table.getAssociations()))
@@ -147,11 +158,12 @@ public class QueryAction
             {
                JoinOn            joinOn = join.getJoinOns().get(0);
                Set<Serializable> values = new HashSet<>();
-               for(QRecord record : queryOutput.getRecords())
+               for(QRecord record : queryOutputRecords)
                {
-                  Serializable value = record.getValue(joinOn.getLeftField());
-                  values.add(value);
-                  outerResultMap.add(List.of(value), record);
+                  Serializable value       = record.getValue(joinOn.getLeftField());
+                  Serializable valueAsType = ValueUtils.getValueAsFieldType(table.getField(joinOn.getLeftField()).getType(), value);
+                  values.add(valueAsType);
+                  outerResultMap.add(List.of(valueAsType), record);
                }
                filter.addCriteria(new QFilterCriteria(joinOn.getRightField(), QCriteriaOperator.IN, new ArrayList<>(values)));
             }
@@ -159,7 +171,7 @@ public class QueryAction
             {
                filter.setBooleanOperator(QQueryFilter.BooleanOperator.OR);
 
-               for(QRecord record : queryOutput.getRecords())
+               for(QRecord record : queryOutputRecords)
                {
                   QQueryFilter subFilter = new QQueryFilter();
                   filter.addSubFilter(subFilter);
@@ -227,7 +239,7 @@ public class QueryAction
     ** not one created via List.of()).  This may include setting display values,
     ** translating possible values, and running post-record customizations.
     *******************************************************************************/
-   public void postRecordActions(List<QRecord> records)
+   public void postRecordActions(List<QRecord> records) throws QException
    {
       if(this.postQueryRecordCustomizer.isPresent())
       {
@@ -247,5 +259,64 @@ public class QueryAction
       {
          QValueFormatter.setDisplayValuesInRecords(queryInput.getTable(), records);
       }
+
+      if(queryInput.getIncludeAssociations())
+      {
+         manageAssociations(queryInput, records);
+      }
+
+      //////////////////////////////
+      // mask any password fields //
+      //////////////////////////////
+      if(queryInput.getShouldOmitHiddenFields() || queryInput.getShouldMaskPasswords())
+      {
+         Set<String> maskedFields = new HashSet<>();
+         Set<String> hiddenFields = new HashSet<>();
+
+         //////////////////////////////////////////////////
+         // build up sets of passwords and hidden fields //
+         //////////////////////////////////////////////////
+         Map<String, QFieldMetaData> fields = QContext.getQInstance().getTable(queryInput.getTableName()).getFields();
+         for(String fieldName : fields.keySet())
+         {
+            QFieldMetaData field = fields.get(fieldName);
+            if(queryInput.getShouldOmitHiddenFields() && field.getIsHidden())
+            {
+               hiddenFields.add(fieldName);
+            }
+            else if(queryInput.getShouldMaskPasswords() && field.getType() != null && field.getType().needsMasked() && !field.hasAdornmentType(AdornmentType.REVEAL))
+            {
+               maskedFields.add(fieldName);
+            }
+         }
+
+         /////////////////////////////////////////////////////
+         // iterate over records replacing values with mask //
+         /////////////////////////////////////////////////////
+         for(QRecord record : records)
+         {
+            /////////////////////////
+            // clear hidden fields //
+            /////////////////////////
+            for(String hiddenFieldName : hiddenFields)
+            {
+               record.removeValue(hiddenFieldName);
+            }
+
+            for(String maskedFieldName : maskedFields)
+            {
+               //////////////////////////////////////////////////////////////////////
+               // empty out the value completely first (which will remove from     //
+               // display fields as well) then update display value if flag is set //
+               //////////////////////////////////////////////////////////////////////
+               record.removeValue(maskedFieldName);
+               record.setValue(maskedFieldName, "************");
+               if(queryInput.getShouldGenerateDisplayValues())
+               {
+                  record.setDisplayValue(maskedFieldName, record.getValueString(maskedFieldName));
+               }
+            }
+         }
+      }
    }
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/UpdateAction.java

@@ -27,12 +27,18 @@ import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Optional;
 import java.util.Set;
 import java.util.stream.Collectors;
 import com.kingsrook.qqq.backend.core.actions.ActionHelper;
 import com.kingsrook.qqq.backend.core.actions.audits.DMLAuditAction;
 import com.kingsrook.qqq.backend.core.actions.automation.AutomationStatus;
 import com.kingsrook.qqq.backend.core.actions.automation.RecordAutomationStatusUpdater;
+import com.kingsrook.qqq.backend.core.actions.customizers.AbstractPostUpdateCustomizer;
+import com.kingsrook.qqq.backend.core.actions.customizers.AbstractPreUpdateCustomizer;
+import com.kingsrook.qqq.backend.core.actions.customizers.QCodeLoader;
+import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizers;
+import com.kingsrook.qqq.backend.core.actions.interfaces.UpdateInterface;
 import com.kingsrook.qqq.backend.core.actions.tables.helpers.ValidateRecordSecurityLockHelper;
 import com.kingsrook.qqq.backend.core.actions.values.ValueBehaviorApplier;
 import com.kingsrook.qqq.backend.core.context.QContext;
@@ -51,12 +57,15 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateOutput;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
-import com.kingsrook.qqq.backend.core.model.metadata.audits.AuditLevel;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
 import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinOn;
 import com.kingsrook.qqq.backend.core.model.metadata.joins.QJoinMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.Association;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+import com.kingsrook.qqq.backend.core.model.statusmessages.BadInputStatusMessage;
+import com.kingsrook.qqq.backend.core.model.statusmessages.NotFoundStatusMessage;
+import com.kingsrook.qqq.backend.core.model.statusmessages.QWarningMessage;
 import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleDispatcher;
 import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleInterface;
 import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
@@ -72,8 +81,6 @@ public class UpdateAction
 {
    private static final QLogger LOG = QLogger.getLogger(UpdateAction.class);
 
-   public static final String NOT_FOUND_ERROR_PREFIX = "No record was found to update";
-
 
 
    /*******************************************************************************
@@ -84,38 +91,78 @@ public class UpdateAction
       ActionHelper.validateSession(updateInput);
       setAutomationStatusField(updateInput);
 
-      ValueBehaviorApplier.applyFieldBehaviors(updateInput.getInstance(), updateInput.getTable(), updateInput.getRecords());
-      // todo - need to handle records with errors coming out of here...
-
-      List<QRecord> oldRecordList = getOldRecordListForAuditIfNeeded(updateInput);
+      QTableMetaData table = updateInput.getTable();
 
+      //////////////////////////////////////////////////////
+      // load the backend module and its update interface //
+      //////////////////////////////////////////////////////
       QBackendModuleDispatcher qBackendModuleDispatcher = new QBackendModuleDispatcher();
       QBackendModuleInterface  qModule                  = qBackendModuleDispatcher.getQBackendModule(updateInput.getBackend());
+      UpdateInterface          updateInterface          = qModule.getUpdateInterface();
 
-      validatePrimaryKeysAreGiven(updateInput);
-      validateRecordsExistAndCanBeAccessed(updateInput, oldRecordList);
-      validateRequiredFields(updateInput);
-      ValidateRecordSecurityLockHelper.validateSecurityFields(updateInput.getTable(), updateInput.getRecords(), ValidateRecordSecurityLockHelper.Action.UPDATE);
+      ////////////////////////////////////////////////////////////////////////////////
+      // fetch the old list of records (if the backend supports it), for audits,    //
+      // for "not-found detection", and for the pre-action to use (if there is one) //
+      ////////////////////////////////////////////////////////////////////////////////
+      Optional<List<QRecord>> oldRecordList = fetchOldRecords(updateInput, updateInterface);
 
-      // todo pre-customization - just get to modify the request?
-      UpdateOutput updateOutput = qModule.getUpdateInterface().execute(updateInput);
-      // todo post-customization - can do whatever w/ the result if you want
+      performValidations(updateInput, oldRecordList, false);
 
-      List<String> errors = updateOutput.getRecords().stream().flatMap(r -> r.getErrors().stream()).toList();
+      ////////////////////////////////////
+      // have the backend do the update //
+      ////////////////////////////////////
+      UpdateOutput updateOutput = updateInterface.execute(updateInput);
+
+      //////////////////////////////
+      // log if there were errors //
+      //////////////////////////////
+      List<String> errors = updateOutput.getRecords().stream().flatMap(r -> r.getErrors().stream().map(Object::toString)).toList();
       if(CollectionUtils.nullSafeHasContents(errors))
       {
-         LOG.warn("Errors in updateAction", logPair("tableName", updateInput.getTableName()), logPair("errorCount", errors.size()), errors.size() < 10 ? logPair("errors", errors) : logPair("first10Errors", errors.subList(0, 10)));
+         LOG.info("Errors in updateAction", logPair("tableName", updateInput.getTableName()), logPair("errorCount", errors.size()), errors.size() < 10 ? logPair("errors", errors) : logPair("first10Errors", errors.subList(0, 10)));
       }
 
+      /////////////////////////////////////////////////////////////////////////////////////
+      // update (inserting and deleting as needed) any associations in the input records //
+      /////////////////////////////////////////////////////////////////////////////////////
       manageAssociations(updateInput);
 
+      //////////////////
+      // do the audit //
+      //////////////////
       if(updateInput.getOmitDmlAudit())
       {
          LOG.debug("Requested to omit DML audit");
       }
       else
       {
-         new DMLAuditAction().execute(new DMLAuditInput().withTableActionInput(updateInput).withRecordList(updateOutput.getRecords()).withOldRecordList(oldRecordList));
+         DMLAuditInput dmlAuditInput = new DMLAuditInput()
+            .withTableActionInput(updateInput)
+            .withRecordList(updateOutput.getRecords())
+            .withAuditContext(updateInput.getAuditContext());
+         oldRecordList.ifPresent(l -> dmlAuditInput.setOldRecordList(l));
+         new DMLAuditAction().execute(dmlAuditInput);
+      }
+
+      //////////////////////////////////////////////////////////////
+      // finally, run the post-update customizer, if there is one //
+      //////////////////////////////////////////////////////////////
+      Optional<AbstractPostUpdateCustomizer> postUpdateCustomizer = QCodeLoader.getTableCustomizer(AbstractPostUpdateCustomizer.class, table, TableCustomizers.POST_UPDATE_RECORD.getRole());
+      if(postUpdateCustomizer.isPresent())
+      {
+         try
+         {
+            postUpdateCustomizer.get().setUpdateInput(updateInput);
+            oldRecordList.ifPresent(l -> postUpdateCustomizer.get().setOldRecordList(l));
+            updateOutput.setRecords(postUpdateCustomizer.get().apply(updateOutput.getRecords()));
+         }
+         catch(Exception e)
+         {
+            for(QRecord record : updateOutput.getRecords())
+            {
+               record.addWarning(new QWarningMessage("An error occurred after the update: " + e.getMessage()));
+            }
+         }
       }
 
       return updateOutput;
@@ -123,6 +170,71 @@ public class UpdateAction
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public void performValidations(UpdateInput updateInput, Optional<List<QRecord>> oldRecordList, boolean isPreview) throws QException
+   {
+      QTableMetaData table = updateInput.getTable();
+
+      /////////////////////////////
+      // run standard validators //
+      /////////////////////////////
+      ValueBehaviorApplier.applyFieldBehaviors(updateInput.getInstance(), table, updateInput.getRecords());
+      validatePrimaryKeysAreGiven(updateInput);
+
+      if(oldRecordList.isPresent())
+      {
+         validateRecordsExistAndCanBeAccessed(updateInput, oldRecordList.get());
+      }
+
+      if(updateInput.getInputSource().shouldValidateRequiredFields())
+      {
+         validateRequiredFields(updateInput);
+      }
+
+      ValidateRecordSecurityLockHelper.validateSecurityFields(table, updateInput.getRecords(), ValidateRecordSecurityLockHelper.Action.UPDATE);
+
+      ///////////////////////////////////////////////////////////////////////////
+      // after all validations, run the pre-update customizer, if there is one //
+      ///////////////////////////////////////////////////////////////////////////
+      Optional<AbstractPreUpdateCustomizer> preUpdateCustomizer = QCodeLoader.getTableCustomizer(AbstractPreUpdateCustomizer.class, table, TableCustomizers.PRE_UPDATE_RECORD.getRole());
+      if(preUpdateCustomizer.isPresent())
+      {
+         preUpdateCustomizer.get().setUpdateInput(updateInput);
+         preUpdateCustomizer.get().setIsPreview(isPreview);
+         oldRecordList.ifPresent(l -> preUpdateCustomizer.get().setOldRecordList(l));
+         updateInput.setRecords(preUpdateCustomizer.get().apply(updateInput.getRecords()));
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private Optional<List<QRecord>> fetchOldRecords(UpdateInput updateInput, UpdateInterface updateInterface) throws QException
+   {
+      if(updateInterface.supportsPreFetchQuery())
+      {
+         String             primaryKeyField   = updateInput.getTable().getPrimaryKeyField();
+         List<Serializable> pkeysBeingUpdated = CollectionUtils.nonNullList(updateInput.getRecords()).stream().map(r -> r.getValue(primaryKeyField)).toList();
+
+         QueryInput queryInput = new QueryInput();
+         queryInput.setTransaction(updateInput.getTransaction());
+         queryInput.setTableName(updateInput.getTableName());
+         queryInput.setFilter(new QQueryFilter(new QFilterCriteria(primaryKeyField, QCriteriaOperator.IN, pkeysBeingUpdated)));
+         // todo - need a limit?  what if too many??
+         QueryOutput queryOutput = new QueryAction().execute(queryInput);
+
+         return (Optional.of(queryOutput.getRecords()));
+      }
+
+      return (Optional.empty());
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/
@@ -136,7 +248,7 @@ public class UpdateAction
          ////////////////////////////////////////////////////////////////////////////////////////////////////////////////
          if(record.getValue(table.getPrimaryKeyField()) == null)
          {
-            record.addError("Missing value in primary key field");
+            record.addError(new BadInputStatusMessage("Missing value in primary key field"));
          }
       }
    }
@@ -196,7 +308,7 @@ public class UpdateAction
 
             if(!lookedUpRecords.containsKey(value))
             {
-               record.addError(NOT_FOUND_ERROR_PREFIX + " for " + primaryKeyField.getLabel() + " = " + value);
+               record.addError(new NotFoundStatusMessage("No record was found to update for " + primaryKeyField.getLabel() + " = " + value));
             }
          }
       }
@@ -225,9 +337,9 @@ public class UpdateAction
                /////////////////////////////////////////////////////////////////////////////////////////////
                if(record.getValues().containsKey(requiredField.getName()))
                {
-                  if(record.getValue(requiredField.getName()) == null || (requiredField.getType().isStringLike() && record.getValueString(requiredField.getName()).trim().equals("")))
+                  if(record.getValue(requiredField.getName()) == null || record.getValueString(requiredField.getName()).trim().equals(""))
                   {
-                     record.addError("Missing value in required field: " + requiredField.getLabel());
+                     record.addError(new BadInputStatusMessage("Missing value in required field: " + requiredField.getLabel()));
                   }
                }
             }
@@ -289,7 +401,8 @@ public class UpdateAction
                         //////////////////////////////////////////////////////////////////////////////////////////////////////////
                         for(JoinOn joinOn : join.getJoinOns())
                         {
-                           associatedRecord.setValue(joinOn.getRightField(), record.getValue(joinOn.getLeftField()));
+                           QFieldType type = table.getField(joinOn.getLeftField()).getType();
+                           associatedRecord.setValue(joinOn.getRightField(), ValueUtils.getValueAsFieldType(type, record.getValue(joinOn.getLeftField())));
                         }
                         nextLevelInserts.add(associatedRecord);
                      }
@@ -300,6 +413,16 @@ public class UpdateAction
                         ///////////////////////////////////////////////////////////////////////////////
                         idsBeingUpdated.add(associatedId);
                         nextLevelUpdates.add(associatedRecord);
+
+                        /////////////////////////////////////////////////////////////////////////////////////////////////
+                        // make sure the child record being updated has its join fields populated (same as an insert). //
+                        // this will make the next update action much happier                                          //
+                        /////////////////////////////////////////////////////////////////////////////////////////////////
+                        for(JoinOn joinOn : join.getJoinOns())
+                        {
+                           QFieldType type = table.getField(joinOn.getLeftField()).getType();
+                           associatedRecord.setValue(joinOn.getRightField(), ValueUtils.getValueAsFieldType(type, record.getValue(joinOn.getLeftField())));
+                        }
                      }
                   }
 
@@ -356,45 +479,6 @@ public class UpdateAction
 
 
 
-   /*******************************************************************************
-    **
-    *******************************************************************************/
-   private static List<QRecord> getOldRecordListForAuditIfNeeded(UpdateInput updateInput)
-   {
-      if(updateInput.getOmitDmlAudit())
-      {
-         return (null);
-      }
-
-      try
-      {
-         AuditLevel    auditLevel    = DMLAuditAction.getAuditLevel(updateInput);
-         List<QRecord> oldRecordList = null;
-         if(AuditLevel.FIELD.equals(auditLevel))
-         {
-            String             primaryKeyField   = updateInput.getTable().getPrimaryKeyField();
-            List<Serializable> pkeysBeingUpdated = CollectionUtils.nonNullList(updateInput.getRecords()).stream().map(r -> r.getValue(primaryKeyField)).toList();
-
-            QueryInput queryInput = new QueryInput();
-            queryInput.setTransaction(updateInput.getTransaction());
-            queryInput.setTableName(updateInput.getTableName());
-            queryInput.setFilter(new QQueryFilter(new QFilterCriteria(primaryKeyField, QCriteriaOperator.IN, pkeysBeingUpdated)));
-            // todo - need a limit?  what if too many??
-            QueryOutput queryOutput = new QueryAction().execute(queryInput);
-
-            oldRecordList = queryOutput.getRecords();
-         }
-         return oldRecordList;
-      }
-      catch(Exception e)
-      {
-         LOG.warn("Error getting old record list for audit", e, logPair("table", updateInput.getTableName()));
-         return (null);
-      }
-   }
-
-
-
    /*******************************************************************************
     ** If the table being updated uses an automation-status field, populate it now.
     *******************************************************************************/

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/helpers/ValidateRecordSecurityLockHelper.java

@@ -45,9 +45,11 @@ import com.kingsrook.qqq.backend.core.model.metadata.joins.QJoinMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.security.QSecurityKeyType;
 import com.kingsrook.qqq.backend.core.model.metadata.security.RecordSecurityLock;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+import com.kingsrook.qqq.backend.core.model.statusmessages.PermissionDeniedMessage;
 import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
 import com.kingsrook.qqq.backend.core.utils.ListingHash;
 import com.kingsrook.qqq.backend.core.utils.StringUtils;
+import com.kingsrook.qqq.backend.core.utils.ValueUtils;
 
 
 /*******************************************************************************
@@ -113,9 +115,10 @@ public class ValidateRecordSecurityLockHelper
             ////////////////////////////////////////////////////////////////////////////////////////////////////////////////
             // else look for the joined record - if it isn't found, assume a fail - else validate security value if found //
             ////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-            QJoinMetaData  leftMostJoin      = QContext.getQInstance().getJoin(recordSecurityLock.getJoinNameChain().get(0));
-            QJoinMetaData  rightMostJoin     = QContext.getQInstance().getJoin(recordSecurityLock.getJoinNameChain().get(recordSecurityLock.getJoinNameChain().size() - 1));
-            QTableMetaData leftMostJoinTable = QContext.getQInstance().getTable(leftMostJoin.getLeftTable());
+            QJoinMetaData  leftMostJoin       = QContext.getQInstance().getJoin(recordSecurityLock.getJoinNameChain().get(0));
+            QJoinMetaData  rightMostJoin      = QContext.getQInstance().getJoin(recordSecurityLock.getJoinNameChain().get(recordSecurityLock.getJoinNameChain().size() - 1));
+            QTableMetaData rightMostJoinTable = QContext.getQInstance().getTable(rightMostJoin.getRightTable());
+            QTableMetaData leftMostJoinTable  = QContext.getQInstance().getTable(leftMostJoin.getLeftTable());
 
             for(List<QRecord> inputRecordPage : CollectionUtils.getPages(records, 500))
             {
@@ -153,7 +156,8 @@ public class ValidateRecordSecurityLockHelper
 
                   for(JoinOn joinOn : rightMostJoin.getJoinOns())
                   {
-                     Serializable inputRecordValue = inputRecord.getValue(joinOn.getRightField());
+                     QFieldType   type             = rightMostJoinTable.getField(joinOn.getRightField()).getType();
+                     Serializable inputRecordValue = ValueUtils.getValueAsFieldType(type, inputRecord.getValue(joinOn.getRightField()));
                      inputRecordJoinValues.add(inputRecordValue);
 
                      subFilter.addCriteria(inputRecordValue == null
@@ -225,7 +229,7 @@ public class ValidateRecordSecurityLockHelper
                      {
                         if(RecordSecurityLock.NullValueBehavior.DENY.equals(recordSecurityLock.getNullValueBehavior()))
                         {
-                           inputRecord.addError("You do not have permission to " + action.name().toLowerCase() + " this record - the referenced " + leftMostJoinTable.getLabel() + " was not found.");
+                           inputRecord.addError(new PermissionDeniedMessage("You do not have permission to " + action.name().toLowerCase() + " this record - the referenced " + leftMostJoinTable.getLabel() + " was not found."));
                         }
                      }
                   }
@@ -261,7 +265,7 @@ public class ValidateRecordSecurityLockHelper
          QSecurityKeyType securityKeyType = QContext.getQInstance().getSecurityKeyType(recordSecurityLock.getSecurityKeyType());
          if(StringUtils.hasContent(securityKeyType.getAllAccessKeyName()) && QContext.getQSession().hasSecurityKeyValue(securityKeyType.getAllAccessKeyName(), true, QFieldType.BOOLEAN))
          {
-            LOG.debug("Session has " + securityKeyType.getAllAccessKeyName() + " - not checking this lock.");
+            LOG.trace("Session has " + securityKeyType.getAllAccessKeyName() + " - not checking this lock.");
          }
          else
          {
@@ -287,7 +291,7 @@ public class ValidateRecordSecurityLockHelper
          if(RecordSecurityLock.NullValueBehavior.DENY.equals(recordSecurityLock.getNullValueBehavior()))
          {
             String lockLabel = CollectionUtils.nullSafeHasContents(recordSecurityLock.getJoinNameChain()) ? recordSecurityLock.getSecurityKeyType() : table.getField(recordSecurityLock.getFieldName()).getLabel();
-            record.addError("You do not have permission to " + action.name().toLowerCase() + " a record without a value in the field: " + lockLabel);
+            record.addError(new PermissionDeniedMessage("You do not have permission to " + action.name().toLowerCase() + " a record without a value in the field: " + lockLabel));
          }
       }
       else
@@ -299,12 +303,12 @@ public class ValidateRecordSecurityLockHelper
                ///////////////////////////////////////////////////////////////////////////////////////////////
                // avoid telling the user a value from a foreign record that they didn't pass in themselves. //
                ///////////////////////////////////////////////////////////////////////////////////////////////
-               record.addError("You do not have permission to " + action.name().toLowerCase() + " this record.");
+               record.addError(new PermissionDeniedMessage("You do not have permission to " + action.name().toLowerCase() + " this record."));
             }
             else
             {
                QFieldMetaData field = table.getField(recordSecurityLock.getFieldName());
-               record.addError("You do not have permission to " + action.name().toLowerCase() + " a record with a value of " + recordSecurityValue + " in the field: " + field.getLabel());
+               record.addError(new PermissionDeniedMessage("You do not have permission to " + action.name().toLowerCase() + " a record with a value of " + recordSecurityValue + " in the field: " + field.getLabel()));
             }
          }
       }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/templates/RenderTemplateAction.java

@@ -26,20 +26,30 @@ import java.io.StringWriter;
 import java.util.Map;
 import com.kingsrook.qqq.backend.core.actions.AbstractQActionFunction;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.logging.QLogger;
 import com.kingsrook.qqq.backend.core.model.actions.AbstractActionInput;
 import com.kingsrook.qqq.backend.core.model.templates.RenderTemplateInput;
 import com.kingsrook.qqq.backend.core.model.templates.RenderTemplateOutput;
 import com.kingsrook.qqq.backend.core.model.templates.TemplateType;
+import com.kingsrook.qqq.backend.core.utils.StringUtils;
 import org.apache.velocity.VelocityContext;
 import org.apache.velocity.app.Velocity;
+import org.apache.velocity.app.event.EventCartridge;
+import org.apache.velocity.app.event.MethodExceptionEventHandler;
 import org.apache.velocity.context.Context;
+import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
 
 
 /*******************************************************************************
  ** Basic action to render a template!
+ **
+ ** hard-coded built to only assume Velocity right now.  could expand (and refactor) in future.
  *******************************************************************************/
 public class RenderTemplateAction extends AbstractQActionFunction<RenderTemplateInput, RenderTemplateOutput>
 {
+   private static final QLogger LOG = QLogger.getLogger(RenderTemplateAction.class);
+
+
 
    /*******************************************************************************
     **
@@ -52,9 +62,12 @@ public class RenderTemplateAction extends AbstractQActionFunction<RenderTemplate
       if(TemplateType.VELOCITY.equals(input.getTemplateType()))
       {
          Velocity.init();
-         Context      context      = new VelocityContext(input.getContext());
+         Context context = new VelocityContext(input.getContext());
+
+         setupEventHandlers(context);
+
          StringWriter stringWriter = new StringWriter();
-         Velocity.evaluate(context, stringWriter, "logTag", input.getCode());
+         Velocity.evaluate(context, stringWriter, StringUtils.hasContent(input.getTemplateIdentifier()) ? input.getTemplateIdentifier() : "anonymous", input.getCode());
          output.setResult(stringWriter.getBuffer().toString());
       }
       else
@@ -67,12 +80,28 @@ public class RenderTemplateAction extends AbstractQActionFunction<RenderTemplate
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private static void setupEventHandlers(Context context)
+   {
+      EventCartridge eventCartridge = new EventCartridge();
+      eventCartridge.addEventHandler((MethodExceptionEventHandler) (ctx, aClass, method, exception, info) ->
+      {
+         LOG.info("Exception in velocity template", exception, logPair("at", info.toString()));
+         return (null);
+      });
+      eventCartridge.attachToContext(context);
+   }
+
+
+
    /*******************************************************************************
     ** Most convenient static wrapper to render a Velocity template.
     *******************************************************************************/
    public static String renderVelocity(AbstractActionInput parentActionInput, Map<String, Object> context, String code) throws QException
    {
-      return (render(parentActionInput, TemplateType.VELOCITY, context, code));
+      return (render(TemplateType.VELOCITY, context, code));
    }
 
 
@@ -80,7 +109,7 @@ public class RenderTemplateAction extends AbstractQActionFunction<RenderTemplate
    /*******************************************************************************
     ** Convenient static wrapper to render a template of an arbitrary type (language).
     *******************************************************************************/
-   public static String render(AbstractActionInput parentActionInput, TemplateType templateType, Map<String, Object> context, String code) throws QException
+   public static String render(TemplateType templateType, Map<String, Object> context, String code) throws QException
    {
       RenderTemplateInput renderTemplateInput = new RenderTemplateInput();
       renderTemplateInput.setCode(code);

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/QPossibleValueTranslator.java

@@ -123,7 +123,7 @@ public class QPossibleValueTranslator
          return;
       }
 
-      LOG.debug("Translating possible values in [" + records.size() + "] records from the [" + table.getName() + "] table.");
+      LOG.trace("Translating possible values in [" + records.size() + "] records from the [" + table.getName() + "] table.");
       primePvsCache(table, records, queryJoins, limitedToFieldNames);
 
       for(QRecord record : records)
@@ -378,11 +378,11 @@ public class QPossibleValueTranslator
          for(String valueField : valueFields)
          {
             Object value = switch(valueField)
-               {
-                  case "id" -> id;
-                  case "label" -> label;
-                  default -> throw new IllegalArgumentException("Unexpected value field: " + valueField);
-               };
+            {
+               case "id" -> id;
+               case "label" -> label;
+               default -> throw new IllegalArgumentException("Unexpected value field: " + valueField);
+            };
             values.add(Objects.requireNonNullElse(value, ""));
          }
       }
@@ -427,7 +427,7 @@ public class QPossibleValueTranslator
          int size = entry.getValue().size();
          if(size > 50_000)
          {
-            LOG.debug("Found a big PVS cache - clearing it.", logPair("name", entry.getKey()), logPair("size", size));
+            LOG.info("Found a big PVS cache - clearing it.", logPair("name", entry.getKey()), logPair("size", size));
          }
       }
 
@@ -483,7 +483,7 @@ public class QPossibleValueTranslator
          {
             if(limitedToFieldNames != null && !limitedToFieldNames.contains(fieldNamePrefix + field.getName()))
             {
-               LOG.debug("Skipping cache priming for translation of possible value field [" + fieldNamePrefix + field.getName() + "] - it's not in the limitedToFieldNames set.");
+               LOG.trace("Skipping cache priming for translation of possible value field [" + fieldNamePrefix + field.getName() + "] - it's not in the limitedToFieldNames set.");
                continue;
             }
 
@@ -556,7 +556,7 @@ public class QPossibleValueTranslator
                queryInput.setFieldsToTranslatePossibleValues(possibleValueFieldsToTranslate);
             }
 
-            LOG.debug("Priming PVS cache for [" + page.size() + "] ids from [" + tableName + "] table.");
+            LOG.trace("Priming PVS cache for [" + page.size() + "] ids from [" + tableName + "] table.");
             QueryOutput queryOutput = new QueryAction().execute(queryInput);
 
             ///////////////////////////////////////////////////////////////////////////////////

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/QValueFormatter.java

@@ -29,14 +29,24 @@ import java.time.LocalTime;
 import java.time.ZonedDateTime;
 import java.time.format.DateTimeFormatter;
 import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Optional;
+import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.logging.QLogger;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.AdornmentType;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.FieldAdornment;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.ExposedJoin;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
 import com.kingsrook.qqq.backend.core.utils.StringUtils;
 import com.kingsrook.qqq.backend.core.utils.ValueUtils;
+import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
 
 
 /*******************************************************************************
@@ -292,9 +302,45 @@ public class QValueFormatter
          return;
       }
 
+      Map<String, QFieldMetaData> fieldMap = new HashMap<>();
+
       for(QRecord record : records)
       {
-         setDisplayValuesInRecord(table.getFields().values(), record);
+         for(String fieldName : record.getValues().keySet())
+         {
+            if(!fieldMap.containsKey(fieldName))
+            {
+               try
+               {
+                  if(fieldName.contains("."))
+                  {
+                     String[] nameParts = fieldName.split("\\.", 2);
+                     for(ExposedJoin exposedJoin : CollectionUtils.nonNullList(table.getExposedJoins()))
+                     {
+                        if(exposedJoin.getJoinTable().equals(nameParts[0]))
+                        {
+                           QTableMetaData joinTable = QContext.getQInstance().getTable(nameParts[0]);
+                           fieldMap.put(fieldName, joinTable.getField(nameParts[1]));
+                        }
+                     }
+                  }
+                  else
+                  {
+                     fieldMap.put(fieldName, table.getField(fieldName));
+                  }
+               }
+               catch(Exception e)
+               {
+                  ///////////////////////////////////////////////////////////
+                  // put an empty field in - so no formatting will be done //
+                  ///////////////////////////////////////////////////////////
+                  LOG.info("Error getting field for setting display value", e, logPair("fieldName", fieldName), logPair("tableName", table.getName()));
+                  fieldMap.put(fieldName, new QFieldMetaData());
+               }
+            }
+         }
+
+         setDisplayValuesInRecord(fieldMap, record);
          record.setRecordLabel(formatRecordLabel(table, record));
       }
    }
@@ -319,6 +365,24 @@ public class QValueFormatter
 
 
 
+   /*******************************************************************************
+    ** For a list of records, set their recordLabels and display values
+    *******************************************************************************/
+   public static void setDisplayValuesInRecords(Map<String, QFieldMetaData> fields, List<QRecord> records)
+   {
+      if(records == null)
+      {
+         return;
+      }
+
+      for(QRecord record : records)
+      {
+         setDisplayValuesInRecord(fields, record);
+      }
+   }
+
+
+
    /*******************************************************************************
     ** For a list of records, set their display values
     *******************************************************************************/
@@ -336,6 +400,26 @@ public class QValueFormatter
 
 
 
+   /*******************************************************************************
+    ** For a list of records, set their display values
+    *******************************************************************************/
+   public static void setDisplayValuesInRecord(Map<String, QFieldMetaData> fields, QRecord record)
+   {
+      for(Map.Entry<String, QFieldMetaData> entry : fields.entrySet())
+      {
+         String         fieldName = entry.getKey();
+         QFieldMetaData field     = entry.getValue();
+
+         if(record.getDisplayValue(fieldName) == null)
+         {
+            String formattedValue = formatValue(field, record.getValue(fieldName));
+            record.setDisplayValue(fieldName, formattedValue);
+         }
+      }
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/
@@ -355,4 +439,127 @@ public class QValueFormatter
       }
    }
 
+
+
+   /*******************************************************************************
+    ** For any BLOB type fields in the list of records, change their value to
+    ** the URL where they can be downloaded, and set their display value to a file name.
+    *******************************************************************************/
+   public static void setBlobValuesToDownloadUrls(QTableMetaData table, List<QRecord> records)
+   {
+      for(QFieldMetaData field : table.getFields().values())
+      {
+         if(field.getType().equals(QFieldType.BLOB))
+         {
+            //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+            // file name comes from:                                                                                            //
+            // if there's a FILE_DOWNLOAD adornment, with a FILE_NAME_FIELD value, then the full filename comes from that field //
+            // - unless it was empty - then we do the "default thing":                                                          //
+            // else - the "default thing" is:                                                                                   //
+            // - tableLabel primaryKey fieldLabel                                                                               //
+            // - and - if the FILE_DOWNLOAD adornment had a DEFAULT_EXTENSION, then it gets added (preceded by a dot)           //
+            //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+            Optional<FieldAdornment>  fileDownloadAdornment = field.getAdornment(AdornmentType.FILE_DOWNLOAD);
+            Map<String, Serializable> adornmentValues       = Collections.emptyMap();
+
+            if(fileDownloadAdornment.isPresent())
+            {
+               adornmentValues = fileDownloadAdornment.get().getValues();
+            }
+
+            String fileNameField    = ValueUtils.getValueAsString(adornmentValues.get(AdornmentType.FileDownloadValues.FILE_NAME_FIELD));
+            String fileNameFormat   = ValueUtils.getValueAsString(adornmentValues.get(AdornmentType.FileDownloadValues.FILE_NAME_FORMAT));
+            String defaultExtension = ValueUtils.getValueAsString(adornmentValues.get(AdornmentType.FileDownloadValues.DEFAULT_EXTENSION));
+
+            for(QRecord record : records)
+            {
+               if(!doesFieldHaveValue(field, record))
+               {
+                  continue;
+               }
+
+               Serializable primaryKey = record.getValue(table.getPrimaryKeyField());
+               String       fileName   = null;
+
+               //////////////////////////////////////////////////
+               // try to make file name from the fileNameField //
+               //////////////////////////////////////////////////
+               if(StringUtils.hasContent(fileNameField))
+               {
+                  fileName = record.getValueString(fileNameField);
+               }
+
+               if(!StringUtils.hasContent(fileName))
+               {
+                  if(StringUtils.hasContent(fileNameFormat))
+                  {
+                     @SuppressWarnings("unchecked") // instance validation should make this safe!
+                     List<String> fileNameFormatFields = (List<String>) adornmentValues.get(AdornmentType.FileDownloadValues.FILE_NAME_FORMAT_FIELDS);
+                     List<String> values = fileNameFormatFields.stream().map(f -> ValueUtils.getValueAsString(record.getValue(f))).toList();
+                     fileName = QValueFormatter.formatStringWithValues(fileNameFormat, values);
+                  }
+               }
+
+               if(!StringUtils.hasContent(fileName))
+               {
+                  //////////////////////////////////
+                  // make default name if missing //
+                  //////////////////////////////////
+                  fileName = table.getLabel() + " " + primaryKey + " " + field.getLabel();
+
+                  if(StringUtils.hasContent(defaultExtension))
+                  {
+                     //////////////////////////////////////////
+                     // add default extension if we have one //
+                     //////////////////////////////////////////
+                     fileName += "." + defaultExtension;
+                  }
+               }
+
+               record.setValue(field.getName(), "/data/" + table.getName() + "/" + primaryKey + "/" + field.getName() + "/" + fileName);
+               record.setDisplayValue(field.getName(), fileName);
+            }
+         }
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private static boolean doesFieldHaveValue(QFieldMetaData field, QRecord record)
+   {
+      boolean fieldHasValue = false;
+
+      try
+      {
+         if(record.getValue(field.getName()) != null)
+         {
+            fieldHasValue = true;
+         }
+         else if(field.getIsHeavy())
+         {
+            /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+            // heavy fields that weren't fetched - they should have a backend-detail specifying their length (or null if null) //
+            /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+            Map<String, Serializable> heavyFieldLengths = (Map<String, Serializable>) record.getBackendDetail(QRecord.BACKEND_DETAILS_TYPE_HEAVY_FIELD_LENGTHS);
+            if(heavyFieldLengths != null)
+            {
+               Integer fieldLength = ValueUtils.getValueAsInteger(heavyFieldLengths.get(field.getName()));
+               if(fieldLength != null && fieldLength > 0)
+               {
+                  fieldHasValue = true;
+               }
+            }
+         }
+      }
+      catch(Exception e)
+      {
+         LOG.info("Error checking if field has value", e, logPair("fieldName", field.getName()), logPair("record", record));
+      }
+
+      return fieldHasValue;
+   }
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/SearchPossibleValueSourceAction.java

@@ -247,7 +247,7 @@ public class SearchPossibleValueSourceAction
       queryFilter.setOrderBys(possibleValueSource.getOrderByFields());
 
       // todo - skip & limit as params
-      queryInput.setLimit(250);
+      queryFilter.setLimit(250);
 
       ///////////////////////////////////////////////////////////////////////////////////////////////////////
       // if given a default filter, make it the 'top level' filter and the one we just created a subfilter //

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/ValueBehaviorApplier.java

@@ -29,6 +29,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.ValueTooLongBehavior;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+import com.kingsrook.qqq.backend.core.model.statusmessages.BadInputStatusMessage;
 import com.kingsrook.qqq.backend.core.utils.StringUtils;
 
 
@@ -76,7 +77,7 @@ public class ValueBehaviorApplier
                {
                   case TRUNCATE -> record.setValue(fieldName, StringUtils.safeTruncate(value, field.getMaxLength()));
                   case TRUNCATE_ELLIPSIS -> record.setValue(fieldName, StringUtils.safeTruncate(value, field.getMaxLength(), "..."));
-                  case ERROR -> record.addError("The value for " + field.getLabel() + " is too long (max allowed length=" + field.getMaxLength() + ")");
+                  case ERROR -> record.addError(new BadInputStatusMessage("The value for " + field.getLabel() + " is too long (max allowed length=" + field.getMaxLength() + ")"));
                   case PASS_THROUGH ->
                   {
                   }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/adapters/CsvToQRecordAdapter.java

@@ -31,6 +31,7 @@ import java.util.List;
 import java.util.Map;
 import java.util.function.Consumer;
 import com.kingsrook.qqq.backend.core.actions.reporting.RecordPipe;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.model.actions.shared.mapping.AbstractQFieldMapping;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
@@ -61,7 +62,7 @@ public class CsvToQRecordAdapter
     ** using a given mapping.
     **
     *******************************************************************************/
-   public void buildRecordsFromCsv(RecordPipe recordPipe, String csv, QTableMetaData table, AbstractQFieldMapping<?> mapping, Consumer<QRecord> recordCustomizer)
+   public void buildRecordsFromCsv(RecordPipe recordPipe, String csv, QTableMetaData table, AbstractQFieldMapping<?> mapping, Consumer<QRecord> recordCustomizer) throws QException
    {
       buildRecordsFromCsv(new InputWrapper().withRecordPipe(recordPipe).withCsv(csv).withTable(table).withMapping(mapping).withRecordCustomizer(recordCustomizer));
    }
@@ -73,7 +74,7 @@ public class CsvToQRecordAdapter
     ** using a given mapping.
     **
     *******************************************************************************/
-   public List<QRecord> buildRecordsFromCsv(String csv, QTableMetaData table, AbstractQFieldMapping<?> mapping)
+   public List<QRecord> buildRecordsFromCsv(String csv, QTableMetaData table, AbstractQFieldMapping<?> mapping) throws QException
    {
       buildRecordsFromCsv(new InputWrapper().withCsv(csv).withTable(table).withMapping(mapping));
       return (recordList);
@@ -87,7 +88,7 @@ public class CsvToQRecordAdapter
     **
     ** todo - meta-data validation, type handling
     *******************************************************************************/
-   public void buildRecordsFromCsv(InputWrapper inputWrapper)
+   public void buildRecordsFromCsv(InputWrapper inputWrapper) throws QException
    {
       String                   csv              = inputWrapper.getCsv();
       AbstractQFieldMapping<?> mapping          = inputWrapper.getMapping();
@@ -297,7 +298,7 @@ public class CsvToQRecordAdapter
    /*******************************************************************************
     ** Add a record - either to the pipe, or list, whichever we're building.
     *******************************************************************************/
-   private void addRecord(QRecord record)
+   private void addRecord(QRecord record) throws QException
    {
       if(recordPipe != null)
       {

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/exceptions/QBadRequestException.java

@@ -0,0 +1,53 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.exceptions;
+
+
+/*******************************************************************************
+ ** User-facing exception for when user provided bad or missing data in their request
+ **
+ *******************************************************************************/
+public class QBadRequestException extends QUserFacingException
+{
+
+
+   /*******************************************************************************
+    ** Constructor of message
+    **
+    *******************************************************************************/
+   public QBadRequestException(String message)
+   {
+      super(message);
+   }
+
+
+
+   /*******************************************************************************
+    ** Constructor of message & cause
+    **
+    *******************************************************************************/
+   public QBadRequestException(String message, Throwable cause)
+   {
+      super(message, cause);
+   }
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java

@@ -32,12 +32,13 @@ import java.util.Locale;
 import java.util.Map;
 import java.util.Set;
 import java.util.stream.Collectors;
+import com.kingsrook.qqq.backend.core.actions.metadata.JoinGraph;
 import com.kingsrook.qqq.backend.core.actions.permissions.BulkTableActionProcessPermissionChecker;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.logging.QLogger;
 import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
 import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
-import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeUsage;
 import com.kingsrook.qqq.backend.core.model.metadata.dashboard.QWidgetMetaDataInterface;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.AdornmentType;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.FieldAdornment;
@@ -59,18 +60,19 @@ import com.kingsrook.qqq.backend.core.model.metadata.processes.QStepMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportDataSource;
 import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportView;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.ExposedJoin;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QFieldSection;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QMiddlewareTableMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.Tier;
+import com.kingsrook.qqq.backend.core.processes.implementations.bulk.delete.BulkDeleteLoadStep;
 import com.kingsrook.qqq.backend.core.processes.implementations.bulk.delete.BulkDeleteTransformStep;
+import com.kingsrook.qqq.backend.core.processes.implementations.bulk.edit.BulkEditLoadStep;
 import com.kingsrook.qqq.backend.core.processes.implementations.bulk.edit.BulkEditTransformStep;
 import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkInsertExtractStep;
+import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkInsertLoadStep;
 import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkInsertTransformStep;
 import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.ExtractViaQueryStep;
-import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.LoadViaDeleteStep;
-import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.LoadViaInsertStep;
-import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.LoadViaUpdateStep;
 import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess;
 import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
 import com.kingsrook.qqq.backend.core.utils.StringUtils;
@@ -87,6 +89,8 @@ public class QInstanceEnricher
 
    private final QInstance qInstance;
 
+   private JoinGraph joinGraph;
+
    //////////////////////////////////////////////////////////
    // todo - come up w/ a way for app devs to set configs! //
    //////////////////////////////////////////////////////////
@@ -144,6 +148,81 @@ public class QInstanceEnricher
       {
          qInstance.getWidgets().values().forEach(this::enrichWidget);
       }
+
+      enrichJoins();
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private void enrichJoins()
+   {
+      try
+      {
+         joinGraph = new JoinGraph(qInstance);
+
+         for(QTableMetaData table : CollectionUtils.nonNullMap(qInstance.getTables()).values())
+         {
+            Set<JoinGraph.JoinConnectionList> joinConnections = joinGraph.getJoinConnections(table.getName());
+            for(ExposedJoin exposedJoin : CollectionUtils.nonNullList(table.getExposedJoins()))
+            {
+               /////////////////////////////////////////////////////////////////////////////////////////////////////
+               // proceed with caution - remember, validator will fail the instance if things are missing/invalid //
+               /////////////////////////////////////////////////////////////////////////////////////////////////////
+               if(exposedJoin.getJoinTable() != null)
+               {
+                  QTableMetaData joinTable = qInstance.getTable(exposedJoin.getJoinTable());
+                  if(joinTable != null)
+                  {
+                     //////////////////////////////////////////////////////////////////////////////////
+                     // default the exposed join's label to the join table's label, if it wasn't set //
+                     //////////////////////////////////////////////////////////////////////////////////
+                     if(!StringUtils.hasContent(exposedJoin.getLabel()))
+                     {
+                        exposedJoin.setLabel(joinTable.getLabel());
+                     }
+
+                     ///////////////////////////////////////////////////////////////////////////////
+                     // default the exposed join's join-path from the joinGraph, if it wasn't set //
+                     ///////////////////////////////////////////////////////////////////////////////
+                     if(CollectionUtils.nullSafeIsEmpty(exposedJoin.getJoinPath()))
+                     {
+                        List<JoinGraph.JoinConnectionList> eligibleJoinConnections = new ArrayList<>();
+                        for(JoinGraph.JoinConnectionList joinConnection : joinConnections)
+                        {
+                           if(joinTable.getName().equals(joinConnection.list().get(joinConnection.list().size() - 1).joinTable()))
+                           {
+                              eligibleJoinConnections.add(joinConnection);
+                           }
+                        }
+
+                        if(eligibleJoinConnections.isEmpty())
+                        {
+                           throw (new QException("Could not infer a joinPath for table [" + table.getName() + "], exposedJoin to [" + exposedJoin.getJoinTable() + "]:  No join connections between these tables exist in this instance."));
+                        }
+                        else if(eligibleJoinConnections.size() > 1)
+                        {
+                           throw (new QException("Could not infer a joinPath for table [" + table.getName() + "], exposedJoin to [" + exposedJoin.getJoinTable() + "]:  "
+                              + eligibleJoinConnections.size() + " join connections exist between these tables.  You need to specify one:\n"
+                              + StringUtils.join("\n", eligibleJoinConnections.stream().map(jcl -> jcl.getJoinNamesAsString()).toList()) + "."
+                           ));
+                        }
+                        else
+                        {
+                           exposedJoin.setJoinPath(eligibleJoinConnections.get(0).getJoinNamesAsList());
+                        }
+                     }
+                  }
+               }
+            }
+         }
+      }
+      catch(Exception e)
+      {
+         throw (new RuntimeException("Error enriching instance joins", e));
+      }
    }
 
 
@@ -618,7 +697,7 @@ public class QInstanceEnricher
       QProcessMetaData process = StreamedETLWithFrontendProcess.defineProcessMetaData(
             BulkInsertExtractStep.class,
             BulkInsertTransformStep.class,
-            LoadViaInsertStep.class,
+            BulkInsertLoadStep.class,
             values
          )
          .withName(processName)
@@ -626,7 +705,7 @@ public class QInstanceEnricher
          .withTableName(table.getName())
          .withIsHidden(true)
          .withPermissionRules(qInstance.getDefaultPermissionRules().clone()
-            .withCustomPermissionChecker(new QCodeReference(BulkTableActionProcessPermissionChecker.class, QCodeUsage.CUSTOMIZER)));
+            .withCustomPermissionChecker(new QCodeReference(BulkTableActionProcessPermissionChecker.class)));
 
       List<QFieldMetaData> editableFields = new ArrayList<>();
       for(QFieldSection section : CollectionUtils.nonNullList(table.getSections()))
@@ -636,7 +715,7 @@ public class QInstanceEnricher
             try
             {
                QFieldMetaData field = table.getField(fieldName);
-               if(field.getIsEditable())
+               if(field.getIsEditable() && !field.getType().equals(QFieldType.BLOB))
                {
                   editableFields.add(field);
                }
@@ -655,7 +734,7 @@ public class QInstanceEnricher
       QFrontendStepMetaData uploadScreen = new QFrontendStepMetaData()
          .withName("upload")
          .withLabel("Upload File")
-         .withFormField(new QFieldMetaData("theFile", QFieldType.BLOB).withIsRequired(true))
+         .withFormField(new QFieldMetaData("theFile", QFieldType.BLOB).withLabel(table.getLabel() + " File").withIsRequired(true))
          .withComponent(new QFrontendComponentMetaData()
             .withType(QComponentType.HELP_TEXT)
             .withValue("previewText", "file upload instructions")
@@ -682,7 +761,7 @@ public class QInstanceEnricher
       QProcessMetaData process = StreamedETLWithFrontendProcess.defineProcessMetaData(
             ExtractViaQueryStep.class,
             BulkEditTransformStep.class,
-            LoadViaUpdateStep.class,
+            BulkEditLoadStep.class,
             values
          )
          .withName(processName)
@@ -690,10 +769,11 @@ public class QInstanceEnricher
          .withTableName(table.getName())
          .withIsHidden(true)
          .withPermissionRules(qInstance.getDefaultPermissionRules().clone()
-            .withCustomPermissionChecker(new QCodeReference(BulkTableActionProcessPermissionChecker.class, QCodeUsage.CUSTOMIZER)));
+            .withCustomPermissionChecker(new QCodeReference(BulkTableActionProcessPermissionChecker.class)));
 
       List<QFieldMetaData> editableFields = table.getFields().values().stream()
          .filter(QFieldMetaData::getIsEditable)
+         .filter(f -> !f.getType().equals(QFieldType.BLOB))
          .toList();
 
       QFrontendStepMetaData editScreen = new QFrontendStepMetaData()
@@ -729,7 +809,7 @@ public class QInstanceEnricher
       QProcessMetaData process = StreamedETLWithFrontendProcess.defineProcessMetaData(
             ExtractViaQueryStep.class,
             BulkDeleteTransformStep.class,
-            LoadViaDeleteStep.class,
+            BulkDeleteLoadStep.class,
             values
          )
          .withName(processName)
@@ -737,7 +817,7 @@ public class QInstanceEnricher
          .withTableName(table.getName())
          .withIsHidden(true)
          .withPermissionRules(qInstance.getDefaultPermissionRules().clone()
-            .withCustomPermissionChecker(new QCodeReference(BulkTableActionProcessPermissionChecker.class, QCodeUsage.CUSTOMIZER)));
+            .withCustomPermissionChecker(new QCodeReference(BulkTableActionProcessPermissionChecker.class)));
 
       List<QFieldMetaData> tableFields = table.getFields().values().stream().toList();
       process.getFrontendStep("review").setRecordListFields(tableFields);
@@ -955,7 +1035,8 @@ public class QInstanceEnricher
       {
          for(String fieldName : table.getFields().keySet())
          {
-            if(!usedFieldNames.contains(fieldName))
+            QFieldMetaData field = table.getField(fieldName);
+            if(!field.getIsHidden() && !usedFieldNames.contains(fieldName))
             {
                otherSection.getFieldNames().add(fieldName);
                usedFieldNames.add(fieldName);
@@ -1015,4 +1096,13 @@ public class QInstanceEnricher
       }
    }
 
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public JoinGraph getJoinGraph()
+   {
+      return (this.joinGraph);
+   }
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidator.java

@@ -22,6 +22,7 @@
 package com.kingsrook.qqq.backend.core.instances;
 
 
+import java.io.Serializable;
 import java.lang.reflect.InvocationTargetException;
 import java.lang.reflect.Modifier;
 import java.util.ArrayList;
@@ -31,11 +32,11 @@ import java.util.List;
 import java.util.Map;
 import java.util.Objects;
 import java.util.Set;
-import java.util.function.Consumer;
 import java.util.function.Supplier;
 import java.util.stream.Stream;
 import com.kingsrook.qqq.backend.core.actions.automation.RecordAutomationHandler;
 import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizers;
+import com.kingsrook.qqq.backend.core.actions.metadata.JoinGraph;
 import com.kingsrook.qqq.backend.core.actions.processes.BackendStep;
 import com.kingsrook.qqq.backend.core.actions.scripts.TestScriptActionInterface;
 import com.kingsrook.qqq.backend.core.actions.values.QCustomPossibleValueProvider;
@@ -50,7 +51,8 @@ import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
 import com.kingsrook.qqq.backend.core.model.metadata.QMiddlewareInstanceMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
 import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeType;
-import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeUsage;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.AdornmentType;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.FieldAdornment;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.ValueTooLongBehavior;
 import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinOn;
@@ -65,10 +67,12 @@ import com.kingsrook.qqq.backend.core.model.metadata.queues.SQSQueueProviderMeta
 import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportDataSource;
 import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportField;
 import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportView;
+import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.QScheduleMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.security.FieldSecurityLock;
 import com.kingsrook.qqq.backend.core.model.metadata.security.RecordSecurityLock;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.AssociatedScript;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.Association;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.ExposedJoin;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QFieldSection;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.Tier;
@@ -80,6 +84,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.tables.cache.CacheOf;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.cache.CacheUseCase;
 import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
 import com.kingsrook.qqq.backend.core.utils.StringUtils;
+import com.kingsrook.qqq.backend.core.utils.ValueUtils;
 
 
 /*******************************************************************************
@@ -115,17 +120,26 @@ public class QInstanceValidator
          return;
       }
 
+      /////////////////////////////////////////////////////////////////////////////////////////////////////
+      // the enricher will build a join graph (if there are any joins).  we'd like to only do that       //
+      // once, during the enrichment/validation work, so, capture it, and store it back in the instance. //
+      /////////////////////////////////////////////////////////////////////////////////////////////////////
+      JoinGraph joinGraph = null;
       try
       {
          /////////////////////////////////////////////////////////////////////////////////////////////////
          // before validation, enrich the object (e.g., to fill in values that the user doesn't have to //
          /////////////////////////////////////////////////////////////////////////////////////////////////
          // TODO - possible point of customization (use a different enricher, or none, or pass it options).
-         new QInstanceEnricher(qInstance).enrich();
+         QInstanceEnricher qInstanceEnricher = new QInstanceEnricher(qInstance);
+         qInstanceEnricher.enrich();
+         joinGraph = qInstanceEnricher.getJoinGraph();
       }
       catch(Exception e)
       {
+         System.out.println();
          LOG.error("Error enriching instance prior to validation", e);
+         System.out.println();
          throw (new QInstanceValidationException("Error enriching qInstance prior to validation.", e));
       }
 
@@ -136,7 +150,7 @@ public class QInstanceValidator
       {
          validateBackends(qInstance);
          validateAutomationProviders(qInstance);
-         validateTables(qInstance);
+         validateTables(qInstance, joinGraph);
          validateProcesses(qInstance);
          validateReports(qInstance);
          validateApps(qInstance);
@@ -158,7 +172,9 @@ public class QInstanceValidator
          throw (new QInstanceValidationException(errors));
       }
 
-      qInstance.setHasBeenValidated(new QInstanceValidationKey());
+      QInstanceValidationKey validationKey = new QInstanceValidationKey();
+      qInstance.setHasBeenValidated(validationKey);
+      qInstance.setJoinGraph(validationKey, joinGraph);
    }
 
 
@@ -366,7 +382,7 @@ public class QInstanceValidator
    /*******************************************************************************
     **
     *******************************************************************************/
-   private void validateTables(QInstance qInstance)
+   private void validateTables(QInstance qInstance, JoinGraph joinGraph)
    {
       if(assertCondition(CollectionUtils.nullSafeHasContents(qInstance.getTables()), "At least 1 table must be defined."))
       {
@@ -405,7 +421,7 @@ public class QInstanceValidator
             {
                table.getFields().forEach((fieldName, field) ->
                {
-                  validateTableField(qInstance, tableName, fieldName, field);
+                  validateTableField(qInstance, tableName, fieldName, table, field);
                });
             }
 
@@ -437,7 +453,14 @@ public class QInstanceValidator
 
             for(String fieldName : CollectionUtils.nonNullMap(table.getFields()).keySet())
             {
-               assertCondition(fieldNamesInSections.contains(fieldName), "Table " + tableName + " field " + fieldName + " is not listed in any field sections.");
+               if(table.getField(fieldName).getIsHidden())
+               {
+                  assertCondition(!fieldNamesInSections.contains(fieldName), "Table " + tableName + " field " + fieldName + " is listed in a field section, but it is marked as hidden.");
+               }
+               else
+               {
+                  assertCondition(fieldNamesInSections.contains(fieldName), "Table " + tableName + " field " + fieldName + " is not listed in any field sections.");
+               }
             }
 
             if(table.getRecordLabelFields() != null && table.getFields() != null)
@@ -459,12 +482,61 @@ public class QInstanceValidator
             validateTableCacheOf(qInstance, table);
             validateTableRecordSecurityLocks(qInstance, table);
             validateTableAssociations(qInstance, table);
+            validateExposedJoins(qInstance, joinGraph, table);
          });
       }
    }
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private void validateExposedJoins(QInstance qInstance, JoinGraph joinGraph, QTableMetaData table)
+   {
+      Set<JoinGraph.JoinConnectionList> joinConnectionsForTable = null;
+      Set<String>                       usedLabels              = new HashSet<>();
+      Set<List<String>>                 usedJoinPaths           = new HashSet<>();
+
+      String tablePrefix = "Table " + table.getName() + " ";
+      for(ExposedJoin exposedJoin : CollectionUtils.nonNullList(table.getExposedJoins()))
+      {
+         String joinPrefix = tablePrefix + "exposedJoin [missingJoinTableName] ";
+         if(assertCondition(StringUtils.hasContent(exposedJoin.getJoinTable()), tablePrefix + "has an exposedJoin that is missing a joinTable name."))
+         {
+            joinPrefix = tablePrefix + "exposedJoin " + exposedJoin.getJoinTable() + " ";
+            if(assertCondition(qInstance.getTable(exposedJoin.getJoinTable()) != null, joinPrefix + "is referencing an unrecognized table"))
+            {
+               if(assertCondition(CollectionUtils.nullSafeHasContents(exposedJoin.getJoinPath()), joinPrefix + "is missing a joinPath."))
+               {
+                  joinConnectionsForTable = Objects.requireNonNullElseGet(joinConnectionsForTable, () -> joinGraph.getJoinConnections(table.getName()));
+
+                  boolean foundJoinConnection = false;
+                  for(JoinGraph.JoinConnectionList joinConnectionList : joinConnectionsForTable)
+                  {
+                     if(joinConnectionList.matchesJoinPath(exposedJoin.getJoinPath()))
+                     {
+                        foundJoinConnection = true;
+                     }
+                  }
+                  assertCondition(foundJoinConnection, joinPrefix + "specified a joinPath [" + exposedJoin.getJoinPath() + "] which does not match a valid join connection in the instance.");
+
+                  assertCondition(!usedJoinPaths.contains(exposedJoin.getJoinPath()), tablePrefix + "has more than one join with the joinPath: " + exposedJoin.getJoinPath());
+                  usedJoinPaths.add(exposedJoin.getJoinPath());
+               }
+            }
+         }
+
+         if(assertCondition(StringUtils.hasContent(exposedJoin.getLabel()), joinPrefix + "is missing a label."))
+         {
+            assertCondition(!usedLabels.contains(exposedJoin.getLabel()), tablePrefix + "has more than one join labeled: " + exposedJoin.getLabel());
+            usedLabels.add(exposedJoin.getLabel());
+         }
+      }
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/
@@ -590,7 +662,7 @@ public class QInstanceValidator
    /*******************************************************************************
     **
     *******************************************************************************/
-   private void validateTableField(QInstance qInstance, String tableName, String fieldName, QFieldMetaData field)
+   private void validateTableField(QInstance qInstance, String tableName, String fieldName, QTableMetaData table, QFieldMetaData field)
    {
       assertCondition(Objects.equals(fieldName, field.getName()),
          "Inconsistent naming in table " + tableName + " for field " + fieldName + "/" + field.getName() + ".");
@@ -632,6 +704,55 @@ public class QInstanceValidator
          assertCondition(fieldSecurityLock.getDefaultBehavior() != null, prefix + "has a fieldSecurityLock that is missing a defaultBehavior");
          assertCondition(CollectionUtils.nullSafeHasContents(fieldSecurityLock.getOverrideValues()), prefix + "has a fieldSecurityLock that is missing overrideValues");
       }
+
+      for(FieldAdornment adornment : CollectionUtils.nonNullList(field.getAdornments()))
+      {
+         Map<String, Serializable> adornmentValues = CollectionUtils.nonNullMap(adornment.getValues());
+         if(assertCondition(adornment.getType() != null, prefix + "has an adornment that is missing a type"))
+         {
+            String adornmentPrefix = prefix.trim() + ", " + adornment.getType() + " adornment ";
+            switch(adornment.getType())
+            {
+               case SIZE ->
+               {
+                  String width = ValueUtils.getValueAsString(adornmentValues.get("width"));
+                  if(assertCondition(StringUtils.hasContent(width), adornmentPrefix + "is missing a width value"))
+                  {
+                     assertNoException(() -> AdornmentType.Size.valueOf(width.toUpperCase()), adornmentPrefix + "has an unrecognized width value [" + width + "]");
+                  }
+               }
+               case FILE_DOWNLOAD ->
+               {
+                  String fileNameField = ValueUtils.getValueAsString(adornmentValues.get(AdornmentType.FileDownloadValues.FILE_NAME_FIELD));
+                  if(StringUtils.hasContent(fileNameField)) // file name isn't required - but if given, must be a field on the table.
+                  {
+                     assertNoException(() -> table.getField(fileNameField), adornmentPrefix + "specifies an unrecognized fileNameField [" + fileNameField + "]");
+                  }
+
+                  if(adornmentValues.containsKey(AdornmentType.FileDownloadValues.FILE_NAME_FORMAT_FIELDS))
+                  {
+                     try
+                     {
+                        @SuppressWarnings("unchecked")
+                        List<String> formatFieldNames = (List<String>) adornmentValues.get(AdornmentType.FileDownloadValues.FILE_NAME_FORMAT_FIELDS);
+                        for(String formatFieldName : CollectionUtils.nonNullList(formatFieldNames))
+                        {
+                           assertNoException(() -> table.getField(formatFieldName), adornmentPrefix + "specifies an unrecognized field name in fileNameFormatFields [" + formatFieldName + "]");
+                        }
+                     }
+                     catch(Exception e)
+                     {
+                        errors.add(adornmentPrefix + "fileNameFormatFields could not be accessed (is it a List<String>?)");
+                     }
+                  }
+               }
+               default ->
+               {
+                  // no validations by default
+               }
+            }
+         }
+      }
    }
 
 
@@ -840,9 +961,9 @@ public class QInstanceValidator
    /*******************************************************************************
     **
     *******************************************************************************/
-   private void validateTableCustomizer(String tableName, String customizerName, QCodeReference codeReference)
+   private void validateTableCustomizer(String tableName, String roleName, QCodeReference codeReference)
    {
-      String prefix = "Table " + tableName + ", customizer " + customizerName + ": ";
+      String prefix = "Table " + tableName + ", customizer " + roleName + ": ";
 
       if(!preAssertionsForCodeReference(codeReference, prefix))
       {
@@ -865,7 +986,7 @@ public class QInstanceValidator
             //////////////////////////////////////////////////
             Object customizerInstance = getInstanceOfCodeReference(prefix, customizerClass);
 
-            TableCustomizers tableCustomizer = TableCustomizers.forRole(customizerName);
+            TableCustomizers tableCustomizer = TableCustomizers.forRole(roleName);
             if(tableCustomizer == null)
             {
                ////////////////////////////////////////////////////////////////////////////////////////////////////
@@ -878,29 +999,9 @@ public class QInstanceValidator
                ////////////////////////////////////////////////////////////////////////
                // make sure the customizer instance can be cast to the expected type //
                ////////////////////////////////////////////////////////////////////////
-               if(customizerInstance != null && tableCustomizer.getTableCustomizer().getExpectedType() != null)
+               if(customizerInstance != null && tableCustomizer.getExpectedType() != null)
                {
-                  Object castedObject = getCastedObject(prefix, tableCustomizer.getTableCustomizer().getExpectedType(), customizerInstance);
-
-                  Consumer<Object> validationFunction = tableCustomizer.getTableCustomizer().getValidationFunction();
-                  if(castedObject != null && validationFunction != null)
-                  {
-                     try
-                     {
-                        validationFunction.accept(castedObject);
-                     }
-                     catch(ClassCastException e)
-                     {
-                        errors.add(prefix + "Error validating customizer type parameters: " + e.getMessage());
-                     }
-                     catch(Exception e)
-                     {
-                        ///////////////////////////////////////////////////////////////////////////////////////////////////////////
-                        // mmm, calling customizers w/ random data is expected to often throw, so, this check is iffy at best... //
-                        // if we run into more trouble here, we might consider disabling the whole "validation function" check.  //
-                        ///////////////////////////////////////////////////////////////////////////////////////////////////////////
-                     }
-                  }
+                  assertObjectCanBeCasted(prefix, tableCustomizer.getExpectedType(), customizerInstance);
                }
             }
          }
@@ -910,18 +1011,18 @@ public class QInstanceValidator
 
 
    /*******************************************************************************
-    **
+    ** Make sure that a given object can be casted to an expected type.
     *******************************************************************************/
-   private <T> T getCastedObject(String prefix, Class<T> expectedType, Object customizerInstance)
+   private <T> T assertObjectCanBeCasted(String errorPrefix, Class<T> expectedType, Object object)
    {
       T castedObject = null;
       try
       {
-         castedObject = expectedType.cast(customizerInstance);
+         castedObject = expectedType.cast(object);
       }
       catch(ClassCastException e)
       {
-         errors.add(prefix + "CodeReference is not of the expected type: " + expectedType);
+         errors.add(errorPrefix + "CodeReference is not of the expected type: " + expectedType);
       }
       return castedObject;
    }
@@ -1108,6 +1209,23 @@ public class QInstanceValidator
                   }
                }
             }
+
+            ///////////////////////////////////////////////////////////////////////////////
+            // if the process has a schedule, make sure required schedule data populated //
+            ///////////////////////////////////////////////////////////////////////////////
+            if(process.getSchedule() != null)
+            {
+               QScheduleMetaData schedule = process.getSchedule();
+               assertCondition(schedule.getRepeatMillis() != null || schedule.getRepeatSeconds() != null, "Either repeat millis or repeat seconds must be set on schedule in process " + processName);
+
+               if(schedule.getBackendVariant() != null)
+               {
+                  assertCondition(schedule.getVariantRunStrategy() != null, "A variant strategy was not set for " + schedule.getBackendVariant() + " on schedule in process " + processName);
+                  assertCondition(schedule.getVariantTableName() != null, "A variant table name was not set for " + schedule.getBackendVariant() + " on schedule in process " + processName);
+                  assertCondition(schedule.getVariantFieldName() != null, "A variant field name was not set for " + schedule.getBackendVariant() + " on schedule in process " + processName);
+               }
+            }
+
          });
       }
    }
@@ -1435,7 +1553,6 @@ public class QInstanceValidator
 
                      if(assertCondition(possibleValueSource.getCustomCodeReference() != null, "custom-type possibleValueSource " + pvsName + " is missing a customCodeReference."))
                      {
-                        assertCondition(QCodeUsage.POSSIBLE_VALUE_PROVIDER.equals(possibleValueSource.getCustomCodeReference().getCodeUsage()), "customCodeReference for possibleValueSource " + pvsName + " is not a possibleValueProvider.");
                         validateSimpleCodeReference("PossibleValueSource " + pvsName + " custom code reference: ", possibleValueSource.getCustomCodeReference(), QCustomPossibleValueProvider.class);
                      }
                   }
@@ -1479,7 +1596,7 @@ public class QInstanceValidator
             ////////////////////////////////////////////////////////////////////////
             if(classInstance != null)
             {
-               getCastedObject(prefix, expectedClass, classInstance);
+               assertObjectCanBeCasted(prefix, expectedClass, classInstance);
             }
          }
       }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/logging/LogUtils.java

@@ -118,7 +118,7 @@ public class LogUtils
    {
       try
       {
-         String        packagesToKeep = "com.kingsrook|com.nutrifresh"; // todo - parameterize!!
+         String        packagesToKeep = "com.kingsrook|com.coldtrack"; // todo - parameterize!!
          StringBuilder rs             = new StringBuilder();
          String[]      lines          = stackTrace.split("\n");
 

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/logging/QLogger.java

@@ -137,6 +137,16 @@ public class QLogger
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public void log(Level level, String message, Throwable t, LogPair... logPairs)
+   {
+      logger.log(level, makeJsonString(message, t, logPairs));
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/audits/DMLAuditInput.java

@@ -38,6 +38,8 @@ public class DMLAuditInput extends AbstractActionInput implements Serializable
    private List<QRecord>            oldRecordList;
    private AbstractTableActionInput tableActionInput;
 
+   private String auditContext = null;
+
 
 
    /*******************************************************************************
@@ -131,4 +133,35 @@ public class DMLAuditInput extends AbstractActionInput implements Serializable
       return (this);
    }
 
+
+
+   /*******************************************************************************
+    ** Getter for auditContext
+    *******************************************************************************/
+   public String getAuditContext()
+   {
+      return (this.auditContext);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for auditContext
+    *******************************************************************************/
+   public void setAuditContext(String auditContext)
+   {
+      this.auditContext = auditContext;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for auditContext
+    *******************************************************************************/
+   public DMLAuditInput withAuditContext(String auditContext)
+   {
+      this.auditContext = auditContext;
+      return (this);
+   }
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/RunBackendStepOutput.java

@@ -29,6 +29,7 @@ import java.util.List;
 import java.util.Map;
 import com.kingsrook.qqq.backend.core.model.actions.AbstractActionOutput;
 import com.kingsrook.qqq.backend.core.model.actions.audits.AuditInput;
+import com.kingsrook.qqq.backend.core.model.actions.audits.AuditSingleInput;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
 import com.kingsrook.qqq.backend.core.utils.ValueUtils;
 
@@ -290,4 +291,25 @@ public class RunBackendStepOutput extends AbstractActionOutput implements Serial
       return (this);
    }
 
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public void addAuditSingleInput(AuditSingleInput auditSingleInput)
+   {
+      if(getAuditInputList() == null)
+      {
+         setAuditInputList(new ArrayList<>());
+      }
+
+      if(getAuditInputList().isEmpty())
+      {
+         getAuditInputList().add(new AuditInput());
+      }
+
+      AuditInput auditInput = getAuditInputList().get(0);
+      auditInput.addAuditSingleInput(auditSingleInput);
+   }
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/scripts/AbstractRunScriptInput.java

@@ -0,0 +1,198 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2023.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.model.actions.scripts;
+
+
+import java.io.Serializable;
+import java.util.Map;
+import com.kingsrook.qqq.backend.core.actions.scripts.logging.QCodeExecutionLoggerInterface;
+import com.kingsrook.qqq.backend.core.model.actions.AbstractTableActionInput;
+import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
+
+
+/*******************************************************************************
+ ** Base class for input wrappers that end up running scripts (ExecuteCodeAction)
+ *******************************************************************************/
+public class AbstractRunScriptInput<C extends QCodeReference> extends AbstractTableActionInput
+{
+   private C                             codeReference;
+   private Map<String, Serializable>     inputValues;
+   private QCodeExecutionLoggerInterface logger;
+   private Serializable                  outputObject;
+   private Serializable                  scriptUtils;
+
+
+
+   /*******************************************************************************
+    ** Getter for codeReference
+    *******************************************************************************/
+   public C getCodeReference()
+   {
+      return (this.codeReference);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for codeReference
+    *******************************************************************************/
+   public void setCodeReference(C codeReference)
+   {
+      this.codeReference = codeReference;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for codeReference
+    *******************************************************************************/
+   public AbstractRunScriptInput<C> withCodeReference(C codeReference)
+   {
+      this.codeReference = codeReference;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for inputValues
+    *******************************************************************************/
+   public Map<String, Serializable> getInputValues()
+   {
+      return (this.inputValues);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for inputValues
+    *******************************************************************************/
+   public void setInputValues(Map<String, Serializable> inputValues)
+   {
+      this.inputValues = inputValues;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for inputValues
+    *******************************************************************************/
+   public AbstractRunScriptInput<C> withInputValues(Map<String, Serializable> inputValues)
+   {
+      this.inputValues = inputValues;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for logger
+    *******************************************************************************/
+   public QCodeExecutionLoggerInterface getLogger()
+   {
+      return (this.logger);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for logger
+    *******************************************************************************/
+   public void setLogger(QCodeExecutionLoggerInterface logger)
+   {
+      this.logger = logger;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for logger
+    *******************************************************************************/
+   public AbstractRunScriptInput<C> withLogger(QCodeExecutionLoggerInterface logger)
+   {
+      this.logger = logger;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for outputObject
+    *******************************************************************************/
+   public Serializable getOutputObject()
+   {
+      return (this.outputObject);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for outputObject
+    *******************************************************************************/
+   public void setOutputObject(Serializable outputObject)
+   {
+      this.outputObject = outputObject;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for outputObject
+    *******************************************************************************/
+   public AbstractRunScriptInput<C> withOutputObject(Serializable outputObject)
+   {
+      this.outputObject = outputObject;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for scriptUtils
+    *******************************************************************************/
+   public Serializable getScriptUtils()
+   {
+      return (this.scriptUtils);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for scriptUtils
+    *******************************************************************************/
+   public void setScriptUtils(Serializable scriptUtils)
+   {
+      this.scriptUtils = scriptUtils;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for scriptUtils
+    *******************************************************************************/
+   public AbstractRunScriptInput<C> withScriptUtils(Serializable scriptUtils)
+   {
+      this.scriptUtils = scriptUtils;
+      return (this);
+   }
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/scripts/RunAdHocRecordScriptInput.java

@@ -24,9 +24,6 @@ package com.kingsrook.qqq.backend.core.model.actions.scripts;
 
 import java.io.Serializable;
 import java.util.List;
-import java.util.Map;
-import com.kingsrook.qqq.backend.core.actions.scripts.logging.QCodeExecutionLoggerInterface;
-import com.kingsrook.qqq.backend.core.model.actions.AbstractTableActionInput;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
 import com.kingsrook.qqq.backend.core.model.metadata.code.AdHocScriptCodeReference;
 
@@ -34,18 +31,10 @@ import com.kingsrook.qqq.backend.core.model.metadata.code.AdHocScriptCodeReferen
 /*******************************************************************************
  **
  *******************************************************************************/
-public class RunAdHocRecordScriptInput extends AbstractTableActionInput
+public class RunAdHocRecordScriptInput extends AbstractRunScriptInput<AdHocScriptCodeReference>
 {
-   private AdHocScriptCodeReference      codeReference;
-   private Map<String, Serializable>     inputValues;
-   private List<Serializable>            recordPrimaryKeyList; // can either supply recordList, or recordPrimaryKeyList
-   private List<QRecord>                 recordList;
-   private String                        tableName;
-   private QCodeExecutionLoggerInterface logger;
-
-   private Serializable outputObject;
-
-   private Serializable scriptUtils;
+   private List<Serializable> recordPrimaryKeyList; // can either supply recordList, or recordPrimaryKeyList
+   private List<QRecord>      recordList;
 
 
 
@@ -58,189 +47,6 @@ public class RunAdHocRecordScriptInput extends AbstractTableActionInput
 
 
 
-   /*******************************************************************************
-    ** Getter for inputValues
-    **
-    *******************************************************************************/
-   public Map<String, Serializable> getInputValues()
-   {
-      return inputValues;
-   }
-
-
-
-   /*******************************************************************************
-    ** Setter for inputValues
-    **
-    *******************************************************************************/
-   public void setInputValues(Map<String, Serializable> inputValues)
-   {
-      this.inputValues = inputValues;
-   }
-
-
-
-   /*******************************************************************************
-    ** Fluent setter for inputValues
-    **
-    *******************************************************************************/
-   public RunAdHocRecordScriptInput withInputValues(Map<String, Serializable> inputValues)
-   {
-      this.inputValues = inputValues;
-      return (this);
-   }
-
-
-
-   /*******************************************************************************
-    ** Getter for outputObject
-    **
-    *******************************************************************************/
-   public Serializable getOutputObject()
-   {
-      return outputObject;
-   }
-
-
-
-   /*******************************************************************************
-    ** Setter for outputObject
-    **
-    *******************************************************************************/
-   public void setOutputObject(Serializable outputObject)
-   {
-      this.outputObject = outputObject;
-   }
-
-
-
-   /*******************************************************************************
-    ** Fluent setter for outputObject
-    **
-    *******************************************************************************/
-   public RunAdHocRecordScriptInput withOutputObject(Serializable outputObject)
-   {
-      this.outputObject = outputObject;
-      return (this);
-   }
-
-
-
-   /*******************************************************************************
-    ** Getter for logger
-    *******************************************************************************/
-   public QCodeExecutionLoggerInterface getLogger()
-   {
-      return (this.logger);
-   }
-
-
-
-   /*******************************************************************************
-    ** Setter for logger
-    *******************************************************************************/
-   public void setLogger(QCodeExecutionLoggerInterface logger)
-   {
-      this.logger = logger;
-   }
-
-
-
-   /*******************************************************************************
-    ** Fluent setter for logger
-    *******************************************************************************/
-   public RunAdHocRecordScriptInput withLogger(QCodeExecutionLoggerInterface logger)
-   {
-      this.logger = logger;
-      return (this);
-   }
-
-
-
-   /*******************************************************************************
-    ** Getter for scriptUtils
-    **
-    *******************************************************************************/
-   public Serializable getScriptUtils()
-   {
-      return scriptUtils;
-   }
-
-
-
-   /*******************************************************************************
-    ** Setter for scriptUtils
-    **
-    *******************************************************************************/
-   public void setScriptUtils(Serializable scriptUtils)
-   {
-      this.scriptUtils = scriptUtils;
-   }
-
-
-
-   /*******************************************************************************
-    ** Getter for codeReference
-    *******************************************************************************/
-   public AdHocScriptCodeReference getCodeReference()
-   {
-      return (this.codeReference);
-   }
-
-
-
-   /*******************************************************************************
-    ** Setter for codeReference
-    *******************************************************************************/
-   public void setCodeReference(AdHocScriptCodeReference codeReference)
-   {
-      this.codeReference = codeReference;
-   }
-
-
-
-   /*******************************************************************************
-    ** Fluent setter for codeReference
-    *******************************************************************************/
-   public RunAdHocRecordScriptInput withCodeReference(AdHocScriptCodeReference codeReference)
-   {
-      this.codeReference = codeReference;
-      return (this);
-   }
-
-
-
-   /*******************************************************************************
-    ** Getter for tableName
-    *******************************************************************************/
-   public String getTableName()
-   {
-      return (this.tableName);
-   }
-
-
-
-   /*******************************************************************************
-    ** Setter for tableName
-    *******************************************************************************/
-   public void setTableName(String tableName)
-   {
-      this.tableName = tableName;
-   }
-
-
-
-   /*******************************************************************************
-    ** Fluent setter for tableName
-    *******************************************************************************/
-   public RunAdHocRecordScriptInput withTableName(String tableName)
-   {
-      this.tableName = tableName;
-      return (this);
-   }
-
-
-
    /*******************************************************************************
     ** Getter for recordList
     *******************************************************************************/

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/scripts/RunAssociatedScriptInput.java

@@ -22,187 +22,46 @@
 package com.kingsrook.qqq.backend.core.model.actions.scripts;
 
 
-import java.io.Serializable;
-import java.util.Map;
-import com.kingsrook.qqq.backend.core.actions.scripts.logging.QCodeExecutionLoggerInterface;
-import com.kingsrook.qqq.backend.core.model.actions.AbstractTableActionInput;
+import com.kingsrook.qqq.backend.core.actions.scripts.AssociatedScriptContextPrimerInterface;
 import com.kingsrook.qqq.backend.core.model.metadata.code.AssociatedScriptCodeReference;
 
 
 /*******************************************************************************
  **
  *******************************************************************************/
-public class RunAssociatedScriptInput extends AbstractTableActionInput
+public class RunAssociatedScriptInput extends AbstractRunScriptInput<AssociatedScriptCodeReference>
 {
-   private AssociatedScriptCodeReference codeReference;
-   private Map<String, Serializable>     inputValues;
-   private QCodeExecutionLoggerInterface logger;
-
-   private Serializable outputObject;
-
-   private Serializable scriptUtils;
+   private AssociatedScriptContextPrimerInterface associatedScriptContextPrimerInterface;
 
 
 
    /*******************************************************************************
-    **
+    ** Getter for associatedScriptContextPrimerInterface
     *******************************************************************************/
-   public RunAssociatedScriptInput()
+   public AssociatedScriptContextPrimerInterface getAssociatedScriptContextPrimerInterface()
    {
+      return (this.associatedScriptContextPrimerInterface);
    }
 
 
 
    /*******************************************************************************
-    ** Getter for codeReference
-    **
+    ** Setter for associatedScriptContextPrimerInterface
     *******************************************************************************/
-   public AssociatedScriptCodeReference getCodeReference()
+   public void setAssociatedScriptContextPrimerInterface(AssociatedScriptContextPrimerInterface associatedScriptContextPrimerInterface)
    {
-      return codeReference;
+      this.associatedScriptContextPrimerInterface = associatedScriptContextPrimerInterface;
    }
 
 
 
    /*******************************************************************************
-    ** Setter for codeReference
-    **
+    ** Fluent setter for associatedScriptContextPrimerInterface
     *******************************************************************************/
-   public void setCodeReference(AssociatedScriptCodeReference codeReference)
+   public RunAssociatedScriptInput withAssociatedScriptContextPrimerInterface(AssociatedScriptContextPrimerInterface associatedScriptContextPrimerInterface)
    {
-      this.codeReference = codeReference;
-   }
-
-
-
-   /*******************************************************************************
-    ** Fluent setter for codeReference
-    **
-    *******************************************************************************/
-   public RunAssociatedScriptInput withCodeReference(AssociatedScriptCodeReference codeReference)
-   {
-      this.codeReference = codeReference;
+      this.associatedScriptContextPrimerInterface = associatedScriptContextPrimerInterface;
       return (this);
    }
 
-
-
-   /*******************************************************************************
-    ** Getter for inputValues
-    **
-    *******************************************************************************/
-   public Map<String, Serializable> getInputValues()
-   {
-      return inputValues;
-   }
-
-
-
-   /*******************************************************************************
-    ** Setter for inputValues
-    **
-    *******************************************************************************/
-   public void setInputValues(Map<String, Serializable> inputValues)
-   {
-      this.inputValues = inputValues;
-   }
-
-
-
-   /*******************************************************************************
-    ** Fluent setter for inputValues
-    **
-    *******************************************************************************/
-   public RunAssociatedScriptInput withInputValues(Map<String, Serializable> inputValues)
-   {
-      this.inputValues = inputValues;
-      return (this);
-   }
-
-
-
-   /*******************************************************************************
-    ** Getter for outputObject
-    **
-    *******************************************************************************/
-   public Serializable getOutputObject()
-   {
-      return outputObject;
-   }
-
-
-
-   /*******************************************************************************
-    ** Setter for outputObject
-    **
-    *******************************************************************************/
-   public void setOutputObject(Serializable outputObject)
-   {
-      this.outputObject = outputObject;
-   }
-
-
-
-   /*******************************************************************************
-    ** Fluent setter for outputObject
-    **
-    *******************************************************************************/
-   public RunAssociatedScriptInput withOutputObject(Serializable outputObject)
-   {
-      this.outputObject = outputObject;
-      return (this);
-   }
-
-
-
-   /*******************************************************************************
-    ** Getter for logger
-    *******************************************************************************/
-   public QCodeExecutionLoggerInterface getLogger()
-   {
-      return (this.logger);
-   }
-
-
-
-   /*******************************************************************************
-    ** Setter for logger
-    *******************************************************************************/
-   public void setLogger(QCodeExecutionLoggerInterface logger)
-   {
-      this.logger = logger;
-   }
-
-
-
-   /*******************************************************************************
-    ** Fluent setter for logger
-    *******************************************************************************/
-   public RunAssociatedScriptInput withLogger(QCodeExecutionLoggerInterface logger)
-   {
-      this.logger = logger;
-      return (this);
-   }
-
-
-
-   /*******************************************************************************
-    ** Getter for scriptUtils
-    **
-    *******************************************************************************/
-   public Serializable getScriptUtils()
-   {
-      return scriptUtils;
-   }
-
-
-
-   /*******************************************************************************
-    ** Setter for scriptUtils
-    **
-    *******************************************************************************/
-   public void setScriptUtils(Serializable scriptUtils)
-   {
-      this.scriptUtils = scriptUtils;
-   }
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/scripts/StoreAssociatedScriptInput.java

@@ -35,6 +35,8 @@ public class StoreAssociatedScriptInput extends AbstractTableActionInput
    private Serializable recordPrimaryKey;
 
    private String code;
+   private String apiName;
+   private String apiVersion;
    private String commitMessage;
 
 
@@ -183,4 +185,66 @@ public class StoreAssociatedScriptInput extends AbstractTableActionInput
       return (this);
    }
 
+
+
+   /*******************************************************************************
+    ** Getter for apiName
+    *******************************************************************************/
+   public String getApiName()
+   {
+      return (this.apiName);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for apiName
+    *******************************************************************************/
+   public void setApiName(String apiName)
+   {
+      this.apiName = apiName;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for apiName
+    *******************************************************************************/
+   public StoreAssociatedScriptInput withApiName(String apiName)
+   {
+      this.apiName = apiName;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for apiVersion
+    *******************************************************************************/
+   public String getApiVersion()
+   {
+      return (this.apiVersion);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for apiVersion
+    *******************************************************************************/
+   public void setApiVersion(String apiVersion)
+   {
+      this.apiVersion = apiVersion;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for apiVersion
+    *******************************************************************************/
+   public StoreAssociatedScriptInput withApiVersion(String apiVersion)
+   {
+      this.apiVersion = apiVersion;
+      return (this);
+   }
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/scripts/TestScriptInput.java

@@ -36,6 +36,9 @@ public class TestScriptInput extends AbstractTableActionInput
    private Map<String, Serializable> inputValues;
    private QCodeReference            codeReference;
 
+   private String apiName;
+   private String apiVersion;
+
 
 
    /*******************************************************************************
@@ -113,4 +116,66 @@ public class TestScriptInput extends AbstractTableActionInput
       return (this);
    }
 
+
+
+   /*******************************************************************************
+    ** Getter for apiName
+    *******************************************************************************/
+   public String getApiName()
+   {
+      return (this.apiName);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for apiName
+    *******************************************************************************/
+   public void setApiName(String apiName)
+   {
+      this.apiName = apiName;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for apiName
+    *******************************************************************************/
+   public TestScriptInput withApiName(String apiName)
+   {
+      this.apiName = apiName;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for apiVersion
+    *******************************************************************************/
+   public String getApiVersion()
+   {
+      return (this.apiVersion);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for apiVersion
+    *******************************************************************************/
+   public void setApiVersion(String apiVersion)
+   {
+      this.apiVersion = apiVersion;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for apiVersion
+    *******************************************************************************/
+   public TestScriptInput withApiVersion(String apiVersion)
+   {
+      this.apiVersion = apiVersion;
+      return (this);
+   }
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/InputSource.java

@@ -0,0 +1,43 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2023.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.model.actions.tables;
+
+
+/*******************************************************************************
+ ** interface to define input sources - idea being, so QQQ can have its standard
+ ** ones (see QInputSource), but applications can define their own as well.
+ **
+ ** We might imagine things like a user's session dictating what InputSource
+ ** gets passed into all DML actions.  Or perhaps API meta-data, or just a method
+ ** on QInstance in the future?
+ **
+ ** We might imagine, maybe, more methods growing in the future...
+ *******************************************************************************/
+public interface InputSource
+{
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   boolean shouldValidateRequiredFields();
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/QInputSource.java

@@ -0,0 +1,56 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2023.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.model.actions.tables;
+
+
+/*******************************************************************************
+ ** QQQ standard input sources -- the system, or users.
+ *******************************************************************************/
+public enum QInputSource implements InputSource
+{
+   SYSTEM(true),
+   USER(true);
+
+
+   private final boolean shouldValidateRequiredFields;
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   QInputSource(boolean shouldValidateRequiredFields)
+   {
+      this.shouldValidateRequiredFields = shouldValidateRequiredFields;
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public boolean shouldValidateRequiredFields()
+   {
+      return (this.shouldValidateRequiredFields);
+   }
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/count/CountInput.java

@@ -37,7 +37,8 @@ public class CountInput extends AbstractTableActionInput
 {
    private QQueryFilter filter;
 
-   private List<QueryJoin> queryJoins = null;
+   private List<QueryJoin> queryJoins           = null;
+   private Boolean         includeDistinctCount = false;
 
 
 
@@ -120,4 +121,35 @@ public class CountInput extends AbstractTableActionInput
       return (this);
    }
 
+
+
+   /*******************************************************************************
+    ** Getter for includeDistinctCount
+    *******************************************************************************/
+   public Boolean getIncludeDistinctCount()
+   {
+      return (this.includeDistinctCount);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for includeDistinctCount
+    *******************************************************************************/
+   public void setIncludeDistinctCount(Boolean includeDistinctCount)
+   {
+      this.includeDistinctCount = includeDistinctCount;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for includeDistinctCount
+    *******************************************************************************/
+   public CountInput withIncludeDistinctCount(Boolean includeDistinctCount)
+   {
+      this.includeDistinctCount = includeDistinctCount;
+      return (this);
+   }
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/count/CountOutput.java

@@ -32,6 +32,7 @@ import com.kingsrook.qqq.backend.core.model.actions.AbstractActionOutput;
 public class CountOutput extends AbstractActionOutput
 {
    private Integer count;
+   private Integer distinctCount;
 
 
 
@@ -52,4 +53,47 @@ public class CountOutput extends AbstractActionOutput
    {
       this.count = count;
    }
+
+
+
+   /*******************************************************************************
+    ** Getter for distinctCount
+    *******************************************************************************/
+   public Integer getDistinctCount()
+   {
+      return (this.distinctCount);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for distinctCount
+    *******************************************************************************/
+   public void setDistinctCount(Integer distinctCount)
+   {
+      this.distinctCount = distinctCount;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for distinctCount
+    *******************************************************************************/
+   public CountOutput withDistinctCount(Integer distinctCount)
+   {
+      this.distinctCount = distinctCount;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for count
+    *******************************************************************************/
+   public CountOutput withCount(Integer count)
+   {
+      this.count = count;
+      return (this);
+   }
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/delete/DeleteInput.java

@@ -26,6 +26,8 @@ import java.io.Serializable;
 import java.util.List;
 import com.kingsrook.qqq.backend.core.actions.QBackendTransaction;
 import com.kingsrook.qqq.backend.core.model.actions.AbstractTableActionInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.InputSource;
+import com.kingsrook.qqq.backend.core.model.actions.tables.QInputSource;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
 import com.kingsrook.qqq.backend.core.utils.collections.MutableList;
 
@@ -39,6 +41,7 @@ public class DeleteInput extends AbstractTableActionInput
    private QBackendTransaction transaction;
    private List<Serializable>  primaryKeys;
    private QQueryFilter        queryFilter;
+   private InputSource         inputSource = QInputSource.SYSTEM;
 
 
 
@@ -154,4 +157,35 @@ public class DeleteInput extends AbstractTableActionInput
       return this;
    }
 
+
+
+   /*******************************************************************************
+    ** Getter for inputSource
+    *******************************************************************************/
+   public InputSource getInputSource()
+   {
+      return (this.inputSource);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for inputSource
+    *******************************************************************************/
+   public void setInputSource(InputSource inputSource)
+   {
+      this.inputSource = inputSource;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for inputSource
+    *******************************************************************************/
+   public DeleteInput withInputSource(InputSource inputSource)
+   {
+      this.inputSource = inputSource;
+      return (this);
+   }
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/delete/DeleteOutput.java

@@ -37,6 +37,7 @@ public class DeleteOutput extends AbstractActionOutput implements Serializable
 {
    private int           deletedRecordCount = 0;
    private List<QRecord> recordsWithErrors;
+   private List<QRecord> recordsWithWarnings;
 
 
 
@@ -81,6 +82,7 @@ public class DeleteOutput extends AbstractActionOutput implements Serializable
    }
 
 
+
    /*******************************************************************************
     **
     *******************************************************************************/
@@ -94,6 +96,7 @@ public class DeleteOutput extends AbstractActionOutput implements Serializable
    }
 
 
+
    /*******************************************************************************
     **
     *******************************************************************************/
@@ -101,4 +104,50 @@ public class DeleteOutput extends AbstractActionOutput implements Serializable
    {
       deletedRecordCount += i;
    }
+
+
+
+   /*******************************************************************************
+    ** Getter for recordsWithWarnings
+    *******************************************************************************/
+   public List<QRecord> getRecordsWithWarnings()
+   {
+      return (this.recordsWithWarnings);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for recordsWithWarnings
+    *******************************************************************************/
+   public void setRecordsWithWarnings(List<QRecord> recordsWithWarnings)
+   {
+      this.recordsWithWarnings = recordsWithWarnings;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for recordsWithWarnings
+    *******************************************************************************/
+   public DeleteOutput withRecordsWithWarnings(List<QRecord> recordsWithWarnings)
+   {
+      this.recordsWithWarnings = recordsWithWarnings;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public void addRecordWithWarning(QRecord recordWithWarning)
+   {
+      if(this.recordsWithWarnings == null)
+      {
+         this.recordsWithWarnings = new ArrayList<>();
+      }
+      this.recordsWithWarnings.add(recordWithWarning);
+   }
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/get/GetInput.java

@@ -43,6 +43,8 @@ public class GetInput extends AbstractTableActionInput
    private boolean shouldTranslatePossibleValues = false;
    private boolean shouldGenerateDisplayValues   = false;
    private boolean shouldFetchHeavyFields        = true;
+   private boolean shouldOmitHiddenFields        = true;
+   private boolean shouldMaskPasswords           = true;
 
 
    ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
@@ -323,4 +325,66 @@ public class GetInput extends AbstractTableActionInput
       return (this);
    }
 
+
+
+   /*******************************************************************************
+    ** Getter for shouldMaskPasswords
+    *******************************************************************************/
+   public boolean getShouldMaskPasswords()
+   {
+      return (this.shouldMaskPasswords);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for shouldMaskPasswords
+    *******************************************************************************/
+   public void setShouldMaskPasswords(boolean shouldMaskPasswords)
+   {
+      this.shouldMaskPasswords = shouldMaskPasswords;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for shouldMaskPasswords
+    *******************************************************************************/
+   public GetInput withShouldMaskPasswords(boolean shouldMaskPasswords)
+   {
+      this.shouldMaskPasswords = shouldMaskPasswords;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for shouldOmitHiddenFields
+    *******************************************************************************/
+   public boolean getShouldOmitHiddenFields()
+   {
+      return (this.shouldOmitHiddenFields);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for shouldOmitHiddenFields
+    *******************************************************************************/
+   public void setShouldOmitHiddenFields(boolean shouldOmitHiddenFields)
+   {
+      this.shouldOmitHiddenFields = shouldOmitHiddenFields;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for shouldOmitHiddenFields
+    *******************************************************************************/
+   public GetInput withShouldOmitHiddenFields(boolean shouldOmitHiddenFields)
+   {
+      this.shouldOmitHiddenFields = shouldOmitHiddenFields;
+      return (this);
+   }
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/insert/InsertInput.java

@@ -25,6 +25,8 @@ package com.kingsrook.qqq.backend.core.model.actions.tables.insert;
 import java.util.List;
 import com.kingsrook.qqq.backend.core.actions.QBackendTransaction;
 import com.kingsrook.qqq.backend.core.model.actions.AbstractTableActionInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.InputSource;
+import com.kingsrook.qqq.backend.core.model.actions.tables.QInputSource;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
 
 
@@ -36,6 +38,7 @@ public class InsertInput extends AbstractTableActionInput
 {
    private QBackendTransaction transaction;
    private List<QRecord>       records;
+   private InputSource         inputSource = QInputSource.SYSTEM;
 
    private boolean skipUniqueKeyCheck = false;
 
@@ -182,4 +185,35 @@ public class InsertInput extends AbstractTableActionInput
       return (this);
    }
 
+
+
+   /*******************************************************************************
+    ** Getter for inputSource
+    *******************************************************************************/
+   public InputSource getInputSource()
+   {
+      return (this.inputSource);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for inputSource
+    *******************************************************************************/
+   public void setInputSource(InputSource inputSource)
+   {
+      this.inputSource = inputSource;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for inputSource
+    *******************************************************************************/
+   public InsertInput withInputSource(InputSource inputSource)
+   {
+      this.inputSource = inputSource;
+      return (this);
+   }
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/JoinsContext.java

@@ -32,12 +32,16 @@ import java.util.Objects;
 import java.util.Set;
 import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.logging.QLogger;
 import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.joins.QJoinMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.security.RecordSecurityLock;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.ExposedJoin;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
+import com.kingsrook.qqq.backend.core.utils.collections.MutableList;
+import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
 
 
 /*******************************************************************************
@@ -46,6 +50,8 @@ import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
  *******************************************************************************/
 public class JoinsContext
 {
+   private static final QLogger LOG = QLogger.getLogger(JoinsContext.class);
+
    private final QInstance       instance;
    private final String          mainTableName;
    private final List<QueryJoin> queryJoins;
@@ -65,7 +71,7 @@ public class JoinsContext
    {
       this.instance = instance;
       this.mainTableName = tableName;
-      this.queryJoins = CollectionUtils.nonNullList(queryJoins);
+      this.queryJoins = new MutableList<>(queryJoins);
 
       for(QueryJoin queryJoin : this.queryJoins)
       {
@@ -105,13 +111,13 @@ public class JoinsContext
             if(join.getLeftTable().equals(tmpTable.getName()))
             {
                QueryJoin queryJoin = new ImplicitQueryJoinForSecurityLock().withJoinMetaData(join).withType(QueryJoin.Type.INNER);
-               this.queryJoins.add(queryJoin); // todo something else with aliases?  probably.
+               this.addQueryJoin(queryJoin);
                tmpTable = instance.getTable(join.getRightTable());
             }
             else if(join.getRightTable().equals(tmpTable.getName()))
             {
                QueryJoin queryJoin = new ImplicitQueryJoinForSecurityLock().withJoinMetaData(join.flip()).withType(QueryJoin.Type.INNER);
-               this.queryJoins.add(queryJoin); // todo something else with aliases?  probably.
+               this.addQueryJoin(queryJoin); //
                tmpTable = instance.getTable(join.getLeftTable());
             }
             else
@@ -123,6 +129,8 @@ public class JoinsContext
 
       ensureFilterIsRepresented(filter);
 
+      addJoinsFromExposedJoinPaths();
+
       /* todo!!
       for(QueryJoin queryJoin : queryJoins)
       {
@@ -132,7 +140,160 @@ public class JoinsContext
             // addCriteriaForRecordSecurityLock(instance, session, joinTable, securityCriteria, recordSecurityLock, joinsContext, queryJoin.getJoinTableOrItsAlias());
          }
       }
-       */
+      */
+   }
+
+
+
+   /*******************************************************************************
+    ** Add a query join to the list of query joins, and "process it"
+    **
+    ** use this method to add to the list, instead of ever adding directly, as it's
+    ** important do to that process step (and we've had bugs when it wasn't done).
+    *******************************************************************************/
+   private void addQueryJoin(QueryJoin queryJoin) throws QException
+   {
+      this.queryJoins.add(queryJoin);
+      processQueryJoin(queryJoin);
+   }
+
+
+
+   /*******************************************************************************
+    ** If there are any joins in the context that don't have a join meta data, see
+    ** if we can find the JoinMetaData to use for them by looking at the main table's
+    ** exposed joins, and using their join paths.
+    *******************************************************************************/
+   private void addJoinsFromExposedJoinPaths() throws QException
+   {
+      ////////////////////////////////////////////////////////////////////////////////
+      // do a double-loop, to avoid concurrent modification on the queryJoins list. //
+      // that is to say, we'll loop over that list, but possibly add things to it,  //
+      // in which case we'll set this flag, and break the inner loop, to go again.  //
+      ////////////////////////////////////////////////////////////////////////////////
+      boolean addedJoin;
+      do
+      {
+         addedJoin = false;
+         for(QueryJoin queryJoin : queryJoins)
+         {
+            /////////////////////////////////////////////////////////////////////
+            // if the join has joinMetaData, then we don't need to process it. //
+            /////////////////////////////////////////////////////////////////////
+            if(queryJoin.getJoinMetaData() == null)
+            {
+               //////////////////////////////////////////////////////////////////////
+               // try to find a direct join between the main table and this table. //
+               // if one is found, then put it (the meta data) on the query join.  //
+               //////////////////////////////////////////////////////////////////////
+               String        baseTableName = Objects.requireNonNullElse(resolveTableNameOrAliasToTableName(queryJoin.getBaseTableOrAlias()), mainTableName);
+               QJoinMetaData found         = findJoinMetaData(instance, baseTableName, queryJoin.getJoinTable());
+               if(found != null)
+               {
+                  queryJoin.setJoinMetaData(found);
+               }
+               else
+               {
+                  /////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+                  // else, the join must be indirect - so look for an exposedJoin that will have a joinPath that will connect us //
+                  /////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+                  LOG.debug("Looking for an exposed join...", logPair("mainTable", mainTableName), logPair("joinTable", queryJoin.getJoinTable()));
+
+                  QTableMetaData mainTable          = instance.getTable(mainTableName);
+                  boolean        addedAnyQueryJoins = false;
+                  for(ExposedJoin exposedJoin : CollectionUtils.nonNullList(mainTable.getExposedJoins()))
+                  {
+                     if(queryJoin.getJoinTable().equals(exposedJoin.getJoinTable()))
+                     {
+                        LOG.debug("Found an exposed join", logPair("mainTable", mainTableName), logPair("joinTable", queryJoin.getJoinTable()), logPair("joinPath", exposedJoin.getJoinPath()));
+
+                        /////////////////////////////////////////////////////////////////////////////////////
+                        // loop backward through the join path (from the joinTable back to the main table) //
+                        // adding joins to the table (if they aren't already in the query)                 //
+                        /////////////////////////////////////////////////////////////////////////////////////
+                        String tmpTable = queryJoin.getJoinTable();
+                        for(int i = exposedJoin.getJoinPath().size() - 1; i >= 0; i--)
+                        {
+                           String        joinName  = exposedJoin.getJoinPath().get(i);
+                           QJoinMetaData joinToAdd = instance.getJoin(joinName);
+
+                           /////////////////////////////////////////////////////////////////////////////
+                           // get the name from the opposite side of the join (flipping it if needed) //
+                           /////////////////////////////////////////////////////////////////////////////
+                           String nextTable;
+                           if(joinToAdd.getRightTable().equals(tmpTable))
+                           {
+                              nextTable = joinToAdd.getLeftTable();
+                           }
+                           else
+                           {
+                              nextTable = joinToAdd.getRightTable();
+                              joinToAdd = joinToAdd.flip();
+                           }
+
+                           if(doesJoinNeedAddedToQuery(joinName))
+                           {
+                              ///////////////////////////////////////////////////////////////////////////////////////////////////////////////
+                              // if this is the last element in the joinPath, then we want to set this joinMetaData on the outer queryJoin //
+                              // - else, we need to add a new queryJoin to this context                                                    //
+                              ///////////////////////////////////////////////////////////////////////////////////////////////////////////////
+                              if(i == exposedJoin.getJoinPath().size() - 1)
+                              {
+                                 if(queryJoin.getBaseTableOrAlias() == null)
+                                 {
+                                    queryJoin.setBaseTableOrAlias(nextTable);
+                                 }
+                                 queryJoin.setJoinMetaData(joinToAdd);
+                              }
+                              else
+                              {
+                                 QueryJoin queryJoinToAdd = makeQueryJoinFromJoinAndTableNames(nextTable, tmpTable, joinToAdd);
+                                 queryJoinToAdd.setType(queryJoin.getType());
+                                 addedAnyQueryJoins = true;
+                                 this.addQueryJoin(queryJoinToAdd);
+                              }
+                           }
+
+                           tmpTable = nextTable;
+                        }
+                     }
+                  }
+
+                  ///////////////////////////////////////////////////////////////////////////////////////////////////
+                  // break the inner loop (it would fail due to a concurrent modification), but continue the outer //
+                  ///////////////////////////////////////////////////////////////////////////////////////////////////
+                  if(addedAnyQueryJoins)
+                  {
+                     addedJoin = true;
+                     break;
+                  }
+               }
+            }
+         }
+      }
+      while(addedJoin);
+
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private boolean doesJoinNeedAddedToQuery(String joinName)
+   {
+      ///////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      // look at all queryJoins already in context - if any have this join's name, then we don't need this join... //
+      ///////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      for(QueryJoin queryJoin : queryJoins)
+      {
+         if(queryJoin.getJoinMetaData() != null && queryJoin.getJoinMetaData().getName().equals(joinName))
+         {
+            return (false);
+         }
+      }
+
+      return (true);
    }
 
 
@@ -256,34 +417,52 @@ public class JoinsContext
       {
          if(!aliasToTableNameMap.containsKey(filterTable) && !Objects.equals(mainTableName, filterTable))
          {
+            boolean found = false;
             for(QJoinMetaData join : CollectionUtils.nonNullMap(QContext.getQInstance().getJoins()).values())
             {
-               QueryJoin queryJoin = null;
-               if(join.getLeftTable().equals(mainTableName) && join.getRightTable().equals(filterTable))
-               {
-                  queryJoin = new QueryJoin().withJoinMetaData(join).withType(QueryJoin.Type.INNER);
-               }
-               else
-               {
-                  join = join.flip();
-                  if(join.getLeftTable().equals(mainTableName) && join.getRightTable().equals(filterTable))
-                  {
-                     queryJoin = new QueryJoin().withJoinMetaData(join).withType(QueryJoin.Type.INNER);
-                  }
-               }
-
+               QueryJoin queryJoin = makeQueryJoinFromJoinAndTableNames(mainTableName, filterTable, join);
                if(queryJoin != null)
                {
-                  this.queryJoins.add(queryJoin); // todo something else with aliases?  probably.
-                  processQueryJoin(queryJoin);
+                  this.addQueryJoin(queryJoin);
+                  found = true;
+                  break;
                }
             }
+
+            if(!found)
+            {
+               QueryJoin queryJoin = new QueryJoin().withJoinTable(filterTable).withType(QueryJoin.Type.INNER);
+               this.addQueryJoin(queryJoin);
+            }
          }
       }
    }
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private QueryJoin makeQueryJoinFromJoinAndTableNames(String tableA, String tableB, QJoinMetaData join)
+   {
+      QueryJoin queryJoin = null;
+      if(join.getLeftTable().equals(tableA) && join.getRightTable().equals(tableB))
+      {
+         queryJoin = new QueryJoin().withJoinMetaData(join).withType(QueryJoin.Type.INNER);
+      }
+      else
+      {
+         join = join.flip();
+         if(join.getLeftTable().equals(tableA) && join.getRightTable().equals(tableB))
+         {
+            queryJoin = new QueryJoin().withJoinMetaData(join).withType(QueryJoin.Type.INNER);
+         }
+      }
+      return queryJoin;
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QCriteriaOperator.java

@@ -30,6 +30,7 @@ public enum QCriteriaOperator
 {
    EQUALS,
    NOT_EQUALS,
+   NOT_EQUALS_OR_IS_NULL,
    IN,
    NOT_IN,
    IS_NULL_OR_IN,

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QQueryFilter.java

@@ -47,6 +47,13 @@ public class QQueryFilter implements Serializable, Cloneable
    private BooleanOperator    booleanOperator = BooleanOperator.AND;
    private List<QQueryFilter> subFilters      = new ArrayList<>();
 
+   ////////////////////////////////////////////////////////////////////////////////////////////////////////////
+   // skip & limit are meant to only apply to QueryAction (at least at the initial time they are added here) //
+   // e.g., they are ignored in CountAction, AggregateAction, etc, where their meanings may be less obvious  //
+   ////////////////////////////////////////////////////////////////////////////////////////////////////////////
+   private Integer skip;
+   private Integer limit;
+
 
 
    /*******************************************************************************
@@ -398,4 +405,66 @@ public class QQueryFilter implements Serializable, Cloneable
       }
    }
 
+
+
+   /*******************************************************************************
+    ** Getter for skip
+    *******************************************************************************/
+   public Integer getSkip()
+   {
+      return (this.skip);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for skip
+    *******************************************************************************/
+   public void setSkip(Integer skip)
+   {
+      this.skip = skip;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for skip
+    *******************************************************************************/
+   public QQueryFilter withSkip(Integer skip)
+   {
+      this.skip = skip;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for limit
+    *******************************************************************************/
+   public Integer getLimit()
+   {
+      return (this.limit);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for limit
+    *******************************************************************************/
+   public void setLimit(Integer limit)
+   {
+      this.limit = limit;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for limit
+    *******************************************************************************/
+   public QQueryFilter withLimit(Integer limit)
+   {
+      this.limit = limit;
+      return (this);
+   }
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QueryInput.java

@@ -39,14 +39,14 @@ public class QueryInput extends AbstractTableActionInput
 {
    private QBackendTransaction transaction;
    private QQueryFilter        filter;
-   private Integer             skip;
-   private Integer             limit;
 
    private RecordPipe recordPipe;
 
    private boolean shouldTranslatePossibleValues = false;
    private boolean shouldGenerateDisplayValues   = false;
    private boolean shouldFetchHeavyFields        = false;
+   private boolean shouldOmitHiddenFields        = true;
+   private boolean shouldMaskPasswords           = true;
 
    /////////////////////////////////////////////////////////////////////////////////////////
    // this field - only applies if shouldTranslatePossibleValues is true.                 //
@@ -55,7 +55,8 @@ public class QueryInput extends AbstractTableActionInput
    /////////////////////////////////////////////////////////////////////////////////////////
    private Set<String> fieldsToTranslatePossibleValues;
 
-   private List<QueryJoin> queryJoins = null;
+   private List<QueryJoin> queryJoins     = null;
+   private boolean         selectDistinct = false;
 
    ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
    // if you say you want to includeAssociations, you can limit which ones by passing them in associationNamesToInclude. //
@@ -98,50 +99,6 @@ public class QueryInput extends AbstractTableActionInput
 
 
 
-   /*******************************************************************************
-    ** Getter for skip
-    **
-    *******************************************************************************/
-   public Integer getSkip()
-   {
-      return skip;
-   }
-
-
-
-   /*******************************************************************************
-    ** Setter for skip
-    **
-    *******************************************************************************/
-   public void setSkip(Integer skip)
-   {
-      this.skip = skip;
-   }
-
-
-
-   /*******************************************************************************
-    ** Getter for limit
-    **
-    *******************************************************************************/
-   public Integer getLimit()
-   {
-      return limit;
-   }
-
-
-
-   /*******************************************************************************
-    ** Setter for limit
-    **
-    *******************************************************************************/
-   public void setLimit(Integer limit)
-   {
-      this.limit = limit;
-   }
-
-
-
    /*******************************************************************************
     ** Getter for recordPipe
     **
@@ -359,28 +316,6 @@ public class QueryInput extends AbstractTableActionInput
 
 
 
-   /*******************************************************************************
-    ** Fluent setter for skip
-    *******************************************************************************/
-   public QueryInput withSkip(Integer skip)
-   {
-      this.skip = skip;
-      return (this);
-   }
-
-
-
-   /*******************************************************************************
-    ** Fluent setter for limit
-    *******************************************************************************/
-   public QueryInput withLimit(Integer limit)
-   {
-      this.limit = limit;
-      return (this);
-   }
-
-
-
    /*******************************************************************************
     ** Fluent setter for recordPipe
     *******************************************************************************/
@@ -497,4 +432,97 @@ public class QueryInput extends AbstractTableActionInput
       return (this);
    }
 
+
+
+   /*******************************************************************************
+    ** Getter for shouldMaskPasswords
+    *******************************************************************************/
+   public boolean getShouldMaskPasswords()
+   {
+      return (this.shouldMaskPasswords);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for shouldMaskPasswords
+    *******************************************************************************/
+   public void setShouldMaskPasswords(boolean shouldMaskPasswords)
+   {
+      this.shouldMaskPasswords = shouldMaskPasswords;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for shouldMaskPasswords
+    *******************************************************************************/
+   public QueryInput withShouldMaskPasswords(boolean shouldMaskPasswords)
+   {
+      this.shouldMaskPasswords = shouldMaskPasswords;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for shouldOmitHiddenFields
+    *******************************************************************************/
+   public boolean getShouldOmitHiddenFields()
+   {
+      return (this.shouldOmitHiddenFields);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for shouldOmitHiddenFields
+    *******************************************************************************/
+   public void setShouldOmitHiddenFields(boolean shouldOmitHiddenFields)
+   {
+      this.shouldOmitHiddenFields = shouldOmitHiddenFields;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for shouldOmitHiddenFields
+    *******************************************************************************/
+   public QueryInput withShouldOmitHiddenFields(boolean shouldOmitHiddenFields)
+   {
+      this.shouldOmitHiddenFields = shouldOmitHiddenFields;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for selectDistinct
+    *******************************************************************************/
+   public boolean getSelectDistinct()
+   {
+      return (this.selectDistinct);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for selectDistinct
+    *******************************************************************************/
+   public void setSelectDistinct(boolean selectDistinct)
+   {
+      this.selectDistinct = selectDistinct;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for selectDistinct
+    *******************************************************************************/
+   public QueryInput withSelectDistinct(boolean selectDistinct)
+   {
+      this.selectDistinct = selectDistinct;
+      return (this);
+   }
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QueryOutput.java

@@ -24,6 +24,7 @@ package com.kingsrook.qqq.backend.core.model.actions.tables.query;
 
 import java.io.Serializable;
 import java.util.List;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.model.actions.AbstractActionOutput;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
 
@@ -63,7 +64,7 @@ public class QueryOutput extends AbstractActionOutput implements Serializable
     ** that could be read asynchronously, at any time, by another thread - SO - only
     ** completely populated records should be passed into this method.
     *******************************************************************************/
-   public void addRecord(QRecord record)
+   public void addRecord(QRecord record) throws QException
    {
       storage.addRecord(record);
    }
@@ -73,7 +74,7 @@ public class QueryOutput extends AbstractActionOutput implements Serializable
    /*******************************************************************************
     ** add a list of records to this output
     *******************************************************************************/
-   public void addRecords(List<QRecord> records)
+   public void addRecords(List<QRecord> records) throws QException
    {
       storage.addRecords(records);
    }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QueryOutputRecordPipe.java

@@ -24,6 +24,7 @@ package com.kingsrook.qqq.backend.core.model.actions.tables.query;
 
 import java.util.List;
 import com.kingsrook.qqq.backend.core.actions.reporting.RecordPipe;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.logging.QLogger;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
 
@@ -53,7 +54,7 @@ class QueryOutputRecordPipe implements QueryOutputStorageInterface
     ** add a record to this output
     *******************************************************************************/
    @Override
-   public void addRecord(QRecord record)
+   public void addRecord(QRecord record) throws QException
    {
       recordPipe.addRecord(record);
    }
@@ -64,7 +65,7 @@ class QueryOutputRecordPipe implements QueryOutputStorageInterface
     ** add a list of records to this output
     *******************************************************************************/
    @Override
-   public void addRecords(List<QRecord> records)
+   public void addRecords(List<QRecord> records) throws QException
    {
       recordPipe.addRecords(records);
    }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QueryOutputStorageInterface.java

@@ -23,6 +23,7 @@ package com.kingsrook.qqq.backend.core.model.actions.tables.query;
 
 
 import java.util.List;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
 
 
@@ -36,13 +37,13 @@ interface QueryOutputStorageInterface
    /*******************************************************************************
     ** add a records to this output
     *******************************************************************************/
-   void addRecord(QRecord record);
+   void addRecord(QRecord record) throws QException;
 
 
    /*******************************************************************************
     ** add a list of records to this output
     *******************************************************************************/
-   void addRecords(List<QRecord> records);
+   void addRecords(List<QRecord> records) throws QException;
 
    /*******************************************************************************
     ** Get all stored records

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/update/UpdateInput.java

@@ -25,6 +25,8 @@ package com.kingsrook.qqq.backend.core.model.actions.tables.update;
 import java.util.List;
 import com.kingsrook.qqq.backend.core.actions.QBackendTransaction;
 import com.kingsrook.qqq.backend.core.model.actions.AbstractTableActionInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.InputSource;
+import com.kingsrook.qqq.backend.core.model.actions.tables.QInputSource;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
 
 
@@ -36,6 +38,7 @@ public class UpdateInput extends AbstractTableActionInput
 {
    private QBackendTransaction transaction;
    private List<QRecord>       records;
+   private InputSource         inputSource = QInputSource.SYSTEM;
 
    ////////////////////////////////////////////////////////////////////////////////////////////
    // allow a caller to specify that they KNOW this optimization (e.g., in SQL) can be made. //
@@ -46,6 +49,7 @@ public class UpdateInput extends AbstractTableActionInput
    private Boolean areAllValuesBeingUpdatedTheSame = null;
 
    private boolean omitDmlAudit = false;
+   private String  auditContext = null;
 
 
 
@@ -187,4 +191,66 @@ public class UpdateInput extends AbstractTableActionInput
       return (this);
    }
 
+
+
+   /*******************************************************************************
+    ** Getter for auditContext
+    *******************************************************************************/
+   public String getAuditContext()
+   {
+      return (this.auditContext);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for auditContext
+    *******************************************************************************/
+   public void setAuditContext(String auditContext)
+   {
+      this.auditContext = auditContext;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for auditContext
+    *******************************************************************************/
+   public UpdateInput withAuditContext(String auditContext)
+   {
+      this.auditContext = auditContext;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for inputSource
+    *******************************************************************************/
+   public InputSource getInputSource()
+   {
+      return (this.inputSource);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for inputSource
+    *******************************************************************************/
+   public void setInputSource(InputSource inputSource)
+   {
+      this.inputSource = inputSource;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for inputSource
+    *******************************************************************************/
+   public UpdateInput withInputSource(InputSource inputSource)
+   {
+      this.inputSource = inputSource;
+      return (this);
+   }
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/dashboard/widgets/ChildRecordListData.java

@@ -39,8 +39,9 @@ public class ChildRecordListData extends QWidgetData
    private QueryOutput    queryOutput;
    private QTableMetaData childTableMetaData;
 
-   private String tablePath;
-   private String viewAllLink;
+   private String  tablePath;
+   private String  viewAllLink;
+   private Integer totalRows;
 
    private boolean                   canAddChildRecord = false;
    private Map<String, Serializable> defaultValuesForNewChildRecords;
@@ -51,13 +52,14 @@ public class ChildRecordListData extends QWidgetData
    /*******************************************************************************
     **
     *******************************************************************************/
-   public ChildRecordListData(String title, QueryOutput queryOutput, QTableMetaData childTableMetaData, String tablePath, String viewAllLink)
+   public ChildRecordListData(String title, QueryOutput queryOutput, QTableMetaData childTableMetaData, String tablePath, String viewAllLink, Integer totalRows)
    {
       this.title = title;
       this.queryOutput = queryOutput;
       this.childTableMetaData = childTableMetaData;
       this.tablePath = tablePath;
       this.viewAllLink = viewAllLink;
+      this.totalRows = totalRows;
    }
 
 
@@ -319,4 +321,35 @@ public class ChildRecordListData extends QWidgetData
       return (this);
    }
 
+
+
+   /*******************************************************************************
+    ** Getter for totalRows
+    *******************************************************************************/
+   public Integer getTotalRows()
+   {
+      return (this.totalRows);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for totalRows
+    *******************************************************************************/
+   public void setTotalRows(Integer totalRows)
+   {
+      this.totalRows = totalRows;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for totalRows
+    *******************************************************************************/
+   public ChildRecordListData withTotalRows(Integer totalRows)
+   {
+      this.totalRows = totalRows;
+      return (this);
+   }
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/data/QField.java

@@ -58,6 +58,11 @@ public @interface QField
     *******************************************************************************/
    boolean isEditable() default true;
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   boolean isHidden() default false;
+
    /*******************************************************************************
     **
     *******************************************************************************/

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/data/QRecord.java

@@ -35,6 +35,8 @@ import java.util.Map;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+import com.kingsrook.qqq.backend.core.model.statusmessages.QErrorMessage;
+import com.kingsrook.qqq.backend.core.model.statusmessages.QWarningMessage;
 import com.kingsrook.qqq.backend.core.utils.ValueUtils;
 import org.apache.commons.lang.SerializationUtils;
 
@@ -54,7 +56,8 @@ import org.apache.commons.lang.SerializationUtils;
  **
  ** Errors are meant to hold information about things that went wrong when
  ** processing a record - e.g., in a list of records that may be the output of an
- ** action, like a bulk load.  TODO - redo as some status object?
+ ** action, like a bulk load.  Warnings play a similar role, but are just advice
+ ** - they don't mean that the action was failed, just something you may need to know.
  *******************************************************************************/
 public class QRecord implements Serializable
 {
@@ -64,11 +67,17 @@ public class QRecord implements Serializable
    private Map<String, Serializable> values         = new LinkedHashMap<>();
    private Map<String, String>       displayValues  = new LinkedHashMap<>();
    private Map<String, Serializable> backendDetails = new LinkedHashMap<>();
-   private List<String>              errors         = new ArrayList<>();
+
+   private List<QErrorMessage>   errors   = new ArrayList<>();
+   private List<QWarningMessage> warnings = new ArrayList<>();
 
    private Map<String, List<QRecord>> associatedRecords = new HashMap<>();
 
-   public static final String BACKEND_DETAILS_TYPE_JSON_SOURCE_OBJECT = "jsonSourceObject";
+   ////////////////////////////////////////////////
+   // well-known keys for the backendDetails map //
+   ////////////////////////////////////////////////
+   public static final String BACKEND_DETAILS_TYPE_JSON_SOURCE_OBJECT  = "jsonSourceObject"; // String of JSON
+   public static final String BACKEND_DETAILS_TYPE_HEAVY_FIELD_LENGTHS = "heavyFieldLengths"; // Map<fieldName, length>
 
 
 
@@ -105,6 +114,7 @@ public class QRecord implements Serializable
       this.displayValues = doDeepCopy(record.displayValues);
       this.backendDetails = doDeepCopy(record.backendDetails);
       this.errors = doDeepCopy(record.errors);
+      this.warnings = doDeepCopy(record.warnings);
       this.associatedRecords = doDeepCopy(record.associatedRecords);
    }
 
@@ -122,7 +132,7 @@ public class QRecord implements Serializable
 
 
    /*******************************************************************************
-    **
+    ** todo - move to a cloning utils maybe?
     *******************************************************************************/
    @SuppressWarnings({ "unchecked" })
    private <K, V> Map<K, V> doDeepCopy(Map<K, V> map)
@@ -143,7 +153,7 @@ public class QRecord implements Serializable
 
 
    /*******************************************************************************
-    **
+    ** todo - move to a cloning utils maybe?
     *******************************************************************************/
    @SuppressWarnings({ "unchecked" })
    private <T> List<T> doDeepCopy(List<T> list)
@@ -196,6 +206,17 @@ public class QRecord implements Serializable
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public void removeValue(String fieldName)
+   {
+      values.remove(fieldName);
+      displayValues.remove(fieldName);
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/
@@ -530,7 +551,7 @@ public class QRecord implements Serializable
     ** Getter for errors
     **
     *******************************************************************************/
-   public List<String> getErrors()
+   public List<QErrorMessage> getErrors()
    {
       return (errors);
    }
@@ -541,7 +562,7 @@ public class QRecord implements Serializable
     ** Setter for errors
     **
     *******************************************************************************/
-   public void setErrors(List<String> errors)
+   public void setErrors(List<QErrorMessage> errors)
    {
       this.errors = errors;
    }
@@ -552,7 +573,7 @@ public class QRecord implements Serializable
     ** Add one error to this record
     **
     *******************************************************************************/
-   public void addError(String error)
+   public void addError(QErrorMessage error)
    {
       this.errors.add(error);
    }
@@ -563,7 +584,7 @@ public class QRecord implements Serializable
     ** Fluently Add one error to this record
     **
     *******************************************************************************/
-   public QRecord withError(String error)
+   public QRecord withError(QErrorMessage error)
    {
       addError(error);
       return (this);
@@ -641,4 +662,46 @@ public class QRecord implements Serializable
       return (this);
    }
 
+
+
+   /*******************************************************************************
+    ** Getter for warnings
+    *******************************************************************************/
+   public List<QWarningMessage> getWarnings()
+   {
+      return (this.warnings);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for warnings
+    *******************************************************************************/
+   public void setWarnings(List<QWarningMessage> warnings)
+   {
+      this.warnings = warnings;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for warnings
+    *******************************************************************************/
+   public QRecord withWarnings(List<QWarningMessage> warnings)
+   {
+      this.warnings = warnings;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Add one warning to this record
+    **
+    *******************************************************************************/
+   public void addWarning(QWarningMessage warning)
+   {
+      this.warnings.add(warning);
+   }
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/data/QRecordEntity.java

@@ -31,8 +31,11 @@ import java.time.LocalDate;
 import java.time.LocalTime;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Locale;
+import java.util.Map;
+import java.util.Objects;
 import java.util.Optional;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.exceptions.QRuntimeException;
@@ -49,6 +52,8 @@ public abstract class QRecordEntity
 
    private static final ListingHash<Class<? extends QRecordEntity>, QRecordEntityField> fieldMapping = new ListingHash<>();
 
+   private Map<String, Serializable> originalRecordValues;
+
 
 
    /*******************************************************************************
@@ -80,11 +85,13 @@ public abstract class QRecordEntity
       try
       {
          List<QRecordEntityField> fieldList = getFieldList(this.getClass());
+         originalRecordValues = new HashMap<>();
          for(QRecordEntityField qRecordEntityField : fieldList)
          {
             Serializable value      = qRecord.getValue(qRecordEntityField.getFieldName());
             Object       typedValue = qRecordEntityField.convertValueType(value);
             qRecordEntityField.getSetter().invoke(this, typedValue);
+            originalRecordValues.put(qRecordEntityField.getFieldName(), value);
          }
       }
       catch(Exception e)
@@ -121,6 +128,41 @@ public abstract class QRecordEntity
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public QRecord toQRecordOnlyChangedFields()
+   {
+      try
+      {
+         QRecord qRecord = new QRecord();
+
+         List<QRecordEntityField> fieldList = getFieldList(this.getClass());
+         for(QRecordEntityField qRecordEntityField : fieldList)
+         {
+            Serializable thisValue     = (Serializable) qRecordEntityField.getGetter().invoke(this);
+            Serializable originalValue = null;
+            if(originalRecordValues != null)
+            {
+               originalValue = originalRecordValues.get(qRecordEntityField.getFieldName());
+            }
+
+            if(!Objects.equals(thisValue, originalValue))
+            {
+               qRecord.setValue(qRecordEntityField.getFieldName(), thisValue);
+            }
+         }
+
+         return (qRecord);
+      }
+      catch(Exception e)
+      {
+         throw (new QRuntimeException("Error building qRecord from entity.", e));
+      }
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/
@@ -253,7 +295,8 @@ public abstract class QRecordEntity
          || returnType.equals(BigDecimal.class)
          || returnType.equals(Instant.class)
          || returnType.equals(LocalDate.class)
-         || returnType.equals(LocalTime.class));
+         || returnType.equals(LocalTime.class)
+         || returnType.equals(byte[].class));
       /////////////////////////////////////////////
       // note - this list has implications upon: //
       // - QFieldType.fromClass                  //

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/data/QRecordEntityField.java

@@ -165,6 +165,11 @@ public class QRecordEntityField
          {
             return (ValueUtils.getValueAsLocalTime(value));
          }
+
+         if(type.equals(byte[].class))
+         {
+            return (ValueUtils.getValueAsByteArray(value));
+         }
       }
       catch(Exception e)
       {

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/data/QRecordEnum.java

@@ -167,7 +167,8 @@ public interface QRecordEnum
          || returnType.equals(BigDecimal.class)
          || returnType.equals(Instant.class)
          || returnType.equals(LocalDate.class)
-         || returnType.equals(LocalTime.class));
+         || returnType.equals(LocalTime.class)
+         || returnType.equals(byte[].class));
       /////////////////////////////////////////////
       // note - this list has implications upon: //
       // - QFieldType.fromClass                  //

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/QBackendMetaData.java

@@ -43,6 +43,9 @@ public class QBackendMetaData
    private String name;
    private String backendType;
 
+   private Boolean usesVariants = false;
+   private String  variantOptionsTableName;
+
    private Set<Capability> enabledCapabilities  = new HashSet<>();
    private Set<Capability> disabledCapabilities = new HashSet<>();
 
@@ -343,4 +346,67 @@ public class QBackendMetaData
       // noop in base class //
       ////////////////////////
    }
+
+
+
+   /*******************************************************************************
+    ** Getter for usesVariants
+    *******************************************************************************/
+   public Boolean getUsesVariants()
+   {
+      return (this.usesVariants);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for usesVariants
+    *******************************************************************************/
+   public void setUsesVariants(Boolean usesVariants)
+   {
+      this.usesVariants = usesVariants;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for usesVariants
+    *******************************************************************************/
+   public QBackendMetaData withUsesVariants(Boolean usesVariants)
+   {
+      this.usesVariants = usesVariants;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for variantsOptionTableName
+    *******************************************************************************/
+   public String getVariantOptionsTableName()
+   {
+      return (this.variantOptionsTableName);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for variantsOptionTableName
+    *******************************************************************************/
+   public void setVariantOptionsTableName(String variantOptionsTableName)
+   {
+      this.variantOptionsTableName = variantOptionsTableName;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for variantsOptionTableName
+    *******************************************************************************/
+   public QBackendMetaData withVariantsOptionTableName(String variantsOptionTableName)
+   {
+      this.variantOptionsTableName = variantsOptionTableName;
+      return (this);
+   }
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/QInstance.java

@@ -30,6 +30,7 @@ import java.util.List;
 import java.util.Map;
 import java.util.Set;
 import com.fasterxml.jackson.annotation.JsonIgnore;
+import com.kingsrook.qqq.backend.core.actions.metadata.JoinGraph;
 import com.kingsrook.qqq.backend.core.actions.metadata.MetaDataAction;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.instances.QInstanceValidationKey;
@@ -106,6 +107,8 @@ public class QInstance
    private Map<String, String> memoizedTablePaths   = new HashMap<>();
    private Map<String, String> memoizedProcessPaths = new HashMap<>();
 
+   private JoinGraph joinGraph;
+
 
 
    /*******************************************************************************
@@ -1136,4 +1139,30 @@ public class QInstance
       this.middlewareMetaData.put(middlewareMetaData.getType(), middlewareMetaData);
       return (this);
    }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public JoinGraph getJoinGraph()
+   {
+      return (this.joinGraph);
+   }
+
+
+
+   /*******************************************************************************
+    ** Only the validation (and enrichment) code should set the instance's joinGraph
+    ** so, we take a package-only-constructable validation key as a param along with
+    ** the joinGraph - and we throw IllegalArgumentException if a non-null key is given.
+    *******************************************************************************/
+   public void setJoinGraph(QInstanceValidationKey key, JoinGraph joinGraph) throws IllegalArgumentException
+   {
+      if(key == null)
+      {
+         throw (new IllegalArgumentException("A ValidationKey must be provided"));
+      }
+      this.joinGraph = joinGraph;
+   }
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/authentication/Auth0AuthenticationMetaData.java

@@ -57,7 +57,7 @@ public class Auth0AuthenticationMetaData extends QAuthenticationMetaData
    /////////////////////////////////////////
    private String       applicationNameField;
    private String       auth0ClientIdField;
-   private String       auth0ClientSecretMaskedField;
+   private String       auth0ClientSecretField;
    private Serializable qqqRecordIdField;
 
 
@@ -67,6 +67,7 @@ public class Auth0AuthenticationMetaData extends QAuthenticationMetaData
    private String clientAuth0ApplicationIdField;
    private String auth0AccessTokenField;
    private String qqqAccessTokenField;
+   private String qqqApiKeyField;
    private String expiresInSecondsField;
 
 
@@ -387,40 +388,6 @@ public class Auth0AuthenticationMetaData extends QAuthenticationMetaData
 
 
 
-   /*******************************************************************************
-    ** Getter for auth0ClientSecretMaskedField
-    **
-    *******************************************************************************/
-   public String getAuth0ClientSecretMaskedField()
-   {
-      return auth0ClientSecretMaskedField;
-   }
-
-
-
-   /*******************************************************************************
-    ** Setter for auth0ClientSecretMaskedField
-    **
-    *******************************************************************************/
-   public void setAuth0ClientSecretMaskedField(String auth0ClientSecretMaskedField)
-   {
-      this.auth0ClientSecretMaskedField = auth0ClientSecretMaskedField;
-   }
-
-
-
-   /*******************************************************************************
-    ** Fluent setter for auth0ClientSecretMaskedField
-    **
-    *******************************************************************************/
-   public Auth0AuthenticationMetaData withAuth0ClientSecretMaskedField(String auth0ClientSecretMaskedField)
-   {
-      this.auth0ClientSecretMaskedField = auth0ClientSecretMaskedField;
-      return (this);
-   }
-
-
-
    /*******************************************************************************
     ** Getter for clientAuth0ApplicationIdField
     **
@@ -555,4 +522,66 @@ public class Auth0AuthenticationMetaData extends QAuthenticationMetaData
       return (this);
    }
 
+
+
+   /*******************************************************************************
+    ** Getter for qqqApiKeyField
+    *******************************************************************************/
+   public String getQqqApiKeyField()
+   {
+      return (this.qqqApiKeyField);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for qqqApiKeyField
+    *******************************************************************************/
+   public void setQqqApiKeyField(String qqqApiKeyField)
+   {
+      this.qqqApiKeyField = qqqApiKeyField;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for qqqApiKeyField
+    *******************************************************************************/
+   public Auth0AuthenticationMetaData withQqqApiKeyField(String qqqApiKeyField)
+   {
+      this.qqqApiKeyField = qqqApiKeyField;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for auth0ClientSecretField
+    *******************************************************************************/
+   public String getAuth0ClientSecretField()
+   {
+      return (this.auth0ClientSecretField);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for auth0ClientSecretField
+    *******************************************************************************/
+   public void setAuth0ClientSecretField(String auth0ClientSecretField)
+   {
+      this.auth0ClientSecretField = auth0ClientSecretField;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for auth0ClientSecretField
+    *******************************************************************************/
+   public Auth0AuthenticationMetaData withAuth0ClientSecretField(String auth0ClientSecretField)
+   {
+      this.auth0ClientSecretField = auth0ClientSecretField;
+      return (this);
+   }
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/branding/QBrandingMetaData.java

@@ -35,6 +35,9 @@ public class QBrandingMetaData
    private String icon;
    private String accentColor;
 
+   private String environmentBannerText;
+   private String environmentBannerColor;
+
 
 
    /*******************************************************************************
@@ -244,4 +247,66 @@ public class QBrandingMetaData
       return (this);
    }
 
+
+
+   /*******************************************************************************
+    ** Getter for environmentBannerText
+    *******************************************************************************/
+   public String getEnvironmentBannerText()
+   {
+      return (this.environmentBannerText);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for environmentBannerText
+    *******************************************************************************/
+   public void setEnvironmentBannerText(String environmentBannerText)
+   {
+      this.environmentBannerText = environmentBannerText;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for environmentBannerText
+    *******************************************************************************/
+   public QBrandingMetaData withEnvironmentBannerText(String environmentBannerText)
+   {
+      this.environmentBannerText = environmentBannerText;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for environmentBannerColor
+    *******************************************************************************/
+   public String getEnvironmentBannerColor()
+   {
+      return (this.environmentBannerColor);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for environmentBannerColor
+    *******************************************************************************/
+   public void setEnvironmentBannerColor(String environmentBannerColor)
+   {
+      this.environmentBannerColor = environmentBannerColor;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for environmentBannerColor
+    *******************************************************************************/
+   public QBrandingMetaData withEnvironmentBannerColor(String environmentBannerColor)
+   {
+      this.environmentBannerColor = environmentBannerColor;
+      return (this);
+   }
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/code/QCodeReference.java

@@ -23,9 +23,6 @@ package com.kingsrook.qqq.backend.core.model.metadata.code;
 
 
 import java.io.Serializable;
-import com.kingsrook.qqq.backend.core.actions.automation.RecordAutomationHandler;
-import com.kingsrook.qqq.backend.core.actions.processes.BackendStep;
-import com.kingsrook.qqq.backend.core.actions.values.QCustomPossibleValueProvider;
 
 
 /*******************************************************************************
@@ -34,9 +31,8 @@ import com.kingsrook.qqq.backend.core.actions.values.QCustomPossibleValueProvide
  *******************************************************************************/
 public class QCodeReference implements Serializable
 {
-   private String     name;
-   private QCodeType  codeType;
-   private QCodeUsage codeUsage;
+   private String    name;
+   private QCodeType codeType;
 
    private String inlineCode;
 
@@ -54,11 +50,10 @@ public class QCodeReference implements Serializable
    /*******************************************************************************
     ** Constructor that takes all args
     *******************************************************************************/
-   public QCodeReference(String name, QCodeType codeType, QCodeUsage codeUsage)
+   public QCodeReference(String name, QCodeType codeType)
    {
       this.name = name;
       this.codeType = codeType;
-      this.codeUsage = codeUsage;
    }
 
 
@@ -81,35 +76,6 @@ public class QCodeReference implements Serializable
    {
       this.name = javaClass.getName();
       this.codeType = QCodeType.JAVA;
-
-      if(BackendStep.class.isAssignableFrom(javaClass))
-      {
-         this.codeUsage = QCodeUsage.BACKEND_STEP;
-      }
-      else if(QCustomPossibleValueProvider.class.isAssignableFrom(javaClass))
-      {
-         this.codeUsage = QCodeUsage.POSSIBLE_VALUE_PROVIDER;
-      }
-      else if(RecordAutomationHandler.class.isAssignableFrom(javaClass))
-      {
-         this.codeUsage = QCodeUsage.RECORD_AUTOMATION_HANDLER;
-      }
-      else
-      {
-         throw (new IllegalStateException("Unable to infer code usage type for class: " + javaClass.getName()));
-      }
-   }
-
-
-
-   /*******************************************************************************
-    ** Constructor that just takes a java class and code usage.
-    *******************************************************************************/
-   public QCodeReference(Class<?> javaClass, QCodeUsage codeUsage)
-   {
-      this.name = javaClass.getName();
-      this.codeType = QCodeType.JAVA;
-      this.codeUsage = codeUsage;
    }
 
 
@@ -182,40 +148,6 @@ public class QCodeReference implements Serializable
 
 
 
-   /*******************************************************************************
-    ** Getter for codeUsage
-    **
-    *******************************************************************************/
-   public QCodeUsage getCodeUsage()
-   {
-      return codeUsage;
-   }
-
-
-
-   /*******************************************************************************
-    ** Setter for codeUsage
-    **
-    *******************************************************************************/
-   public void setCodeUsage(QCodeUsage codeUsage)
-   {
-      this.codeUsage = codeUsage;
-   }
-
-
-
-   /*******************************************************************************
-    ** Setter for codeUsage
-    **
-    *******************************************************************************/
-   public QCodeReference withCodeUsage(QCodeUsage codeUsage)
-   {
-      this.codeUsage = codeUsage;
-      return (this);
-   }
-
-
-
    /*******************************************************************************
     ** Getter for inlineCode
     **

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/dashboard/QWidgetMetaData.java

@@ -52,6 +52,9 @@ public class QWidgetMetaData implements QWidgetMetaDataInterface
    private List<WidgetDropdownData> dropdowns;
    private boolean                  storeDropdownSelections;
 
+   private boolean showReloadButton = true;
+   private boolean showExportButton = true;
+
    protected Map<String, Serializable> defaultValues = new LinkedHashMap<>();
 
 
@@ -529,4 +532,66 @@ public class QWidgetMetaData implements QWidgetMetaDataInterface
       return (this);
    }
 
+
+
+   /*******************************************************************************
+    ** Getter for showReloadButton
+    *******************************************************************************/
+   public boolean getShowReloadButton()
+   {
+      return (this.showReloadButton);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for showReloadButton
+    *******************************************************************************/
+   public void setShowReloadButton(boolean showReloadButton)
+   {
+      this.showReloadButton = showReloadButton;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for showReloadButton
+    *******************************************************************************/
+   public QWidgetMetaData withShowReloadButton(boolean showReloadButton)
+   {
+      this.showReloadButton = showReloadButton;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for showExportButton
+    *******************************************************************************/
+   public boolean getShowExportButton()
+   {
+      return (this.showExportButton);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for showExportButton
+    *******************************************************************************/
+   public void setShowExportButton(boolean showExportButton)
+   {
+      this.showExportButton = showExportButton;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for showExportButton
+    *******************************************************************************/
+   public QWidgetMetaData withShowExportButton(boolean showExportButton)
+   {
+      this.showExportButton = showExportButton;
+      return (this);
+   }
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/dashboard/nocode/WidgetQueryField.java

@@ -62,8 +62,9 @@ public class WidgetQueryField extends AbstractWidgetValueSourceWithFilter
    {
       QueryInput queryInput = new QueryInput();
       queryInput.setTableName(tableName);
-      queryInput.setFilter(getEffectiveFilter(input));
-      queryInput.setLimit(1);
+      QQueryFilter effectiveFilter = getEffectiveFilter(input);
+      queryInput.setFilter(effectiveFilter);
+      effectiveFilter.setLimit(1);
       QueryOutput queryOutput = new QueryAction().execute(queryInput);
       if(CollectionUtils.nullSafeHasContents(queryOutput.getRecords()))
       {

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/AdornmentType.java

@@ -39,6 +39,8 @@ public enum AdornmentType
    SIZE,
    CODE_EDITOR,
    RENDER_HTML,
+   REVEAL,
+   FILE_DOWNLOAD,
    ERROR;
    //////////////////////////////////////////////////////////////////////////
    // keep these values in sync with AdornmentType.ts in qqq-frontend-core //
@@ -57,6 +59,26 @@ public enum AdornmentType
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public interface FileDownloadValues
+   {
+      String FILE_NAME_FIELD   = "fileNameField";
+      String DEFAULT_EXTENSION = "defaultExtension";
+      String DEFAULT_MIME_TYPE = "defaultMimeType";
+
+      ////////////////////////////////////////////////////
+      // use these two together, as in:                 //
+      // FILE_NAME_FORMAT = "Order %s Packing Slip.pdf" //
+      // FILE_NAME_FORMAT_FIELDS = "orderId"            //
+      ////////////////////////////////////////////////////
+      String FILE_NAME_FORMAT        = "fileNameFormat";
+      String FILE_NAME_FORMAT_FIELDS = "fileNameFormatFields";
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/
@@ -111,6 +133,7 @@ public enum AdornmentType
       XSMALL,
       SMALL,
       MEDIUM,
+      MEDLARGE,
       LARGE,
       XLARGE;
 

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/FieldAdornment.java

@@ -25,6 +25,8 @@ package com.kingsrook.qqq.backend.core.model.metadata.fields;
 import java.io.Serializable;
 import java.util.HashMap;
 import java.util.Map;
+import java.util.Optional;
+import com.fasterxml.jackson.annotation.JsonIgnore;
 import com.kingsrook.qqq.backend.core.utils.Pair;
 
 
@@ -116,6 +118,22 @@ public class FieldAdornment
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @JsonIgnore
+   public Optional<Serializable> getValue(String key)
+   {
+      if(key != null && values != null)
+      {
+         return (Optional.ofNullable(values.get(key)));
+      }
+
+      return (Optional.empty());
+   }
+
+
+
    /*******************************************************************************
     ** Setter for values
     **

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/QFieldMetaData.java

@@ -31,6 +31,7 @@ import java.util.List;
 import java.util.Map;
 import java.util.Optional;
 import java.util.Set;
+import com.fasterxml.jackson.annotation.JsonIgnore;
 import com.github.hervian.reflection.Fun;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
@@ -54,6 +55,7 @@ public class QFieldMetaData implements Cloneable
    private QFieldType type;
    private boolean    isRequired = false;
    private boolean    isEditable = true;
+   private boolean    isHidden   = false;
    private boolean    isHeavy    = false;
 
    private FieldSecurityLock fieldSecurityLock;
@@ -183,6 +185,7 @@ public class QFieldMetaData implements Cloneable
             QField fieldAnnotation = optionalFieldAnnotation.get();
             setIsRequired(fieldAnnotation.isRequired());
             setIsEditable(fieldAnnotation.isEditable());
+            setIsHidden(fieldAnnotation.isHidden());
 
             if(StringUtils.hasContent(fieldAnnotation.label()))
             {
@@ -521,6 +524,25 @@ public class QFieldMetaData implements Cloneable
 
 
 
+   /*******************************************************************************
+    ** does this field have the given addornment
+    **
+    *******************************************************************************/
+   public boolean hasAdornmentType(AdornmentType adornmentType)
+   {
+      for(FieldAdornment thisAdornment : CollectionUtils.nonNullList(this.adornments))
+      {
+         if(AdornmentType.REVEAL.equals(thisAdornment.getType()))
+         {
+            return (true);
+         }
+      }
+
+      return (false);
+   }
+
+
+
    /*******************************************************************************
     ** Getter for adornments
     **
@@ -532,6 +554,29 @@ public class QFieldMetaData implements Cloneable
 
 
 
+   /*******************************************************************************
+    ** Getter for adornments
+    **
+    *******************************************************************************/
+   @JsonIgnore
+   public Optional<FieldAdornment> getAdornment(AdornmentType adornmentType)
+   {
+      if(adornmentType != null && adornments != null)
+      {
+         for(FieldAdornment adornment : adornments)
+         {
+            if(adornmentType.equals(adornment.getType()))
+            {
+               return Optional.of((adornment));
+            }
+         }
+      }
+
+      return (Optional.empty());
+   }
+
+
+
    /*******************************************************************************
     ** Setter for adornments
     **
@@ -851,4 +896,36 @@ public class QFieldMetaData implements Cloneable
       this.middlewareMetaData.put(middlewareMetaData.getType(), middlewareMetaData);
       return (this);
    }
+
+
+
+   /*******************************************************************************
+    ** Getter for isHidden
+    *******************************************************************************/
+   public boolean getIsHidden()
+   {
+      return (this.isHidden);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for isHidden
+    *******************************************************************************/
+   public void setIsHidden(boolean isHidden)
+   {
+      this.isHidden = isHidden;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for isHidden
+    *******************************************************************************/
+   public QFieldMetaData withIsHidden(boolean isHidden)
+   {
+      this.isHidden = isHidden;
+      return (this);
+   }
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/QFieldType.java

@@ -85,6 +85,10 @@ public enum QFieldType
       {
          return (BOOLEAN);
       }
+      if(c.equals(byte[].class))
+      {
+         return (BLOB);
+      }
 
       throw (new QException("Unrecognized class [" + c + "]"));
    }
@@ -108,4 +112,14 @@ public enum QFieldType
    {
       return this == QFieldType.INTEGER || this == QFieldType.DECIMAL;
    }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public boolean needsMasked()
+   {
+      return this == QFieldType.PASSWORD;
+   }
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/frontend/QFrontendExposedJoin.java

@@ -0,0 +1,178 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2023.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.model.metadata.frontend;
+
+
+import java.util.ArrayList;
+import java.util.List;
+import com.kingsrook.qqq.backend.core.model.metadata.joins.QJoinMetaData;
+
+
+/*******************************************************************************
+ ** Version of an ExposedJoin for a frontend to see
+ *******************************************************************************/
+public class QFrontendExposedJoin
+{
+   private String                 label;
+   private Boolean                isMany;
+   private QFrontendTableMetaData joinTable;
+   private List<QJoinMetaData>    joinPath;
+
+
+
+   /*******************************************************************************
+    ** Getter for label
+    *******************************************************************************/
+   public String getLabel()
+   {
+      return (this.label);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for label
+    *******************************************************************************/
+   public void setLabel(String label)
+   {
+      this.label = label;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for label
+    *******************************************************************************/
+   public QFrontendExposedJoin withLabel(String label)
+   {
+      this.label = label;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for joinTable
+    *******************************************************************************/
+   public QFrontendTableMetaData getJoinTable()
+   {
+      return (this.joinTable);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for joinTable
+    *******************************************************************************/
+   public void setJoinTable(QFrontendTableMetaData joinTable)
+   {
+      this.joinTable = joinTable;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for joinTable
+    *******************************************************************************/
+   public QFrontendExposedJoin withJoinTable(QFrontendTableMetaData joinTable)
+   {
+      this.joinTable = joinTable;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for joinPath
+    *******************************************************************************/
+   public List<QJoinMetaData> getJoinPath()
+   {
+      return (this.joinPath);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for joinPath
+    *******************************************************************************/
+   public void setJoinPath(List<QJoinMetaData> joinPath)
+   {
+      this.joinPath = joinPath;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for joinPath
+    *******************************************************************************/
+   public QFrontendExposedJoin withJoinPath(List<QJoinMetaData> joinPath)
+   {
+      this.joinPath = joinPath;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Add one join to the join path in here
+    *******************************************************************************/
+   public void addJoin(QJoinMetaData join)
+   {
+      if(this.joinPath == null)
+      {
+         this.joinPath = new ArrayList<>();
+      }
+      this.joinPath.add(join);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for isMany
+    *******************************************************************************/
+   public Boolean getIsMany()
+   {
+      return (this.isMany);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for isMany
+    *******************************************************************************/
+   public void setIsMany(Boolean isMany)
+   {
+      this.isMany = isMany;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for isMany
+    *******************************************************************************/
+   public QFrontendExposedJoin withIsMany(Boolean isMany)
+   {
+      this.isMany = isMany;
+      return (this);
+   }
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/frontend/QFrontendFieldMetaData.java

@@ -43,6 +43,7 @@ public class QFrontendFieldMetaData
    private QFieldType type;
    private boolean    isRequired;
    private boolean    isEditable;
+   private boolean    isHeavy;
    private String     possibleValueSourceName;
    private String     displayFormat;
 
@@ -64,6 +65,7 @@ public class QFrontendFieldMetaData
       this.type = fieldMetaData.getType();
       this.isRequired = fieldMetaData.getIsRequired();
       this.isEditable = fieldMetaData.getIsEditable();
+      this.isHeavy = fieldMetaData.getIsHeavy();
       this.possibleValueSourceName = fieldMetaData.getPossibleValueSourceName();
       this.displayFormat = fieldMetaData.getDisplayFormat();
       this.adornments = fieldMetaData.getAdornments();
@@ -126,6 +128,17 @@ public class QFrontendFieldMetaData
 
 
 
+   /*******************************************************************************
+    ** Getter for isHeavy
+    **
+    *******************************************************************************/
+   public boolean getIsHeavy()
+   {
+      return isHeavy;
+   }
+
+
+
    /*******************************************************************************
     ** Getter for displayFormat
     **

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/frontend/QFrontendTableMetaData.java

@@ -22,6 +22,7 @@
 package com.kingsrook.qqq.backend.core.model.metadata.frontend;
 
 
+import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
@@ -32,12 +33,16 @@ import com.fasterxml.jackson.annotation.JsonInclude;
 import com.fasterxml.jackson.annotation.JsonInclude.Include;
 import com.kingsrook.qqq.backend.core.actions.permissions.PermissionsHelper;
 import com.kingsrook.qqq.backend.core.actions.permissions.TablePermissionSubType;
+import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.model.actions.AbstractActionInput;
 import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.Capability;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.ExposedJoin;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QFieldSection;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
 
 
 /*******************************************************************************
@@ -58,6 +63,8 @@ public class QFrontendTableMetaData
    private Map<String, QFrontendFieldMetaData> fields;
    private List<QFieldSection>                 sections;
 
+   private List<QFrontendExposedJoin> exposedJoins;
+
    private Set<String> capabilities;
 
    private boolean readPermission;
@@ -74,7 +81,7 @@ public class QFrontendTableMetaData
    /*******************************************************************************
     **
     *******************************************************************************/
-   public QFrontendTableMetaData(AbstractActionInput actionInput, QBackendMetaData backendForTable, QTableMetaData tableMetaData, boolean includeFields)
+   public QFrontendTableMetaData(AbstractActionInput actionInput, QBackendMetaData backendForTable, QTableMetaData tableMetaData, boolean includeFields, boolean includeJoins)
    {
       this.name = tableMetaData.getName();
       this.label = tableMetaData.getLabel();
@@ -84,14 +91,39 @@ public class QFrontendTableMetaData
       {
          this.primaryKeyField = tableMetaData.getPrimaryKeyField();
          this.fields = new HashMap<>();
-         for(Map.Entry<String, QFieldMetaData> entry : tableMetaData.getFields().entrySet())
+         for(String fieldName : tableMetaData.getFields().keySet())
          {
-            this.fields.put(entry.getKey(), new QFrontendFieldMetaData(entry.getValue()));
+            QFieldMetaData field = tableMetaData.getField(fieldName);
+            if(!field.getIsHidden())
+            {
+               this.fields.put(fieldName, new QFrontendFieldMetaData(field));
+            }
          }
 
          this.sections = tableMetaData.getSections();
       }
 
+      if(includeJoins)
+      {
+         QInstance qInstance = QContext.getQInstance();
+
+         this.exposedJoins = new ArrayList<>();
+         for(ExposedJoin exposedJoin : CollectionUtils.nonNullList(tableMetaData.getExposedJoins()))
+         {
+            QFrontendExposedJoin frontendExposedJoin = new QFrontendExposedJoin();
+            this.exposedJoins.add(frontendExposedJoin);
+
+            QTableMetaData joinTable = qInstance.getTable(exposedJoin.getJoinTable());
+            frontendExposedJoin.setLabel(exposedJoin.getLabel());
+            frontendExposedJoin.setIsMany(exposedJoin.getIsMany());
+            frontendExposedJoin.setJoinTable(new QFrontendTableMetaData(actionInput, backendForTable, joinTable, includeFields, false));
+            for(String joinName : exposedJoin.getJoinPath())
+            {
+               frontendExposedJoin.addJoin(qInstance.getJoin(joinName));
+            }
+         }
+      }
+
       if(tableMetaData.getIcon() != null)
       {
          this.iconName = tableMetaData.getIcon().getName();
@@ -259,4 +291,15 @@ public class QFrontendTableMetaData
    {
       return deletePermission;
    }
+
+
+
+   /*******************************************************************************
+    ** Getter for exposedJoins
+    **
+    *******************************************************************************/
+   public List<QFrontendExposedJoin> getExposedJoins()
+   {
+      return exposedJoins;
+   }
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/frontend/QFrontendWidgetMetaData.java

@@ -27,6 +27,7 @@ import com.fasterxml.jackson.annotation.JsonInclude;
 import com.fasterxml.jackson.annotation.JsonInclude.Include;
 import com.kingsrook.qqq.backend.core.actions.permissions.PermissionsHelper;
 import com.kingsrook.qqq.backend.core.model.actions.AbstractActionInput;
+import com.kingsrook.qqq.backend.core.model.metadata.dashboard.QWidgetMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.dashboard.QWidgetMetaDataInterface;
 import com.kingsrook.qqq.backend.core.model.metadata.dashboard.WidgetDropdownData;
 
@@ -50,6 +51,9 @@ public class QFrontendWidgetMetaData
    private final boolean                  storeDropdownSelections;
    private final List<WidgetDropdownData> dropdowns;
 
+   private boolean showReloadButton = false;
+   private boolean showExportButton = false;
+
    private final boolean hasPermission;
 
    //////////////////////////////////////////////////////////////////////////////////
@@ -74,6 +78,12 @@ public class QFrontendWidgetMetaData
       this.dropdowns = widgetMetaData.getDropdowns();
       this.storeDropdownSelections = widgetMetaData.getStoreDropdownSelections();
 
+      if(widgetMetaData instanceof QWidgetMetaData qWidgetMetaData)
+      {
+         this.showExportButton = qWidgetMetaData.getShowExportButton();
+         this.showReloadButton = qWidgetMetaData.getShowReloadButton();
+      }
+
       hasPermission = PermissionsHelper.hasWidgetPermission(actionInput, name);
    }
 
@@ -198,4 +208,25 @@ public class QFrontendWidgetMetaData
       return storeDropdownSelections;
    }
 
+
+
+   /*******************************************************************************
+    ** Getter for showReloadButton
+    **
+    *******************************************************************************/
+   public boolean getShowReloadButton()
+   {
+      return showReloadButton;
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for showExportButton
+    **
+    *******************************************************************************/
+   public boolean getShowExportButton()
+   {
+      return showExportButton;
+   }
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/processes/AbstractProcessMetaDataBuilder.java

@@ -70,6 +70,17 @@ public class AbstractProcessMetaDataBuilder
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public AbstractProcessMetaDataBuilder withInputFieldDefaultValue(String fieldName, Serializable value)
+   {
+      setInputFieldDefaultValue(fieldName, value);
+      return (this);
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/scheduleing/QScheduleMetaData.java

@@ -22,6 +22,9 @@
 package com.kingsrook.qqq.backend.core.model.metadata.scheduleing;
 
 
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
+
+
 /*******************************************************************************
  ** Meta-data to define scheduled actions within QQQ.
  **
@@ -33,11 +36,22 @@ package com.kingsrook.qqq.backend.core.model.metadata.scheduleing;
  *******************************************************************************/
 public class QScheduleMetaData
 {
+   public enum RunStrategy
+   {PARALLEL, SERIAL}
+
+
+
    private Integer repeatSeconds;
    private Integer repeatMillis;
    private Integer initialDelaySeconds;
    private Integer initialDelayMillis;
 
+   private RunStrategy  variantRunStrategy;
+   private String       backendVariant;
+   private String       variantTableName;
+   private QQueryFilter variantFilter;
+   private String       variantFieldName;
+
 
 
    /*******************************************************************************
@@ -174,4 +188,159 @@ public class QScheduleMetaData
       return (this);
    }
 
+
+
+   /*******************************************************************************
+    ** Getter for backendVariant
+    *******************************************************************************/
+   public String getBackendVariant()
+   {
+      return (this.backendVariant);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for backendVariant
+    *******************************************************************************/
+   public void setBackendVariant(String backendVariant)
+   {
+      this.backendVariant = backendVariant;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for backendVariant
+    *******************************************************************************/
+   public QScheduleMetaData withBackendVariant(String backendVariant)
+   {
+      this.backendVariant = backendVariant;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for variantTableName
+    *******************************************************************************/
+   public String getVariantTableName()
+   {
+      return (this.variantTableName);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for variantTableName
+    *******************************************************************************/
+   public void setVariantTableName(String variantTableName)
+   {
+      this.variantTableName = variantTableName;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for variantTableName
+    *******************************************************************************/
+   public QScheduleMetaData withVariantTableName(String variantTableName)
+   {
+      this.variantTableName = variantTableName;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for variantFilter
+    *******************************************************************************/
+   public QQueryFilter getVariantFilter()
+   {
+      return (this.variantFilter);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for variantFilter
+    *******************************************************************************/
+   public void setVariantFilter(QQueryFilter variantFilter)
+   {
+      this.variantFilter = variantFilter;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for variantFilter
+    *******************************************************************************/
+   public QScheduleMetaData withVariantFilter(QQueryFilter variantFilter)
+   {
+      this.variantFilter = variantFilter;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for variantFieldName
+    *******************************************************************************/
+   public String getVariantFieldName()
+   {
+      return (this.variantFieldName);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for variantFieldName
+    *******************************************************************************/
+   public void setVariantFieldName(String variantFieldName)
+   {
+      this.variantFieldName = variantFieldName;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for variantFieldName
+    *******************************************************************************/
+   public QScheduleMetaData withVariantFieldName(String variantFieldName)
+   {
+      this.variantFieldName = variantFieldName;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for variantRunStrategy
+    *******************************************************************************/
+   public RunStrategy getVariantRunStrategy()
+   {
+      return (this.variantRunStrategy);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for variantRunStrategy
+    *******************************************************************************/
+   public void setVariantRunStrategy(RunStrategy variantRunStrategy)
+   {
+      this.variantRunStrategy = variantRunStrategy;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for variantRunStrategy
+    *******************************************************************************/
+   public QScheduleMetaData withVariantRunStrategy(RunStrategy variantRunStrategy)
+   {
+      this.variantRunStrategy = variantRunStrategy;
+      return (this);
+   }
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/serialization/DeserializerUtils.java

@@ -37,6 +37,7 @@ import java.util.function.Consumer;
 import com.fasterxml.jackson.core.TreeNode;
 import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.node.ArrayNode;
+import com.fasterxml.jackson.databind.node.BooleanNode;
 import com.fasterxml.jackson.databind.node.NullNode;
 import com.fasterxml.jackson.databind.node.ObjectNode;
 import com.fasterxml.jackson.databind.node.TextNode;
@@ -283,6 +284,10 @@ public class DeserializerUtils
          {
             setterMap.get(fieldName).accept(textNode.asText());
          }
+         else if(fieldNode instanceof BooleanNode booleanNode)
+         {
+            setterMap.get(fieldName).accept(booleanNode);
+         }
          else if(fieldNode instanceof ObjectNode)
          {
             setterMap.get(fieldName).accept(fieldNode);