Merge branch 'release/0.13.0'

Bump json from 20220320 to 20230227 in /qqq-backend-core

.circleci/config.yml

@@ -1,7 +1,6 @@
 version: 2.1
 
 orbs:
-  slack: circleci/slack@4.10.1
   localstack: localstack/platform@1.0
 
 commands:
@@ -12,18 +11,25 @@ commands:
     steps:
       - store_artifacts:
           path: << parameters.module >>/target/site/jacoco/index.html
+          when: always
       - store_artifacts:
           path: << parameters.module >>/target/site/jacoco/jacoco-resources
+          when: always
 
   install_java17:
     steps:
       - run:
           name: Install Java 17
           command: |
-            sudo add-apt-repository -y ppa:openjdk-r/ppa
+            sudo apt-get update
             sudo apt install -y openjdk-17-jdk
             sudo rm /etc/alternatives/java
             sudo ln -s /usr/lib/jvm/java-17-openjdk-amd64/bin/java /etc/alternatives/java
+      - run:
+          name: Install html2text
+          command: |
+            sudo apt-get update
+            sudo apt-get install -y html2text
 
   mvn_verify:
     steps:
@@ -59,6 +65,11 @@ commands:
           when: always
       - store_test_results:
           path: ~/test-results
+      - run:
+          name: Find Un-tested Classes
+          command: |
+            set +o pipefail && for i in */target/site/jacoco/*/index.html; do html2text -width 500 -nobs $i | sed '1,/^Total/d;' | grep -v Created | sed 's/ \+/ /g' | sed 's/ [[:digit:]]$//' | grep -v 0$ | cut -d' ' -f1; done
+          when: always
       - save_cache:
           paths:
             - ~/.m2
@@ -86,8 +97,6 @@ jobs:
       - localstack/startup
       - install_java17
       - mvn_verify
-      - slack/notify:
-          event: fail
 
   mvn_deploy:
     executor: localstack/default
@@ -96,14 +105,12 @@ jobs:
       - install_java17
       - mvn_verify
       - mvn_jar_deploy
-      - slack/notify:
-          event: always
 
 workflows:
   test_only:
     jobs:
       - mvn_test:
-          context: [ qqq-maven-registry-credentials, kingsrook-slack, build-qqq-sample-app ]
+          context: [ qqq-maven-registry-credentials, build-qqq-sample-app ]
           filters:
             branches:
               ignore: /dev/
@@ -113,7 +120,7 @@ workflows:
   deploy:
     jobs:
       - mvn_deploy:
-          context: [ qqq-maven-registry-credentials, kingsrook-slack, build-qqq-sample-app ]
+          context: [ qqq-maven-registry-credentials, build-qqq-sample-app ]
           filters:
             branches:
               only: /dev/

.gitignore

@@ -2,6 +2,7 @@ target/
 *.iml
 .env
 
+.idea
 
 #############################################
 ## Original contents from github template: ##
@@ -29,6 +30,8 @@ target/
 
 # virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml
 hs_err_pid*
+
 .DS_Store
 *.swp
 .flattened-pom.xml
+dependency-reduced-pom.xml

README.md

@@ -35,3 +35,4 @@ GNU Affero General Public License for more details.
 
 You should have received a copy of the GNU Affero General Public License
 along with this program.  If not, see <https://www.gnu.org/licenses/>.
+

checkstyle/config.xml

@@ -179,9 +179,7 @@
          <property name="ignoreFinal" value="false"/>
          <property name="allowedAbbreviationLength" value="1"/>
       </module>
-      -->
       <module name="OverloadMethodsDeclarationOrder"/>
-      <!--
       <module name="VariableDeclarationUsageDistance"/>
       <module name="CustomImportOrder">
          <property name="sortImportsInGroupAlphabetically" value="true"/>
@@ -261,5 +259,10 @@
       <module name="MissingOverride"/>
 
    </module>
+   <module name="Header">
+     <property name="headerFile" value="checkstyle/license.txt"/>
+     <property name="fileExtensions" value="java"/>
+     <property name="ignoreLines" value="3"/>
+   </module>
    <module name="SuppressWarningsFilter"/>
 </module>

checkstyle/license.txt

@@ -0,0 +1,20 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */

pom.xml

@@ -30,15 +30,21 @@
 
    <modules>
       <module>qqq-backend-core</module>
-      <module>qqq-backend-module-rdbms</module>
+      <module>qqq-backend-module-api</module>
       <module>qqq-backend-module-filesystem</module>
+      <module>qqq-backend-module-rdbms</module>
+      <module>qqq-language-support-javascript</module>
       <module>qqq-middleware-picocli</module>
       <module>qqq-middleware-javalin</module>
+      <module>qqq-middleware-lambda</module>
+      <module>qqq-middleware-slack</module>
+      <module>qqq-middleware-api</module>
+      <module>qqq-utility-lambdas</module>
       <module>qqq-sample-project</module>
    </modules>
 
    <properties>
-      <revision>0.3.0</revision>
+      <revision>0.13.0</revision>
 
       <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
       <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
@@ -48,8 +54,21 @@
       <maven.compiler.showWarnings>true</maven.compiler.showWarnings>
       <coverage.haltOnFailure>true</coverage.haltOnFailure>
       <coverage.instructionCoveredRatioMinimum>0.80</coverage.instructionCoveredRatioMinimum>
+      <coverage.classCoveredRatioMinimum>0.95</coverage.classCoveredRatioMinimum>
+      <plugin.shade.phase>none</plugin.shade.phase>
    </properties>
 
+   <profiles>
+      <profile>
+         <!-- For qqq-middleware-lambda - to build its shaded jar, its qqq dependencies also need
+          to build a shaded jar.  So to activate that mode, use this profile (-P buildShadedJar)-->
+         <id>buildShadedJar</id>
+         <properties>
+            <plugin.shade.phase>package</plugin.shade.phase>
+         </properties>
+      </profile>
+   </profiles>
+
    <dependencyManagement>
       <dependencies>
          <dependency>
@@ -73,13 +92,19 @@
             <version>5.8.1</version>
             <scope>test</scope>
          </dependency>
+         <dependency>
+            <groupId>org.junit.jupiter</groupId>
+            <artifactId>junit-jupiter-params</artifactId>
+            <version>5.8.1</version>
+            <scope>test</scope>
+         </dependency>
          <dependency>
             <groupId>org.assertj</groupId>
             <artifactId>assertj-core</artifactId>
             <version>3.23.1</version>
             <scope>test</scope>
          </dependency>
-       </dependencies>
+      </dependencies>
    </dependencyManagement>
 
    <build>
@@ -121,7 +146,8 @@
                   <id>validate</id>
                   <phase>validate</phase>
                   <configuration>
-                     <configLocation>checkstyle.xml</configLocation>
+                     <configLocation>checkstyle/config.xml</configLocation>
+                     <headerLocation>checkstyle/license.txt</headerLocation>
                      <!-- <suppressionsLocation>checkstyle-suppressions.xml</suppressionsLocation> -->
                      <encoding>UTF-8</encoding>
                      <consoleOutput>true</consoleOutput>
@@ -211,6 +237,11 @@
                                  <value>COVEREDRATIO</value>
                                  <minimum>${coverage.instructionCoveredRatioMinimum}</minimum>
                               </limit>
+                              <limit>
+                                 <counter>CLASS</counter>
+                                 <value>COVEREDRATIO</value>
+                                 <minimum>${coverage.classCoveredRatioMinimum}</minimum>
+                              </limit>
                            </limits>
                         </rule>
                      </rules>
@@ -255,7 +286,7 @@ echo "------------------------------------------------------------"
 which xpath > /dev/null 2>&1
 if [ "$?" == "0" ]; then
    echo "Element\nInstructions Missed\nInstruction Coverage\nBranches Missed\nBranch Coverage\nComplexity Missed\nComplexity Hit\nLines Missed\nLines Hit\nMethods Missed\nMethods Hit\nClasses Missed\nClasses Hit\n" > /tmp/$$.headers
-   xpath -q -e '/html/body/table/tfoot/tr[1]/td/text()' target/site/jacoco/index.html > /tmp/$$.values
+   xpath -n -q -e '/html/body/table/tfoot/tr[1]/td/text()' target/site/jacoco/index.html > /tmp/$$.values
    paste /tmp/$$.headers /tmp/$$.values | tail +2 | awk -v FS='\t' '{printf("%-20s %s\n",$1,$2)}'
    rm /tmp/$$.headers /tmp/$$.values
 else

qqq-backend-core/pom.xml

@@ -36,25 +36,51 @@
       <!-- noe at this time -->
    </properties>
 
+   <dependencyManagement>
+      <dependencies>
+         <dependency>
+            <groupId>software.amazon.awssdk</groupId>
+            <artifactId>bom</artifactId>
+            <version>2.17.259</version>
+            <type>pom</type>
+            <scope>import</scope>
+         </dependency>
+      </dependencies>
+   </dependencyManagement>
    <dependencies>
       <!-- other qqq modules deps -->
       <!-- none, this is core.  -->
 
       <!-- 3rd party deps specifically for this module -->
+      <dependency>
+         <groupId>software.amazon.awssdk</groupId>
+         <artifactId>quicksight</artifactId>
+      </dependency>
+      <dependency>
+         <groupId>com.amazonaws</groupId>
+         <artifactId>aws-java-sdk-secretsmanager</artifactId>
+         <version>1.12.385</version>
+      </dependency>
+
       <dependency>
          <groupId>com.fasterxml.jackson.core</groupId>
          <artifactId>jackson-databind</artifactId>
-         <version>2.13.2.1</version>
+         <version>2.14.0</version>
       </dependency>
       <dependency>
          <groupId>com.fasterxml.jackson.datatype</groupId>
          <artifactId>jackson-datatype-jsr310</artifactId>
-         <version>2.13.0</version>
+         <version>2.14.0</version>
+      </dependency>
+      <dependency>
+         <groupId>com.fasterxml.jackson.dataformat</groupId>
+         <artifactId>jackson-dataformat-yaml</artifactId>
+         <version>2.14.0</version>
       </dependency>
       <dependency>
          <groupId>org.json</groupId>
          <artifactId>json</artifactId>
-         <version>20210307</version>
+         <version>20230227</version>
       </dependency>
       <dependency>
          <groupId>org.apache.commons</groupId>
@@ -82,6 +108,46 @@
          <artifactId>java-dotenv</artifactId>
          <version>5.2.2</version>
       </dependency>
+      <dependency>
+         <groupId>org.apache.velocity</groupId>
+         <artifactId>velocity-engine-core</artifactId>
+         <version>2.3</version>
+      </dependency>
+
+      <!-- the next 2 deps are for html to pdf - per https://www.baeldung.com/java-html-to-pdf -->
+      <dependency>
+         <groupId>org.jsoup</groupId>
+         <artifactId>jsoup</artifactId>
+         <version>1.15.3</version>
+      </dependency>
+      <dependency>
+         <groupId>org.xhtmlrenderer</groupId>
+         <artifactId>flying-saucer-pdf-openpdf</artifactId>
+         <version>9.1.22</version>
+      </dependency>
+
+      <!-- the next 3 deps are being added for google drive support -->
+      <dependency>
+         <groupId>com.google.api-client</groupId>
+         <artifactId>google-api-client</artifactId>
+         <version>1.35.2</version>
+      </dependency>
+      <dependency>
+         <groupId>com.google.auth</groupId>
+         <artifactId>google-auth-library-oauth2-http</artifactId>
+         <version>1.11.0</version>
+      </dependency>
+      <dependency>
+         <groupId>com.google.apis</groupId>
+         <artifactId>google-api-services-drive</artifactId>
+         <version>v3-rev20220815-2.0.0</version>
+      </dependency>
+
+      <dependency>
+         <groupId>com.amazonaws</groupId>
+         <artifactId>aws-java-sdk-sqs</artifactId>
+         <version>1.12.321</version>
+      </dependency>
 
       <!-- Common deps for all qqq modules -->
       <dependency>
@@ -101,12 +167,24 @@
          <artifactId>junit-jupiter-engine</artifactId>
          <scope>test</scope>
       </dependency>
+      <dependency>
+         <groupId>org.junit.jupiter</groupId>
+         <artifactId>junit-jupiter-params</artifactId>
+         <scope>test</scope>
+      </dependency>
       <dependency>
          <groupId>org.assertj</groupId>
          <artifactId>assertj-core</artifactId>
          <scope>test</scope>
       </dependency>
 
+      <dependency>
+         <groupId>org.mockito</groupId>
+         <artifactId>mockito-core</artifactId>
+         <version>4.8.1</version>
+         <scope>test</scope>
+      </dependency>
+
    </dependencies>
 
    <build>
@@ -121,6 +199,30 @@
                </excludes>
             </configuration>
          </plugin>
+         <plugin>
+            <groupId>org.apache.maven.plugins</groupId>
+            <artifactId>maven-shade-plugin</artifactId>
+            <version>2.4.3</version>
+            <configuration>
+               <createDependencyReducedPom>false</createDependencyReducedPom>
+               <filters>
+                  <filter>
+                     <artifact>*:*</artifact>
+                     <excludes>
+                        <exclude>META-INF/*</exclude>
+                     </excludes>
+                  </filter>
+               </filters>
+            </configuration>
+            <executions>
+               <execution>
+                  <phase>${plugin.shade.phase}</phase>
+                  <goals>
+                     <goal>shade</goal>
+                  </goals>
+               </execution>
+            </executions>
+         </plugin>
       </plugins>
    </build>
 

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/AbstractQActionBiConsumer.java

@@ -0,0 +1,77 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.actions;
+
+
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import com.kingsrook.qqq.backend.core.context.CapturedContext;
+import com.kingsrook.qqq.backend.core.context.QContext;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.actions.AbstractActionInput;
+import com.kingsrook.qqq.backend.core.model.actions.AbstractActionOutput;
+
+
+/*******************************************************************************
+ ** Base class for QQQ Actions (both framework and application defined) that
+ ** have a signature like a BiConsumer - taking both Input and Output objects as
+ ** parameters, with void output.
+ *******************************************************************************/
+public abstract class AbstractQActionBiConsumer<I extends AbstractActionInput, O extends AbstractActionOutput>
+{
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public abstract void execute(I input, O output) throws QException;
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public Future<Void> executeAsync(I input, O output)
+   {
+      CapturedContext         capturedContext   = QContext.capture();
+      CompletableFuture<Void> completableFuture = new CompletableFuture<>();
+      Executors.newCachedThreadPool().submit(() ->
+      {
+         try
+         {
+            QContext.init(capturedContext);
+            execute(input, output);
+            completableFuture.complete(null);
+         }
+         catch(QException e)
+         {
+            completableFuture.completeExceptionally(e);
+         }
+         finally
+         {
+            QContext.clear();
+         }
+      });
+      return (completableFuture);
+   }
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/AbstractQActionFunction.java

@@ -0,0 +1,77 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.actions;
+
+
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import com.kingsrook.qqq.backend.core.context.CapturedContext;
+import com.kingsrook.qqq.backend.core.context.QContext;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.actions.AbstractActionInput;
+import com.kingsrook.qqq.backend.core.model.actions.AbstractActionOutput;
+
+
+/*******************************************************************************
+ ** Base class for QQQ Actions (both framework and application defined) that
+ ** have a signature like a Function - taking an Input object as a parameter,
+ ** and returning an Output object.
+ *******************************************************************************/
+public abstract class AbstractQActionFunction<I extends AbstractActionInput, O extends AbstractActionOutput>
+{
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public abstract O execute(I input) throws QException;
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public Future<O> executeAsync(I input)
+   {
+      CapturedContext      capturedContext   = QContext.capture();
+      CompletableFuture<O> completableFuture = new CompletableFuture<>();
+      Executors.newCachedThreadPool().submit(() ->
+      {
+         try
+         {
+            QContext.init(capturedContext);
+            O output = execute(input);
+            completableFuture.complete(output);
+         }
+         catch(QException e)
+         {
+            completableFuture.completeExceptionally(e);
+         }
+         finally
+         {
+            QContext.clear();
+         }
+      });
+      return (completableFuture);
+   }
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/ActionHelper.java

@@ -22,9 +22,15 @@
 package com.kingsrook.qqq.backend.core.actions;
 
 
+import java.io.Serializable;
+import java.util.List;
+import java.util.function.Function;
+import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.exceptions.QAuthenticationException;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.model.actions.AbstractActionInput;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.session.QSession;
 import com.kingsrook.qqq.backend.core.modules.authentication.QAuthenticationModuleDispatcher;
 import com.kingsrook.qqq.backend.core.modules.authentication.QAuthenticationModuleInterface;
 
@@ -40,12 +46,37 @@ public class ActionHelper
     *******************************************************************************/
    public static void validateSession(AbstractActionInput request) throws QException
    {
+      QInstance qInstance = QContext.getQInstance();
+      QSession  qSession  = QContext.getQSession();
+
+      if(qInstance == null)
+      {
+         throw (new QException("QInstance was not set in QContext."));
+      }
+
+      if(qSession == null)
+      {
+         throw (new QException("QSession was not set in QContext."));
+      }
+
       QAuthenticationModuleDispatcher qAuthenticationModuleDispatcher = new QAuthenticationModuleDispatcher();
-      QAuthenticationModuleInterface authenticationModule = qAuthenticationModuleDispatcher.getQModule(request.getAuthenticationMetaData());
-      if(!authenticationModule.isSessionValid(request.getInstance(), request.getSession()))
+      QAuthenticationModuleInterface  authenticationModule            = qAuthenticationModuleDispatcher.getQModule(qInstance.getAuthentication());
+      if(!authenticationModule.isSessionValid(qInstance, qSession))
       {
          throw new QAuthenticationException("Invalid session in request");
       }
    }
 
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public static void editFirstValue(List<Serializable> values, Function<String, String> editFunction)
+   {
+      if(values.size() > 0)
+      {
+         values.set(0, editFunction.apply(String.valueOf(values.get(0))));
+      }
+   }
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/async/AsyncJobCallback.java

@@ -31,7 +31,6 @@ import com.kingsrook.qqq.backend.core.state.UUIDAndTypeStateKey;
  ** Argument passed to an AsyncJob when it runs, which can be used to communicate
  ** data back out of the job.
  **
- ** TODO - future - allow cancellation to be indicated here?
  *******************************************************************************/
 public class AsyncJobCallback
 {
@@ -51,6 +50,17 @@ public class AsyncJobCallback
 
 
 
+   /*******************************************************************************
+    ** Setter for asyncJobStatus
+    **
+    *******************************************************************************/
+   public void setAsyncJobStatus(AsyncJobStatus asyncJobStatus)
+   {
+      this.asyncJobStatus = asyncJobStatus;
+   }
+
+
+
    /*******************************************************************************
     ** Update the message
     *******************************************************************************/
@@ -68,9 +78,7 @@ public class AsyncJobCallback
    public void updateStatus(String message, int current, int total)
    {
       this.asyncJobStatus.setMessage(message);
-      this.asyncJobStatus.setCurrent(current);
-      this.asyncJobStatus.setTotal(total);
-      storeUpdatedStatus();
+      updateStatus(current, total); // this call will storeUpdatedStatus.
    }
 
 
@@ -80,13 +88,65 @@ public class AsyncJobCallback
     *******************************************************************************/
    public void updateStatus(int current, int total)
    {
-      this.asyncJobStatus.setCurrent(current);
+      this.asyncJobStatus.setCurrent(current > total ? total : current);
       this.asyncJobStatus.setTotal(total);
       storeUpdatedStatus();
    }
 
 
 
+   /*******************************************************************************
+    ** Update the current and total fields, but ONLY if the new values are
+    ** both >= the previous values.
+    *******************************************************************************/
+   public void updateStatusOnlyUpwards(int current, int total)
+   {
+      boolean currentIsOkay = (this.asyncJobStatus.getCurrent() == null || this.asyncJobStatus.getCurrent() <= current);
+      boolean totalIsOkay   = (this.asyncJobStatus.getTotal() == null || this.asyncJobStatus.getTotal() <= total);
+
+      if(currentIsOkay && totalIsOkay)
+      {
+         updateStatus(current, total);
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** Increase the 'current' value in the '1 of 2' sense.
+    *******************************************************************************/
+   public void incrementCurrent()
+   {
+      incrementCurrent(1);
+   }
+
+
+
+   /*******************************************************************************
+    ** Increase the 'current' value in the '1 of 2' sense.
+    *******************************************************************************/
+   public void incrementCurrent(int amount)
+   {
+      if(asyncJobStatus.getCurrent() != null)
+      {
+         if(asyncJobStatus.getTotal() != null && asyncJobStatus.getCurrent() + amount > asyncJobStatus.getTotal())
+         {
+            /////////////////////////////////////////////////////
+            // make sure we don't ever make current > total... //
+            /////////////////////////////////////////////////////
+            asyncJobStatus.setCurrent(asyncJobStatus.getTotal());
+         }
+         else
+         {
+            asyncJobStatus.setCurrent(asyncJobStatus.getCurrent() + amount);
+         }
+
+         storeUpdatedStatus();
+      }
+   }
+
+
+
    /*******************************************************************************
     ** Remove the values from the current & total fields
     *******************************************************************************/
@@ -107,4 +167,17 @@ public class AsyncJobCallback
       AsyncJobManager.getStateProvider().put(new UUIDAndTypeStateKey(jobUUID, StateType.ASYNC_JOB_STATUS), asyncJobStatus);
    }
 
+
+
+   /*******************************************************************************
+    ** Check if the asyncJobStatus had a cancellation requested.
+    **
+    ** TODO - concern about multiple threads writing this object to a non-in-memory
+    **  state provider, and this value getting lost...
+    *******************************************************************************/
+   public boolean wasCancelRequested()
+   {
+      return (this.asyncJobStatus.getCancelRequested());
+   }
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/async/AsyncJobManager.java

@@ -30,13 +30,15 @@ import java.util.concurrent.CompletionException;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.TimeoutException;
+import com.kingsrook.qqq.backend.core.context.CapturedContext;
+import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.logging.QLogger;
 import com.kingsrook.qqq.backend.core.state.InMemoryStateProvider;
 import com.kingsrook.qqq.backend.core.state.StateProviderInterface;
 import com.kingsrook.qqq.backend.core.state.StateType;
 import com.kingsrook.qqq.backend.core.state.UUIDAndTypeStateKey;
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
+import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
 
 
 /*******************************************************************************
@@ -44,7 +46,7 @@ import org.apache.logging.log4j.Logger;
  *******************************************************************************/
 public class AsyncJobManager
 {
-   private static final Logger LOG = LogManager.getLogger(AsyncJobManager.class);
+   private static final QLogger LOG = QLogger.getLogger(AsyncJobManager.class);
 
 
 
@@ -72,8 +74,10 @@ public class AsyncJobManager
 
       try
       {
+         CapturedContext capturedContext = QContext.capture();
          CompletableFuture<T> future = CompletableFuture.supplyAsync(() ->
          {
+            QContext.init(capturedContext);
             return (runAsyncJob(jobName, asyncJob, uuidAndTypeStateKey, asyncJobStatus));
          });
 
@@ -91,7 +95,7 @@ public class AsyncJobManager
       }
       catch(TimeoutException e)
       {
-         LOG.info("Job going async " + uuidAndTypeStateKey.getUuid());
+         LOG.debug("Job going async " + uuidAndTypeStateKey.getUuid());
          throw (new JobGoingAsyncException(uuidAndTypeStateKey.getUuid().toString()));
       }
    }
@@ -135,11 +139,11 @@ public class AsyncJobManager
       Thread.currentThread().setName("Job:" + jobName + ":" + uuidAndTypeStateKey.getUuid().toString().substring(0, 8));
       try
       {
-         LOG.info("Starting job " + uuidAndTypeStateKey.getUuid());
+         LOG.debug("Starting job " + uuidAndTypeStateKey.getUuid());
          T result = asyncJob.run(new AsyncJobCallback(uuidAndTypeStateKey.getUuid(), asyncJobStatus));
          asyncJobStatus.setState(AsyncJobState.COMPLETE);
          getStateProvider().put(uuidAndTypeStateKey, asyncJobStatus);
-         LOG.info("Completed job " + uuidAndTypeStateKey.getUuid());
+         LOG.debug("Completed job " + uuidAndTypeStateKey.getUuid());
          return (result);
       }
       catch(Exception e)
@@ -147,12 +151,13 @@ public class AsyncJobManager
          asyncJobStatus.setState(AsyncJobState.ERROR);
          asyncJobStatus.setCaughtException(e);
          getStateProvider().put(uuidAndTypeStateKey, asyncJobStatus);
-         LOG.warn("Job " + uuidAndTypeStateKey.getUuid() + " ended with an exception: ", e);
+         LOG.warn("Job ended with an exception", e, logPair("jobId", uuidAndTypeStateKey.getUuid()));
          throw (new CompletionException(e));
       }
       finally
       {
          Thread.currentThread().setName(originalThreadName);
+         QContext.clear();
       }
    }
 
@@ -183,4 +188,15 @@ public class AsyncJobManager
       // return TempFileStateProvider.getInstance();
    }
 
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public void cancelJob(String jobUUID)
+   {
+      Optional<AsyncJobStatus> jobStatus = getJobStatus(jobUUID);
+      jobStatus.ifPresent(asyncJobStatus -> asyncJobStatus.setCancelRequested(true));
+   }
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/async/AsyncJobStatus.java

@@ -37,6 +37,8 @@ public class AsyncJobStatus implements Serializable
    private Integer       total;
    private Exception     caughtException;
 
+   private boolean cancelRequested;
+
 
 
    /*******************************************************************************
@@ -163,4 +165,26 @@ public class AsyncJobStatus implements Serializable
    {
       this.caughtException = caughtException;
    }
+
+
+
+   /*******************************************************************************
+    ** Getter for cancelRequested
+    **
+    *******************************************************************************/
+   public boolean getCancelRequested()
+   {
+      return cancelRequested;
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for cancelRequested
+    **
+    *******************************************************************************/
+   public void setCancelRequested(boolean cancelRequested)
+   {
+      this.cancelRequested = cancelRequested;
+   }
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/async/AsyncRecordPipeLoop.java

@@ -0,0 +1,211 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.actions.async;
+
+
+import java.io.Serializable;
+import java.util.Optional;
+import java.util.concurrent.TimeUnit;
+import com.kingsrook.qqq.backend.core.actions.reporting.RecordPipe;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.logging.QLogger;
+import com.kingsrook.qqq.backend.core.utils.SleepUtils;
+import com.kingsrook.qqq.backend.core.utils.lambdas.UnsafeFunction;
+import com.kingsrook.qqq.backend.core.utils.lambdas.UnsafeSupplier;
+
+
+/*******************************************************************************
+ ** Class that knows how to Run an asynchronous job (lambda, supplier) that writes into a
+ ** RecordPipe, with another lambda (consumer) that consumes records from the pipe.
+ **
+ ** Takes care of the job status monitoring, blocking when the pipe is empty, etc.
+ *******************************************************************************/
+public class AsyncRecordPipeLoop
+{
+   private static final QLogger LOG = QLogger.getLogger(AsyncRecordPipeLoop.class);
+
+   private static final int TIMEOUT_AFTER_NO_RECORDS_MS = 10 * 60 * 1000;
+
+   private static final int MAX_SLEEP_MS  = 1000;
+   private static final int INIT_SLEEP_MS = 10;
+
+   private Integer minRecordsToConsume = 10;
+
+
+
+   /*******************************************************************************
+    ** Run an async-record-pipe-loop.
+    **
+    ** @param jobName name for the async job thread
+    ** @param recordLimit optionally, cancel the supplier/job after this number of records.
+    *                     e.g., for a preview step.
+    ** @param recordPipe constructed before this call, and used in both of the lambdas
+    ** @param supplier lambda that adds records into the pipe.
+    *                  e.g., a query or extract step.
+    ** @param consumer lambda that consumes records from the pipe
+    *                  e.g., a transform/load step.
+    *******************************************************************************/
+   public int run(String jobName, Integer recordLimit, RecordPipe recordPipe, UnsafeFunction<AsyncJobCallback, ? extends Serializable, QException> supplier, UnsafeSupplier<Integer, QException> consumer) throws QException
+   {
+      ///////////////////////////////////////////////////
+      // start the extraction function as an async job //
+      ///////////////////////////////////////////////////
+      AsyncJobManager asyncJobManager = new AsyncJobManager();
+      String          jobUUID         = asyncJobManager.startJob(jobName, supplier::apply);
+      LOG.debug("Started supplier job [" + jobUUID + "] for record pipe.");
+
+      AsyncJobState  jobState       = AsyncJobState.RUNNING;
+      AsyncJobStatus asyncJobStatus = null;
+
+      int     recordCount           = 0;
+      int     nextSleepMillis       = INIT_SLEEP_MS;
+      long    lastReceivedRecordsAt = System.currentTimeMillis();
+      long    jobStartTime          = System.currentTimeMillis();
+      boolean everCalledConsumer    = false;
+
+      while(jobState.equals(AsyncJobState.RUNNING))
+      {
+         if(recordPipe.countAvailableRecords() < minRecordsToConsume)
+         {
+            ///////////////////////////////////////////////////////////////
+            // if the pipe is too empty, sleep to let the producer work. //
+            // todo - smarter sleep?  like get notified vs. sleep?       //
+            ///////////////////////////////////////////////////////////////
+            LOG.trace("Too few records are available in the pipe. Sleeping [" + nextSleepMillis + "] ms to give producer a chance to work");
+            SleepUtils.sleep(nextSleepMillis, TimeUnit.MILLISECONDS);
+            nextSleepMillis = Math.min(nextSleepMillis * 2, MAX_SLEEP_MS);
+
+            long timeSinceLastReceivedRecord = System.currentTimeMillis() - lastReceivedRecordsAt;
+            if(timeSinceLastReceivedRecord > TIMEOUT_AFTER_NO_RECORDS_MS)
+            {
+               throw (new QException("Job appears to have stopped producing records (last record received " + timeSinceLastReceivedRecord + " ms ago)."));
+            }
+         }
+         else
+         {
+            ////////////////////////////////////////////////////////////////////////////////////////////////////////
+            // if the pipe has records, consume them.  reset the sleep timer so if we sleep again it'll be short. //
+            ////////////////////////////////////////////////////////////////////////////////////////////////////////
+            lastReceivedRecordsAt = System.currentTimeMillis();
+            nextSleepMillis = INIT_SLEEP_MS;
+
+            everCalledConsumer = true;
+            recordCount += consumer.get();
+            LOG.debug(String.format("Processed %,d records so far", recordCount));
+
+            if(recordLimit != null && recordCount >= recordLimit)
+            {
+               asyncJobManager.cancelJob(jobUUID);
+
+               ////////////////////////////////////////////////////////////////////////////////////////////////////
+               // in case the extract function doesn't recognize the cancellation request,                       //
+               // tell the pipe to "terminate" - meaning - flush its queue and just noop when given new records. //
+               // this should prevent anyone writing to such a pipe from potentially filling & blocking.         //
+               ////////////////////////////////////////////////////////////////////////////////////////////////////
+               recordPipe.terminate();
+
+               break;
+            }
+         }
+
+         //////////////////////////////
+         // refresh the job's status //
+         //////////////////////////////
+         Optional<AsyncJobStatus> optionalAsyncJobStatus = asyncJobManager.getJobStatus(jobUUID);
+         if(optionalAsyncJobStatus.isEmpty())
+         {
+            /////////////////////////////////////////////////
+            // todo - ... maybe some version of try-again? //
+            /////////////////////////////////////////////////
+            throw (new QException("Could not get status of job [" + jobUUID + "]"));
+         }
+         asyncJobStatus = optionalAsyncJobStatus.get();
+         jobState = asyncJobStatus.getState();
+      }
+
+      LOG.debug("Job [" + jobUUID + "][" + jobName + "] completed with status: " + asyncJobStatus);
+
+      ///////////////////////////////////
+      // propagate errors from the job //
+      ///////////////////////////////////
+      if(asyncJobStatus != null && asyncJobStatus.getState().equals(AsyncJobState.ERROR))
+      {
+         throw (new QException("Job failed with an error", asyncJobStatus.getCaughtException()));
+      }
+
+      ///////////////////////////////////////////////////////////////////////////////////////////
+      // send the final records to the consumer                                                //
+      // note - we'll only make this "final" call to the consumer if:                          //
+      // - there are currently records in the pipe                                             //
+      // - OR we never called the consumer (e.g., there were 0 rows produced by the supplier   //
+      // This prevents cases where a consumer may get pages of records in the loop, but then   //
+      // be called here post-loop w/ 0 records, and may interpret it as a sign that no records //
+      // were ever supplied.                                                                   //
+      ///////////////////////////////////////////////////////////////////////////////////////////
+      if(recordPipe.countAvailableRecords() > 0 || !everCalledConsumer)
+      {
+         recordCount += consumer.get();
+      }
+
+      long endTime = System.currentTimeMillis();
+
+      if(recordCount > 0)
+      {
+         LOG.info(String.format("Processed %,d records", recordCount)
+            + String.format(" at end of job [%s] in %,d ms (%.2f records/second).", jobName, (endTime - jobStartTime), 1000d * (recordCount / (.001d + (endTime - jobStartTime)))));
+      }
+
+      return (recordCount);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for minRecordsToConsume
+    *******************************************************************************/
+   public Integer getMinRecordsToConsume()
+   {
+      return (this.minRecordsToConsume);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for minRecordsToConsume
+    *******************************************************************************/
+   public void setMinRecordsToConsume(Integer minRecordsToConsume)
+   {
+      this.minRecordsToConsume = minRecordsToConsume;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for minRecordsToConsume
+    *******************************************************************************/
+   public AsyncRecordPipeLoop withMinRecordsToConsume(Integer minRecordsToConsume)
+   {
+      this.minRecordsToConsume = minRecordsToConsume;
+      return (this);
+   }
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/audits/AuditAction.java

@@ -0,0 +1,333 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2023.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.actions.audits;
+
+
+import java.io.Serializable;
+import java.time.Instant;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import com.kingsrook.qqq.backend.core.actions.AbstractQActionFunction;
+import com.kingsrook.qqq.backend.core.actions.tables.GetAction;
+import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
+import com.kingsrook.qqq.backend.core.context.QContext;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.logging.QLogger;
+import com.kingsrook.qqq.backend.core.model.actions.audits.AuditInput;
+import com.kingsrook.qqq.backend.core.model.actions.audits.AuditOutput;
+import com.kingsrook.qqq.backend.core.model.actions.audits.AuditSingleInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetOutput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertOutput;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.metadata.security.RecordSecurityLock;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+import com.kingsrook.qqq.backend.core.model.session.QUser;
+import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
+import com.kingsrook.qqq.backend.core.utils.Pair;
+
+
+/*******************************************************************************
+ ** Insert 1 or more audits (and optionally their children, auditDetails)
+ **
+ ** Takes care of managing the foreign key tables (auditTable, auditUser).
+ **
+ ** Enforces that security key values are provided, if the table has any.  Note that
+ ** might mean a null is given for a particular key, but at least the key must be present.
+ *******************************************************************************/
+public class AuditAction extends AbstractQActionFunction<AuditInput, AuditOutput>
+{
+   private static final QLogger LOG = QLogger.getLogger(AuditAction.class);
+
+   private Map<Pair<String, String>, Integer> cachedFetches = new HashMap<>();
+
+
+
+   /*******************************************************************************
+    ** Execute to insert 1 audit, with no details (child records)
+    *******************************************************************************/
+   public static void execute(String tableName, Integer recordId, Map<String, Serializable> securityKeyValues, String message)
+   {
+      execute(tableName, recordId, securityKeyValues, message, null);
+   }
+
+
+
+   /*******************************************************************************
+    ** Execute to insert 1 audit, with a list of detail child records
+    *******************************************************************************/
+   public static void execute(String tableName, Integer recordId, Map<String, Serializable> securityKeyValues, String message, List<QRecord> details)
+   {
+      new AuditAction().execute(new AuditInput().withAuditSingleInput(new AuditSingleInput()
+         .withAuditTableName(tableName)
+         .withRecordId(recordId)
+         .withSecurityKeyValues(securityKeyValues)
+         .withMessage(message)
+         .withDetails(details)
+      ));
+   }
+
+
+
+   /*******************************************************************************
+    ** Add 1 auditSingleInput to an AuditInput object - with no details (child records).
+    *******************************************************************************/
+   public static AuditInput appendToInput(AuditInput auditInput, String tableName, Integer recordId, Map<String, Serializable> securityKeyValues, String message)
+   {
+      return (appendToInput(auditInput, tableName, recordId, securityKeyValues, message, null));
+   }
+
+
+
+   /*******************************************************************************
+    ** Add 1 auditSingleInput to an AuditInput object - with a list of details (child records).
+    *******************************************************************************/
+   public static AuditInput appendToInput(AuditInput auditInput, String tableName, Integer recordId, Map<String, Serializable> securityKeyValues, String message, List<QRecord> details)
+   {
+      if(auditInput == null)
+      {
+         auditInput = new AuditInput();
+      }
+
+      return auditInput.withAuditSingleInput(new AuditSingleInput()
+         .withAuditTableName(tableName)
+         .withRecordId(recordId)
+         .withSecurityKeyValues(securityKeyValues)
+         .withMessage(message)
+         .withDetails(details)
+      );
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public AuditOutput execute(AuditInput input)
+   {
+      AuditOutput auditOutput = new AuditOutput();
+
+      if(CollectionUtils.nullSafeHasContents(input.getAuditSingleInputList()))
+      {
+         try
+         {
+            List<QRecord> auditRecords = new ArrayList<>();
+
+            for(AuditSingleInput auditSingleInput : CollectionUtils.nonNullList(input.getAuditSingleInputList()))
+            {
+               /////////////////////////////////////////
+               // validate table is known in instance //
+               /////////////////////////////////////////
+               QTableMetaData table = QContext.getQInstance().getTable(auditSingleInput.getAuditTableName());
+               if(table == null)
+               {
+                  throw (new QException("Requested audit for an unrecognized table name: " + auditSingleInput.getAuditTableName()));
+               }
+
+               ///////////////////////////////////////////////////
+               // validate security keys on the table are given //
+               ///////////////////////////////////////////////////
+               for(RecordSecurityLock recordSecurityLock : CollectionUtils.nonNullList(table.getRecordSecurityLocks()))
+               {
+                  if(auditSingleInput.getSecurityKeyValues() == null || !auditSingleInput.getSecurityKeyValues().containsKey(recordSecurityLock.getSecurityKeyType()))
+                  {
+                     throw (new QException("Missing securityKeyValue [" + recordSecurityLock.getSecurityKeyType() + "] in audit request for table " + auditSingleInput.getAuditTableName()));
+                  }
+               }
+
+               ////////////////////////////////////////////////
+               // map names to ids and handle default values //
+               ////////////////////////////////////////////////
+               Integer auditTableId = getIdForName("auditTable", auditSingleInput.getAuditTableName());
+               Integer auditUserId  = getIdForName("auditUser", Objects.requireNonNullElse(auditSingleInput.getAuditUserName(), getSessionUserName()));
+               Instant timestamp    = Objects.requireNonNullElse(auditSingleInput.getTimestamp(), Instant.now());
+
+               //////////////////
+               // build record //
+               //////////////////
+               QRecord record = new QRecord()
+                  .withValue("auditTableId", auditTableId)
+                  .withValue("auditUserId", auditUserId)
+                  .withValue("timestamp", timestamp)
+                  .withValue("message", auditSingleInput.getMessage())
+                  .withValue("recordId", auditSingleInput.getRecordId());
+
+               if(auditSingleInput.getSecurityKeyValues() != null)
+               {
+                  for(Map.Entry<String, Serializable> entry : auditSingleInput.getSecurityKeyValues().entrySet())
+                  {
+                     record.setValue(entry.getKey(), entry.getValue());
+                  }
+               }
+
+               auditRecords.add(record);
+            }
+
+            /////////////////////////////
+            // do a single bulk insert //
+            /////////////////////////////
+            InsertInput insertInput = new InsertInput();
+            insertInput.setTableName("audit");
+            insertInput.setRecords(auditRecords);
+            InsertOutput insertOutput = new InsertAction().execute(insertInput);
+
+            //////////////////////////////////////////
+            // now look for children (auditDetails) //
+            //////////////////////////////////////////
+            int           i                  = 0;
+            List<QRecord> auditDetailRecords = new ArrayList<>();
+            for(AuditSingleInput auditSingleInput : CollectionUtils.nonNullList(input.getAuditSingleInputList()))
+            {
+               Integer auditId = insertOutput.getRecords().get(i++).getValueInteger("id");
+               if(auditId == null)
+               {
+                  LOG.warn("Missing an id for inserted audit - so won't be able to store its child details...");
+                  continue;
+               }
+
+               for(QRecord detail : CollectionUtils.nonNullList(auditSingleInput.getDetails()))
+               {
+                  auditDetailRecords.add(detail.withValue("auditId", auditId));
+               }
+            }
+
+            insertInput = new InsertInput();
+            insertInput.setTableName("auditDetail");
+            insertInput.setRecords(auditDetailRecords);
+            new InsertAction().execute(insertInput);
+         }
+         catch(Exception e)
+         {
+            LOG.error("Error performing an audit", e);
+         }
+      }
+
+      return (auditOutput);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private static String getSessionUserName()
+   {
+      QUser user = QContext.getQSession().getUser();
+      if(user == null)
+      {
+         return ("Unknown");
+      }
+      return (user.getFullName());
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private Integer getIdForName(String tableName, String nameValue) throws QException
+   {
+      Pair<String, String> key = new Pair<>(tableName, nameValue);
+      if(!cachedFetches.containsKey(key))
+      {
+
+         Integer id = fetchIdFromName(tableName, nameValue);
+         if(id != null)
+         {
+            cachedFetches.put(key, id);
+            return id;
+         }
+
+         try
+         {
+            LOG.debug("Inserting " + tableName + " named " + nameValue);
+            InsertInput insertInput = new InsertInput();
+            insertInput.setTableName(tableName);
+            QRecord record = new QRecord().withValue("name", nameValue);
+
+            if(tableName.equals("auditTable"))
+            {
+               QTableMetaData table = QContext.getQInstance().getTable(nameValue);
+               if(table != null)
+               {
+                  record.setValue("label", table.getLabel());
+               }
+            }
+
+            insertInput.setRecords(List.of(record));
+            InsertOutput insertOutput = new InsertAction().execute(insertInput);
+            id = insertOutput.getRecords().get(0).getValueInteger("id");
+            if(id != null)
+            {
+               cachedFetches.put(key, id);
+               return id;
+            }
+         }
+         catch(Exception e)
+         {
+            ////////////////////////////////////////////////////////////////////
+            // assume this may mean a dupe-key - so - try another fetch below //
+            ////////////////////////////////////////////////////////////////////
+            LOG.debug("Caught error inserting " + tableName + " named " + nameValue + " - will try to re-fetch", e);
+         }
+
+         id = fetchIdFromName(tableName, nameValue);
+         if(id != null)
+         {
+            cachedFetches.put(key, id);
+            return id;
+         }
+
+         /////////////
+         // give up //
+         /////////////
+         throw (new QException("Unable to get id for " + tableName + " named " + nameValue));
+      }
+
+      return (cachedFetches.get(key));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private Integer fetchIdFromName(String tableName, String nameValue) throws QException
+   {
+      GetInput getInput = new GetInput();
+      getInput.setTableName(tableName);
+      getInput.setUniqueKey(Map.of("name", nameValue));
+      GetOutput getOutput = new GetAction().execute(getInput);
+      if(getOutput.getRecord() != null)
+      {
+         return (getOutput.getRecord().getValueInteger("id"));
+      }
+
+      return (null);
+   }
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/audits/DMLAuditAction.java

@@ -0,0 +1,398 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2023.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.actions.audits;
+
+
+import java.io.Serializable;
+import java.time.Instant;
+import java.time.ZoneId;
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import com.kingsrook.qqq.backend.core.actions.AbstractQActionFunction;
+import com.kingsrook.qqq.backend.core.actions.values.QPossibleValueTranslator;
+import com.kingsrook.qqq.backend.core.actions.values.QValueFormatter;
+import com.kingsrook.qqq.backend.core.context.QContext;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.logging.QLogger;
+import com.kingsrook.qqq.backend.core.model.actions.AbstractActionInput;
+import com.kingsrook.qqq.backend.core.model.actions.AbstractTableActionInput;
+import com.kingsrook.qqq.backend.core.model.actions.audits.AuditInput;
+import com.kingsrook.qqq.backend.core.model.actions.audits.DMLAuditInput;
+import com.kingsrook.qqq.backend.core.model.actions.audits.DMLAuditOutput;
+import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.metadata.audits.AuditLevel;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
+import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.security.RecordSecurityLock;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+import com.kingsrook.qqq.backend.core.model.session.QSession;
+import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
+import com.kingsrook.qqq.backend.core.utils.StringUtils;
+import com.kingsrook.qqq.backend.core.utils.ValueUtils;
+import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
+
+
+/*******************************************************************************
+ ** Audit for a standard DML (Data Manipulation Language) activity - e.g.,
+ ** insert, edit, or delete.
+ *******************************************************************************/
+public class DMLAuditAction extends AbstractQActionFunction<DMLAuditInput, DMLAuditOutput>
+{
+   private static final QLogger LOG = QLogger.getLogger(DMLAuditAction.class);
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public DMLAuditOutput execute(DMLAuditInput input) throws QException
+   {
+      DMLAuditOutput           output           = new DMLAuditOutput();
+      AbstractTableActionInput tableActionInput = input.getTableActionInput();
+      List<QRecord>            recordList       = input.getRecordList();
+      List<QRecord>            oldRecordList    = input.getOldRecordList();
+      QTableMetaData           table            = tableActionInput.getTable();
+      long                     start            = System.currentTimeMillis();
+      DMLType                  dmlType          = getDMLType(tableActionInput);
+
+      try
+      {
+         AuditLevel auditLevel = getAuditLevel(tableActionInput);
+         if(auditLevel == null || auditLevel.equals(AuditLevel.NONE) || CollectionUtils.nullSafeIsEmpty(recordList))
+         {
+            /////////////////////////////////////////////
+            // return with noop for null or level NONE //
+            /////////////////////////////////////////////
+            return (output);
+         }
+
+         String                        contextSuffix = "";
+         Optional<AbstractActionInput> actionInput   = QContext.getFirstActionInStack();
+         if(actionInput.isPresent() && actionInput.get() instanceof RunProcessInput runProcessInput)
+         {
+            String           processName = runProcessInput.getProcessName();
+            QProcessMetaData process     = QContext.getQInstance().getProcess(processName);
+            if(process != null)
+            {
+               contextSuffix = " during process: " + process.getLabel();
+            }
+         }
+
+         QSession qSession   = QContext.getQSession();
+         String   apiVersion = qSession.getValue("apiVersion");
+         if(apiVersion != null)
+         {
+            String apiLabel = qSession.getValue("apiLabel");
+            if(!StringUtils.hasContent(apiLabel))
+            {
+               apiLabel = "API";
+            }
+            contextSuffix += (" via " + apiLabel + " Version: " + apiVersion);
+         }
+
+         AuditInput auditInput = new AuditInput();
+         if(auditLevel.equals(AuditLevel.RECORD) || (auditLevel.equals(AuditLevel.FIELD) && !dmlType.supportsFields))
+         {
+            ////////////////////////////////////////////////////////////////////////////////////////////////////////////
+            // make many simple audits (no details) for RECORD level                                                  //
+            // or for FIELD level, but on a DML type that doesn't support field-level details (e.g., DELETE or OTHER) //
+            ////////////////////////////////////////////////////////////////////////////////////////////////////////////
+            for(QRecord record : recordList)
+            {
+               AuditAction.appendToInput(auditInput, table.getName(), record.getValueInteger(table.getPrimaryKeyField()), getRecordSecurityKeyValues(table, record), "Record was " + dmlType.pastTenseVerb + contextSuffix);
+            }
+         }
+         else if(auditLevel.equals(AuditLevel.FIELD))
+         {
+            Map<Serializable, QRecord> oldRecordMap = buildOldRecordMap(table, oldRecordList);
+
+            ///////////////////////////////////////////////////////////////////
+            // do many audits, all with field level details, for FIELD level //
+            ///////////////////////////////////////////////////////////////////
+            QPossibleValueTranslator qPossibleValueTranslator = new QPossibleValueTranslator(QContext.getQInstance(), qSession);
+            qPossibleValueTranslator.translatePossibleValuesInRecords(table, CollectionUtils.mergeLists(recordList, oldRecordList));
+
+            //////////////////////////////////////////
+            // sort the field names by their labels //
+            //////////////////////////////////////////
+            List<String> sortedFieldNames = table.getFields().keySet().stream()
+               .sorted(Comparator.comparing(fieldName -> table.getFields().get(fieldName).getLabel()))
+               .toList();
+
+            QFieldMetaData primaryKeyField = table.getField(table.getPrimaryKeyField());
+
+            //////////////////////////////////////////////
+            // build single audit input for each record //
+            //////////////////////////////////////////////
+            for(QRecord record : recordList)
+            {
+               QRecord oldRecord = oldRecordMap.get(ValueUtils.getValueAsFieldType(primaryKeyField.getType(), record.getValue(primaryKeyField.getName())));
+
+               List<QRecord> details = new ArrayList<>();
+               for(String fieldName : sortedFieldNames)
+               {
+                  if(!record.getValues().containsKey(fieldName))
+                  {
+                     ////////////////////////////////////////////////////////////////////////////////////////////////
+                     // if the stored record doesn't have this field name, then don't audit anything about it      //
+                     // this is to deal with our Patch style updates not looking like every field was cleared out. //
+                     ////////////////////////////////////////////////////////////////////////////////////////////////
+                     continue;
+                  }
+
+                  if(fieldName.equals("modifyDate") || fieldName.equals("createDate") || fieldName.equals("automationStatus"))
+                  {
+                     continue;
+                  }
+
+                  QFieldMetaData field        = table.getField(fieldName);
+                  Serializable   value        = ValueUtils.getValueAsFieldType(field.getType(), record.getValue(fieldName));
+                  Serializable   oldValue     = oldRecord == null ? null : ValueUtils.getValueAsFieldType(field.getType(), oldRecord.getValue(fieldName));
+                  QRecord        detailRecord = null;
+
+                  if(oldRecord == null)
+                  {
+                     if(DMLType.INSERT.equals(dmlType) && value == null)
+                     {
+                        continue;
+                     }
+
+                     String formattedValue = getFormattedValueForAuditDetail(record, fieldName, field, value);
+                     detailRecord = new QRecord().withValue("message", "Set " + field.getLabel() + " to " + formattedValue);
+                     detailRecord.withValue("newValue", formattedValue);
+                  }
+                  else
+                  {
+                     if(!Objects.equals(oldValue, value))
+                     {
+                        String formattedValue    = getFormattedValueForAuditDetail(record, fieldName, field, value);
+                        String formattedOldValue = getFormattedValueForAuditDetail(oldRecord, fieldName, field, oldValue);
+
+                        if(oldValue == null)
+                        {
+                           detailRecord = new QRecord().withValue("message", "Set " + field.getLabel() + " to " + formatFormattedValueForDetailMessage(field, formattedValue));
+                           detailRecord.withValue("newValue", formattedValue);
+                        }
+                        else if(value == null)
+                        {
+                           detailRecord = new QRecord().withValue("message", "Removed " + formatFormattedValueForDetailMessage(field, formattedOldValue) + " from " + field.getLabel());
+                           detailRecord.withValue("oldValue", formattedOldValue);
+                        }
+                        else
+                        {
+                           detailRecord = new QRecord().withValue("message", "Changed " + field.getLabel() + " from " + formatFormattedValueForDetailMessage(field, formattedOldValue) + " to " + formatFormattedValueForDetailMessage(field, formattedValue));
+                           detailRecord.withValue("oldValue", formattedOldValue);
+                           detailRecord.withValue("newValue", formattedValue);
+                        }
+                     }
+                  }
+
+                  if(detailRecord != null)
+                  {
+                     detailRecord.withValue("fieldName", fieldName);
+                     details.add(detailRecord);
+                  }
+               }
+
+               if(details.isEmpty() && DMLType.UPDATE.equals(dmlType))
+               {
+                  // no, let's just noop.
+                  // details.add(new QRecord().withValue("message", "No fields values were changed."));
+               }
+               else
+               {
+                  AuditAction.appendToInput(auditInput, table.getName(), record.getValueInteger(table.getPrimaryKeyField()), getRecordSecurityKeyValues(table, record), "Record was " + dmlType.pastTenseVerb + contextSuffix, details);
+               }
+            }
+         }
+
+         // new AuditAction().executeAsync(auditInput); // todo async??? maybe get that from rules???
+         new AuditAction().execute(auditInput);
+         long end = System.currentTimeMillis();
+         LOG.debug("Audit performance", logPair("auditLevel", String.valueOf(auditLevel)), logPair("recordCount", recordList.size()), logPair("millis", (end - start)));
+      }
+      catch(Exception e)
+      {
+         LOG.error("Error performing DML audit", e, logPair("type", String.valueOf(dmlType)), logPair("table", table.getName()));
+      }
+
+      return (output);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private static String getFormattedValueForAuditDetail(QRecord record, String fieldName, QFieldMetaData field, Serializable value)
+   {
+      String formattedValue = null;
+      if(value != null)
+      {
+         if(field.getType().equals(QFieldType.DATE_TIME) && value instanceof Instant instant)
+         {
+            formattedValue = QValueFormatter.formatDateTimeWithZone(instant.atZone(ZoneId.of(Objects.requireNonNullElse(QContext.getQInstance().getDefaultTimeZoneId(), "UTC"))));
+         }
+         else if(record.getDisplayValue(fieldName) != null)
+         {
+            formattedValue = record.getDisplayValue(fieldName);
+         }
+         else
+         {
+            formattedValue = QValueFormatter.formatValue(field, value);
+         }
+      }
+
+      return formattedValue;
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private static String formatFormattedValueForDetailMessage(QFieldMetaData field, String formattedValue)
+   {
+      if(formattedValue == null || "null".equals(formattedValue))
+      {
+         formattedValue = "--";
+      }
+      else
+      {
+         if(QFieldType.STRING.equals(field.getType()) || field.getPossibleValueSourceName() != null)
+         {
+            formattedValue = '"' + formattedValue + '"';
+         }
+      }
+
+      return (formattedValue);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private Map<Serializable, QRecord> buildOldRecordMap(QTableMetaData table, List<QRecord> oldRecordList)
+   {
+      Map<Serializable, QRecord> rs = new HashMap<>();
+      for(QRecord record : CollectionUtils.nonNullList(oldRecordList))
+      {
+         rs.put(record.getValue(table.getPrimaryKeyField()), record);
+      }
+      return (rs);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private DMLType getDMLType(AbstractTableActionInput tableActionInput)
+   {
+      if(tableActionInput instanceof InsertInput)
+      {
+         return DMLType.INSERT;
+      }
+      else if(tableActionInput instanceof UpdateInput)
+      {
+         return DMLType.UPDATE;
+      }
+      else if(tableActionInput instanceof DeleteInput)
+      {
+         return DMLType.DELETE;
+      }
+      else
+      {
+         return DMLType.OTHER;
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private static Map<String, Serializable> getRecordSecurityKeyValues(QTableMetaData table, QRecord record)
+   {
+      Map<String, Serializable> securityKeyValues = new HashMap<>();
+      for(RecordSecurityLock recordSecurityLock : CollectionUtils.nonNullList(table.getRecordSecurityLocks()))
+      {
+         securityKeyValues.put(recordSecurityLock.getSecurityKeyType(), record == null ? null : record.getValue(recordSecurityLock.getFieldName()));
+      }
+      return securityKeyValues;
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public static AuditLevel getAuditLevel(AbstractTableActionInput tableActionInput)
+   {
+      QTableMetaData table = tableActionInput.getTable();
+      if(table.getAuditRules() == null)
+      {
+         return (AuditLevel.NONE);
+      }
+
+      return (table.getAuditRules().getAuditLevel());
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private enum DMLType
+   {
+      INSERT("Inserted", true),
+      UPDATE("Edited", true),
+      DELETE("Deleted", false),
+      OTHER("Processed", false);
+
+      private final String  pastTenseVerb;
+      private final boolean supportsFields;
+
+
+
+      /*******************************************************************************
+       **
+       *******************************************************************************/
+      DMLType(String pastTenseVerb, boolean supportsFields)
+      {
+         this.pastTenseVerb = pastTenseVerb;
+         this.supportsFields = supportsFields;
+      }
+   }
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/automation/AutomationStatus.java

@@ -0,0 +1,115 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.actions.automation;
+
+
+import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.PossibleValueEnum;
+
+
+/*******************************************************************************
+ ** enum of possible values for a record's Automation Status.
+ *******************************************************************************/
+public enum AutomationStatus implements PossibleValueEnum<Integer>
+{
+   PENDING_INSERT_AUTOMATIONS(1, "Pending Insert Automations"),
+   RUNNING_INSERT_AUTOMATIONS(2, "Running Insert Automations"),
+   FAILED_INSERT_AUTOMATIONS(3, "Failed Insert Automations"),
+   PENDING_UPDATE_AUTOMATIONS(4, "Pending Update Automations"),
+   RUNNING_UPDATE_AUTOMATIONS(5, "Running Update Automations"),
+   FAILED_UPDATE_AUTOMATIONS(6, "Failed Update Automations"),
+   OK(7, "OK");
+
+
+   private final Integer id;
+   private final String  label;
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   AutomationStatus(int id, String label)
+   {
+      this.id = id;
+      this.label = label;
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for id
+    **
+    *******************************************************************************/
+   public Integer getId()
+   {
+      return (id);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for label
+    **
+    *******************************************************************************/
+   public String getLabel()
+   {
+      return (label);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public Integer getPossibleValueId()
+   {
+      return (getId());
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public String getPossibleValueLabel()
+   {
+      return (getLabel());
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @SuppressWarnings("checkstyle:indentation")
+   public String getInsertOrUpdate()
+   {
+      return switch(this)
+         {
+            case PENDING_INSERT_AUTOMATIONS, RUNNING_INSERT_AUTOMATIONS, FAILED_INSERT_AUTOMATIONS -> "Insert";
+            case PENDING_UPDATE_AUTOMATIONS, RUNNING_UPDATE_AUTOMATIONS, FAILED_UPDATE_AUTOMATIONS -> "Update";
+            case OK -> "";
+         };
+   }
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/automation/RecordAutomationHandler.java

@@ -0,0 +1,40 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.actions.automation;
+
+
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.automation.RecordAutomationInput;
+
+
+/*******************************************************************************
+ ** Base class for custom-codes to run as an automation action
+ *******************************************************************************/
+public abstract class RecordAutomationHandler
+{
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public abstract void execute(RecordAutomationInput recordAutomationInput) throws QException;
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/automation/RecordAutomationStatusUpdater.java

@@ -0,0 +1,228 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.actions.automation;
+
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Objects;
+import com.kingsrook.qqq.backend.core.actions.tables.CountAction;
+import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction;
+import com.kingsrook.qqq.backend.core.context.QContext;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.logging.QLogger;
+import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountOutput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
+import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
+import com.kingsrook.qqq.backend.core.model.automation.TableTrigger;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.AutomationStatusTrackingType;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.QTableAutomationDetails;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.TableAutomationAction;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.TriggerEvent;
+import com.kingsrook.qqq.backend.core.model.session.QSession;
+import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
+import org.apache.commons.lang.NotImplementedException;
+
+
+/*******************************************************************************
+ ** Utility class for updating the automation status data for records
+ *******************************************************************************/
+public class RecordAutomationStatusUpdater
+{
+   private static final QLogger LOG = QLogger.getLogger(RecordAutomationStatusUpdater.class);
+
+
+
+   /*******************************************************************************
+    ** for a list of records from a table, set their automation status - based on
+    ** how the table is configured.
+    *******************************************************************************/
+   public static boolean setAutomationStatusInRecords(QSession session, QTableMetaData table, List<QRecord> records, AutomationStatus automationStatus)
+   {
+      if(table == null || table.getAutomationDetails() == null || CollectionUtils.nullSafeIsEmpty(records))
+      {
+         return (false);
+      }
+
+      ///////////////////////////////////////////////////////////////////////////////////////////////////
+      // In case an automation is running, and it updates records - don't let those records be marked  //
+      // as PENDING_UPDATE_AUTOMATIONS... this is meant to avoid having a record's automation update   //
+      // itself, and then continue to do so in a loop (infinitely).                                    //
+      // BUT - shouldn't this be allowed to update OTHER records to be pending updates?  It seems like //
+      // yes - so -that'll probably be a bug to fix at some point in the future todo                   //
+      ///////////////////////////////////////////////////////////////////////////////////////////////////
+      if(automationStatus.equals(AutomationStatus.PENDING_UPDATE_AUTOMATIONS))
+      {
+         Exception e = new Exception();
+         for(StackTraceElement stackTraceElement : e.getStackTrace())
+         {
+            String className = stackTraceElement.getClassName();
+            if(className.contains("com.kingsrook.qqq.backend.core.actions.automation") && !className.equals(RecordAutomationStatusUpdater.class.getName()) && !className.endsWith("Test"))
+            {
+               LOG.debug("Avoiding re-setting automation status to PENDING_UPDATE while running an automation");
+               return (false);
+            }
+         }
+      }
+
+      if(canWeSkipPendingAndGoToOkay(table, automationStatus))
+      {
+         automationStatus = AutomationStatus.OK;
+      }
+
+      QTableAutomationDetails automationDetails = table.getAutomationDetails();
+      if(automationDetails.getStatusTracking() != null && AutomationStatusTrackingType.FIELD_IN_TABLE.equals(automationDetails.getStatusTracking().getType()))
+      {
+         for(QRecord record : records)
+         {
+            ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+            // todo - seems like there's some case here, where if an order was in PENDING_INSERT, but then some other job updated the record, that we'd  //
+            //  lose that pending status, which would be a Bad Thing™...                                                                                 //
+            //  problem is - we may not have the full record in here, so we can't necessarily check the record to see what status it's currently in...   //
+            ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
+            record.setValue(automationDetails.getStatusTracking().getFieldName(), automationStatus.getId());
+            // todo - another field - for the automation timestamp??
+         }
+      }
+
+      return (true);
+   }
+
+
+
+   /*******************************************************************************
+    ** If a table has no automation actions defined for Insert (or Update), and we're
+    ** being asked to set status to PENDING_INSERT (or PENDING_UPDATE), then just
+    ** move the status straight to OK.
+    *******************************************************************************/
+   private static boolean canWeSkipPendingAndGoToOkay(QTableMetaData table, AutomationStatus automationStatus)
+   {
+      List<TableAutomationAction> tableActions = Objects.requireNonNullElse(table.getAutomationDetails().getActions(), new ArrayList<>());
+
+      if(automationStatus.equals(AutomationStatus.PENDING_INSERT_AUTOMATIONS))
+      {
+         if(tableActions.stream().anyMatch(a -> TriggerEvent.POST_INSERT.equals(a.getTriggerEvent())))
+         {
+            return (false);
+         }
+         else if(areThereTableTriggersForTable(table, TriggerEvent.POST_INSERT))
+         {
+            return (false);
+         }
+      }
+      else if(automationStatus.equals(AutomationStatus.PENDING_UPDATE_AUTOMATIONS))
+      {
+         if(tableActions.stream().anyMatch(a -> TriggerEvent.POST_UPDATE.equals(a.getTriggerEvent())))
+         {
+            return (false);
+         }
+         else if(areThereTableTriggersForTable(table, TriggerEvent.POST_UPDATE))
+         {
+            return (false);
+         }
+      }
+
+      return (true);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private static boolean areThereTableTriggersForTable(QTableMetaData table, TriggerEvent triggerEvent)
+   {
+      if(QContext.getQInstance().getTable(TableTrigger.TABLE_NAME) == null)
+      {
+         return (false);
+      }
+
+      try
+      {
+         ///////////////////
+         // todo - cache? //
+         ///////////////////
+         CountInput countInput = new CountInput();
+         countInput.setTableName(TableTrigger.TABLE_NAME);
+         countInput.setFilter(new QQueryFilter(
+            new QFilterCriteria("tableName", QCriteriaOperator.EQUALS, table.getName()),
+            new QFilterCriteria(triggerEvent.equals(TriggerEvent.POST_INSERT) ? "postInsert" : "postUpdate", QCriteriaOperator.EQUALS, true)
+         ));
+         CountOutput countOutput = new CountAction().execute(countInput);
+         return (countOutput.getCount() != null && countOutput.getCount() > 0);
+      }
+      catch(Exception e)
+      {
+         ///////////////////////////////////////////////////////////////////////////////////////////////////////////
+         // if the count query failed, we're a bit safer to err on the side of "yeah, there might be automations" //
+         ///////////////////////////////////////////////////////////////////////////////////////////////////////////
+         return (true);
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** for a list of records, update their automation status and actually Update the
+    ** backend as well.
+    *******************************************************************************/
+   public static void setAutomationStatusInRecordsAndUpdate(QInstance instance, QSession session, QTableMetaData table, List<QRecord> records, AutomationStatus automationStatus) throws QException
+   {
+      QTableAutomationDetails automationDetails = table.getAutomationDetails();
+      if(automationDetails != null && AutomationStatusTrackingType.FIELD_IN_TABLE.equals(automationDetails.getStatusTracking().getType()))
+      {
+         boolean didSetStatusField = setAutomationStatusInRecords(session, table, records, automationStatus);
+         if(didSetStatusField)
+         {
+            UpdateInput updateInput = new UpdateInput();
+            updateInput.setTableName(table.getName());
+
+            /////////////////////////////////////////////////////////////////////////////////////
+            // build records with just their pkey & status field for this update, to avoid     //
+            // changing other values (relies on assumption of Patch semantics in UpdateAction) //
+            /////////////////////////////////////////////////////////////////////////////////////
+            updateInput.setRecords(records.stream().map(r -> new QRecord()
+               .withTableName(r.getTableName())
+               .withValue(table.getPrimaryKeyField(), r.getValue(table.getPrimaryKeyField()))
+               .withValue(automationDetails.getStatusTracking().getFieldName(), r.getValue(automationDetails.getStatusTracking().getFieldName()))).toList());
+            updateInput.setAreAllValuesBeingUpdatedTheSame(true);
+            updateInput.setOmitDmlAudit(true);
+
+            new UpdateAction().execute(updateInput);
+         }
+      }
+      else
+      {
+         // todo - verify if this is valid as other types are built
+         throw (new NotImplementedException("Updating record automation status is not implemented for table [" + table + "], tracking type: "
+            + (automationDetails == null ? "null" : automationDetails.getStatusTracking().getType())));
+      }
+   }
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/automation/RunRecordScriptAutomationHandler.java

@@ -0,0 +1,95 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2023.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.actions.automation;
+
+
+import java.io.Serializable;
+import java.util.Map;
+import com.kingsrook.qqq.backend.core.actions.scripts.RunAdHocRecordScriptAction;
+import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
+import com.kingsrook.qqq.backend.core.context.QContext;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.logging.QLogger;
+import com.kingsrook.qqq.backend.core.model.actions.scripts.RunAdHocRecordScriptInput;
+import com.kingsrook.qqq.backend.core.model.actions.scripts.RunAdHocRecordScriptOutput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryJoin;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput;
+import com.kingsrook.qqq.backend.core.model.automation.RecordAutomationInput;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.metadata.code.AdHocScriptCodeReference;
+import com.kingsrook.qqq.backend.core.model.scripts.Script;
+import com.kingsrook.qqq.backend.core.model.scripts.ScriptRevision;
+import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
+import com.kingsrook.qqq.backend.core.utils.ValueUtils;
+import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
+
+
+/*******************************************************************************
+ **
+ *******************************************************************************/
+public class RunRecordScriptAutomationHandler extends RecordAutomationHandler
+{
+   private static final QLogger LOG = QLogger.getLogger(RunRecordScriptAutomationHandler.class);
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public void execute(RecordAutomationInput recordAutomationInput) throws QException
+   {
+      String                    tableName = recordAutomationInput.getTableName();
+      Map<String, Serializable> values    = recordAutomationInput.getAction().getValues();
+      Integer                   scriptId  = ValueUtils.getValueAsInteger(values.get("scriptId"));
+
+      if(scriptId == null)
+      {
+         throw (new QException("ScriptId was not provided in values map for record automations on table: " + tableName));
+      }
+
+      QueryInput queryInput = new QueryInput();
+      queryInput.setTableName(ScriptRevision.TABLE_NAME);
+      queryInput.setFilter(new QQueryFilter(new QFilterCriteria("scriptId", QCriteriaOperator.EQUALS, scriptId)));
+      queryInput.withQueryJoin(new QueryJoin(Script.TABLE_NAME).withBaseTableOrAlias(ScriptRevision.TABLE_NAME).withJoinMetaData(QContext.getQInstance().getJoin("currentScriptRevision")));
+      QueryOutput queryOutput = new QueryAction().execute(queryInput);
+      if(CollectionUtils.nullSafeIsEmpty(queryOutput.getRecords()))
+      {
+         throw (new QException("Could not find current revision for scriptId: " + scriptId + " on table " + tableName));
+      }
+
+      QRecord scriptRevision = queryOutput.getRecords().get(0);
+      LOG.info("Running script against records", logPair("scriptRevisionId", scriptRevision.getValue("id")), logPair("scriptId", scriptRevision.getValue("scriptIdd")));
+
+      RunAdHocRecordScriptInput input = new RunAdHocRecordScriptInput();
+      input.setCodeReference(new AdHocScriptCodeReference().withScriptRevisionRecord(scriptRevision));
+      input.setTableName(tableName);
+      input.setRecordList(recordAutomationInput.getRecordList());
+      RunAdHocRecordScriptOutput output = new RunAdHocRecordScriptOutput();
+      new RunAdHocRecordScriptAction().run(input, output);
+   }
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/automation/polling/PollingAutomationPerTableRunner.java

@@ -0,0 +1,465 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.actions.automation.polling;
+
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.function.Supplier;
+import java.util.stream.Collectors;
+import com.kingsrook.qqq.backend.core.actions.async.AsyncRecordPipeLoop;
+import com.kingsrook.qqq.backend.core.actions.automation.AutomationStatus;
+import com.kingsrook.qqq.backend.core.actions.automation.RecordAutomationHandler;
+import com.kingsrook.qqq.backend.core.actions.automation.RecordAutomationStatusUpdater;
+import com.kingsrook.qqq.backend.core.actions.automation.RunRecordScriptAutomationHandler;
+import com.kingsrook.qqq.backend.core.actions.customizers.QCodeLoader;
+import com.kingsrook.qqq.backend.core.actions.processes.QProcessCallback;
+import com.kingsrook.qqq.backend.core.actions.processes.RunProcessAction;
+import com.kingsrook.qqq.backend.core.actions.reporting.RecordPipe;
+import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
+import com.kingsrook.qqq.backend.core.context.QContext;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.logging.QLogger;
+import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput;
+import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessOutput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput;
+import com.kingsrook.qqq.backend.core.model.automation.RecordAutomationInput;
+import com.kingsrook.qqq.backend.core.model.automation.TableTrigger;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.AutomationStatusTrackingType;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.QTableAutomationDetails;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.TableAutomationAction;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.TriggerEvent;
+import com.kingsrook.qqq.backend.core.model.session.QSession;
+import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
+import com.kingsrook.qqq.backend.core.utils.StringUtils;
+import com.kingsrook.qqq.backend.core.utils.collections.MapBuilder;
+import org.apache.commons.lang.NotImplementedException;
+
+
+/*******************************************************************************
+ ** Runnable for the Polling Automation Provider, that looks for records that
+ ** need automations, and executes them.
+ **
+ ** An instance of this class should be created for each table/automation-status
+ ** - see the TableActions inner record for that definition, and the static
+ ** getTableActions method that helps someone who wants to start these threads
+ ** figure out which ones are needed.
+ *******************************************************************************/
+public class PollingAutomationPerTableRunner implements Runnable
+{
+   private static final QLogger LOG = QLogger.getLogger(PollingAutomationPerTableRunner.class);
+
+   private final TableActions tableActions;
+   private final String       name;
+
+   private QInstance          instance;
+   private Supplier<QSession> sessionSupplier;
+
+   private static Map<TriggerEvent, AutomationStatus> triggerEventAutomationStatusMap = Map.of(
+      TriggerEvent.POST_INSERT, AutomationStatus.PENDING_INSERT_AUTOMATIONS,
+      TriggerEvent.POST_UPDATE, AutomationStatus.PENDING_UPDATE_AUTOMATIONS
+   );
+
+   private static Map<AutomationStatus, TriggerEvent> automationStatusTriggerEventMap = Map.of(
+      AutomationStatus.PENDING_INSERT_AUTOMATIONS, TriggerEvent.POST_INSERT,
+      AutomationStatus.PENDING_UPDATE_AUTOMATIONS, TriggerEvent.POST_UPDATE
+   );
+
+   private static Map<AutomationStatus, AutomationStatus> pendingToRunningStatusMap = Map.of(
+      AutomationStatus.PENDING_INSERT_AUTOMATIONS, AutomationStatus.RUNNING_INSERT_AUTOMATIONS,
+      AutomationStatus.PENDING_UPDATE_AUTOMATIONS, AutomationStatus.RUNNING_UPDATE_AUTOMATIONS
+   );
+
+   private static Map<AutomationStatus, AutomationStatus> pendingToFailedStatusMap = Map.of(
+      AutomationStatus.PENDING_INSERT_AUTOMATIONS, AutomationStatus.FAILED_INSERT_AUTOMATIONS,
+      AutomationStatus.PENDING_UPDATE_AUTOMATIONS, AutomationStatus.FAILED_UPDATE_AUTOMATIONS
+   );
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public record TableActions(String tableName, AutomationStatus status)
+   {
+   }
+
+
+
+   /*******************************************************************************
+    ** basically just get a list of tables which at least *could* have automations
+    ** run - either meta-data automations, or table-triggers (data/user defined).
+    *******************************************************************************/
+   public static List<TableActions> getTableActions(QInstance instance, String providerName)
+   {
+      List<TableActions> tableActionList = new ArrayList<>();
+
+      for(QTableMetaData table : instance.getTables().values())
+      {
+         if(table.getAutomationDetails() != null && providerName.equals(table.getAutomationDetails().getProviderName()))
+         {
+            tableActionList.add(new TableActions(table.getName(), AutomationStatus.PENDING_INSERT_AUTOMATIONS));
+            tableActionList.add(new TableActions(table.getName(), AutomationStatus.PENDING_UPDATE_AUTOMATIONS));
+         }
+      }
+
+      return (tableActionList);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public PollingAutomationPerTableRunner(QInstance instance, String providerName, Supplier<QSession> sessionSupplier, TableActions tableActions)
+   {
+      this.instance = instance;
+      this.sessionSupplier = sessionSupplier;
+      this.tableActions = tableActions;
+      this.name = providerName + ">" + tableActions.tableName() + ">" + tableActions.status().getInsertOrUpdate();
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public void run()
+   {
+      QContext.init(instance, sessionSupplier.get());
+
+      String originalThreadName = Thread.currentThread().getName();
+      Thread.currentThread().setName(name);
+      LOG.debug("Running " + this.getClass().getSimpleName() + "[" + name + "]");
+
+      try
+      {
+         QSession session = sessionSupplier != null ? sessionSupplier.get() : new QSession();
+         processTableInsertOrUpdate(instance.getTable(tableActions.tableName()), session, tableActions.status());
+      }
+      catch(Exception e)
+      {
+         LOG.warn("Error running automations", e);
+      }
+      finally
+      {
+         Thread.currentThread().setName(originalThreadName);
+         QContext.clear();
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** Query for and process records that have a PENDING_INSERT or PENDING_UPDATE status on a given table.
+    *******************************************************************************/
+   public void processTableInsertOrUpdate(QTableMetaData table, QSession session, AutomationStatus automationStatus) throws QException
+   {
+      /////////////////////////////////////////////////////////////////////////
+      // get the actions to run against this table in this automation status //
+      /////////////////////////////////////////////////////////////////////////
+      List<TableAutomationAction> actions = getTableActions(table, automationStatus);
+      if(CollectionUtils.nullSafeIsEmpty(actions))
+      {
+         return;
+      }
+
+      LOG.debug("  Query for records " + automationStatus + " in " + table);
+
+      ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      // run an async-pipe loop - that will query for records in PENDING - put them in a pipe - then apply actions to them //
+      ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      QTableAutomationDetails automationDetails   = table.getAutomationDetails();
+      AsyncRecordPipeLoop     asyncRecordPipeLoop = new AsyncRecordPipeLoop();
+
+      RecordPipe recordPipe = automationDetails.getOverrideBatchSize() == null
+         ? new RecordPipe() : new RecordPipe(automationDetails.getOverrideBatchSize());
+
+      asyncRecordPipeLoop.run("PollingAutomationRunner>Query>" + automationStatus + ">" + table.getName(), null, recordPipe, (status) ->
+         {
+            QueryInput queryInput = new QueryInput();
+            queryInput.setTableName(table.getName());
+
+            AutomationStatusTrackingType statusTrackingType = automationDetails.getStatusTracking().getType();
+            if(AutomationStatusTrackingType.FIELD_IN_TABLE.equals(statusTrackingType))
+            {
+               queryInput.setFilter(new QQueryFilter().withCriteria(new QFilterCriteria(automationDetails.getStatusTracking().getFieldName(), QCriteriaOperator.EQUALS, List.of(automationStatus.getId()))));
+            }
+            else
+            {
+               throw (new NotImplementedException("Automation Status Tracking type [" + statusTrackingType + "] is not yet implemented in here."));
+            }
+
+            queryInput.setRecordPipe(recordPipe);
+            return (new QueryAction().execute(queryInput));
+         }, () ->
+         {
+            List<QRecord> records = recordPipe.consumeAvailableRecords();
+            applyActionsToRecords(session, table, records, actions, automationStatus);
+            return (records.size());
+         }
+      );
+   }
+
+
+
+   /*******************************************************************************
+    ** get the actions to run against a table in an automation status.  both from
+    ** metaData and tableTriggers/data.
+    *******************************************************************************/
+   private List<TableAutomationAction> getTableActions(QTableMetaData table, AutomationStatus automationStatus) throws QException
+   {
+      List<TableAutomationAction> rs           = new ArrayList<>();
+      TriggerEvent                triggerEvent = automationStatusTriggerEventMap.get(automationStatus);
+
+      ///////////////////////////////////////////////////////////
+      // start with any actions defined in the table meta data //
+      ///////////////////////////////////////////////////////////
+      for(TableAutomationAction action : table.getAutomationDetails().getActions())
+      {
+         if(action.getTriggerEvent().equals(triggerEvent))
+         {
+            rs.add(action);
+         }
+      }
+
+      /////////////////////////////////////////////////
+      // next add any tableTriggers, defined in data //
+      /////////////////////////////////////////////////
+      if(QContext.getQInstance().getTable(TableTrigger.TABLE_NAME) != null)
+      {
+         QueryInput queryInput = new QueryInput();
+         queryInput.setTableName(TableTrigger.TABLE_NAME);
+         queryInput.setFilter(new QQueryFilter(
+            new QFilterCriteria("tableName", QCriteriaOperator.EQUALS, table.getName()),
+            new QFilterCriteria(triggerEvent.equals(TriggerEvent.POST_INSERT) ? "postInsert" : "postUpdate", QCriteriaOperator.EQUALS, true)
+         ));
+         QueryOutput queryOutput = new QueryAction().execute(queryInput);
+         for(QRecord record : queryOutput.getRecords())
+         {
+            // todo - get filter if there is/was one
+            rs.add(new TableAutomationAction()
+               .withName("Script:" + record.getValue("scriptId"))
+               .withFilter(null)
+               .withTriggerEvent(triggerEvent)
+               .withPriority(record.getValueInteger("priority"))
+               .withCodeReference(new QCodeReference(RunRecordScriptAutomationHandler.class))
+               .withValues(MapBuilder.of("scriptId", record.getValue("scriptId")))
+            );
+         }
+      }
+
+      rs.sort(Comparator.comparing(taa -> Objects.requireNonNullElse(taa.getPriority(), Integer.MAX_VALUE)));
+
+      return (rs);
+   }
+
+
+
+   /*******************************************************************************
+    ** For a set of records that were found to be in a PENDING state - run all the
+    ** table's actions against them - IF they are found to match the action's filter
+    ** (assuming it has one - if it doesn't, then all records match).
+    *******************************************************************************/
+   private void applyActionsToRecords(QSession session, QTableMetaData table, List<QRecord> records, List<TableAutomationAction> actions, AutomationStatus automationStatus) throws QException
+   {
+      if(CollectionUtils.nullSafeIsEmpty(records))
+      {
+         return;
+      }
+
+      ///////////////////////////////////////////////////
+      // mark the records as RUNNING their automations //
+      ///////////////////////////////////////////////////
+      RecordAutomationStatusUpdater.setAutomationStatusInRecordsAndUpdate(instance, session, table, records, pendingToRunningStatusMap.get(automationStatus));
+
+      ///////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      // foreach action - run it against the records (but only if they match the action's filter, if there is one) //
+      ///////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      boolean anyActionsFailed = false;
+      for(TableAutomationAction action : actions)
+      {
+         try
+         {
+            //////////////////////////////////////////////////////////////////////////////////////////////////////////////
+            // note - this method - will re-query the objects, so we should have confidence that their data is fresh... //
+            //////////////////////////////////////////////////////////////////////////////////////////////////////////////
+            List<QRecord> matchingQRecords = getRecordsMatchingActionFilter(table, records, action);
+            LOG.debug("Of the {} records that were pending automations, {} of them match the filter on the action {}", records.size(), matchingQRecords.size(), action);
+            if(CollectionUtils.nullSafeHasContents(matchingQRecords))
+            {
+               LOG.debug("  Processing " + matchingQRecords.size() + " records in " + table + " for action " + action);
+               applyActionToMatchingRecords(table, matchingQRecords, action);
+            }
+         }
+         catch(Exception e)
+         {
+            LOG.warn("Caught exception processing records on " + table + " for action " + action, e);
+            anyActionsFailed = true;
+         }
+      }
+
+      ////////////////////////////////////////
+      // update status on all these records //
+      ////////////////////////////////////////
+      if(anyActionsFailed)
+      {
+         RecordAutomationStatusUpdater.setAutomationStatusInRecordsAndUpdate(instance, session, table, records, pendingToFailedStatusMap.get(automationStatus));
+      }
+      else
+      {
+         RecordAutomationStatusUpdater.setAutomationStatusInRecordsAndUpdate(instance, session, table, records, AutomationStatus.OK);
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** For a given action, and a list of records - return a new list, of the ones
+    ** which match the action's filter (if there is one - if not, then all match).
+    **
+    ** Note that this WILL re-query the objects  (ALWAYS - even if the action has no filter).
+    ** This has the nice side effect of always giving fresh/updated records, despite having
+    ** some cost.
+    **
+    ** At one point, we considered just applying the filter using java-comparisons,
+    ** but that will almost certainly give potentially different results than a true
+    ** backend - e.g., just consider if the DB is case-sensitive for strings...
+    *******************************************************************************/
+   private List<QRecord> getRecordsMatchingActionFilter(QTableMetaData table, List<QRecord> records, TableAutomationAction action) throws QException
+   {
+      QueryInput queryInput = new QueryInput();
+      queryInput.setTableName(table.getName());
+
+      QQueryFilter filter = new QQueryFilter();
+
+      /////////////////////////////////////////////////////////////////////////////////////////////////////
+      // copy filter criteria from the action's filter to a new filter that we'll run here.              //
+      // Critically - don't modify the filter object on the action!  as that object has a long lifespan. //
+      /////////////////////////////////////////////////////////////////////////////////////////////////////
+      if(action.getFilter() != null)
+      {
+         if(action.getFilter().getCriteria() != null)
+         {
+            action.getFilter().getCriteria().forEach(filter::addCriteria);
+         }
+         if(action.getFilter().getOrderBys() != null)
+         {
+            action.getFilter().getOrderBys().forEach(filter::addOrderBy);
+         }
+      }
+
+      filter.addCriteria(new QFilterCriteria(table.getPrimaryKeyField(), QCriteriaOperator.IN, records.stream().map(r -> r.getValue(table.getPrimaryKeyField())).toList()));
+
+      /////////////////////////////////////////////////////////////////////////////////////////////
+      // always add order-by the primary key, to give more predictable/consistent results        //
+      // todo - in future - if this becomes a source of slowness, make this a config to opt-out? //
+      /////////////////////////////////////////////////////////////////////////////////////////////
+      filter.addOrderBy(new QFilterOrderBy().withFieldName(table.getPrimaryKeyField()));
+
+      queryInput.setFilter(filter);
+
+      return (new QueryAction().execute(queryInput).getRecords());
+   }
+
+
+
+   /*******************************************************************************
+    ** Finally, actually run action code against a list of known matching records.
+    ** todo not commit - move to somewhere genericer
+    *******************************************************************************/
+   public static void applyActionToMatchingRecords(QTableMetaData table, List<QRecord> records, TableAutomationAction action) throws Exception
+   {
+      if(StringUtils.hasContent(action.getProcessName()))
+      {
+         /////////////////////////////////////////////////////////////////////////////////////////
+         // if the action has a process associated with it - run that process.                  //
+         // tell it to SKIP frontend steps.                                                     //
+         // give the process a callback w/ a query filter that has the p-keys of these records. //
+         /////////////////////////////////////////////////////////////////////////////////////////
+         RunProcessInput runProcessInput = new RunProcessInput();
+         runProcessInput.setProcessName(action.getProcessName());
+         runProcessInput.setFrontendStepBehavior(RunProcessInput.FrontendStepBehavior.SKIP);
+         runProcessInput.setCallback(new QProcessCallback()
+         {
+            @Override
+            public QQueryFilter getQueryFilter()
+            {
+               List<Serializable> recordIds = records.stream().map(r -> r.getValueInteger(table.getPrimaryKeyField())).collect(Collectors.toList());
+               return (new QQueryFilter().withCriteria(new QFilterCriteria(table.getPrimaryKeyField(), QCriteriaOperator.IN, recordIds)));
+            }
+         });
+
+         try
+         {
+            QContext.pushAction(runProcessInput);
+
+            RunProcessAction runProcessAction = new RunProcessAction();
+            RunProcessOutput runProcessOutput = runProcessAction.execute(runProcessInput);
+            if(runProcessOutput.getException().isPresent())
+            {
+               throw (runProcessOutput.getException().get());
+            }
+         }
+         finally
+         {
+            QContext.popAction();
+         }
+      }
+      else if(action.getCodeReference() != null)
+      {
+         LOG.debug("    Executing action: [" + action.getName() + "] as code reference: " + action.getCodeReference());
+         RecordAutomationInput input = new RecordAutomationInput();
+         input.setTableName(table.getName());
+         input.setRecordList(records);
+         input.setAction(action);
+
+         RecordAutomationHandler recordAutomationHandler = QCodeLoader.getRecordAutomationHandler(action);
+         recordAutomationHandler.execute(input);
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for name
+    **
+    *******************************************************************************/
+   public String getName()
+   {
+      return name;
+   }
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/AbstractPostInsertCustomizer.java

@@ -0,0 +1,65 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.actions.customizers;
+
+
+import java.util.List;
+import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+
+
+/*******************************************************************************
+ **
+ *******************************************************************************/
+public abstract class AbstractPostInsertCustomizer
+{
+   protected InsertInput insertInput;
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public abstract List<QRecord> apply(List<QRecord> records);
+
+
+
+   /*******************************************************************************
+    ** Getter for insertInput
+    **
+    *******************************************************************************/
+   public InsertInput getInsertInput()
+   {
+      return insertInput;
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for insertInput
+    **
+    *******************************************************************************/
+   public void setInsertInput(InsertInput insertInput)
+   {
+      this.insertInput = insertInput;
+   }
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/AbstractPostQueryCustomizer.java

@@ -0,0 +1,65 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.actions.customizers;
+
+
+import java.util.List;
+import com.kingsrook.qqq.backend.core.model.actions.AbstractTableActionInput;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+
+
+/*******************************************************************************
+ **
+ *******************************************************************************/
+public abstract class AbstractPostQueryCustomizer
+{
+   protected AbstractTableActionInput input;
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public abstract List<QRecord> apply(List<QRecord> records);
+
+
+
+   /*******************************************************************************
+    ** Getter for input
+    **
+    *******************************************************************************/
+   public AbstractTableActionInput getInput()
+   {
+      return (input);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for input
+    **
+    *******************************************************************************/
+   public void setInput(AbstractTableActionInput input)
+   {
+      this.input = input;
+   }
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/ChildInserterPostInsertCustomizer.java

@@ -0,0 +1,160 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.actions.customizers;
+
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
+import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertOutput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+
+
+/*******************************************************************************
+ ** Standard/re-usable post-insert customizer, for the use case where, when we
+ ** do an insert into table "parent", we want a record automatically inserted into
+ ** table "child", and there's a foreign key in "parent", pointed at "child"
+ ** e.g., named: "parent.childId".
+ **
+ ** A similar use-case would have the foreign key in the child table - in which case,
+ ** we could add a "Type" enum, plus abstract method to get our "Type", then logic
+ ** to switch behavior based on type.  See existing type enum, but w/ only 1 case :)
+ *******************************************************************************/
+public abstract class ChildInserterPostInsertCustomizer extends AbstractPostInsertCustomizer
+{
+   public enum RelationshipType
+   {
+      PARENT_POINTS_AT_CHILD
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public abstract QRecord buildChildForRecord(QRecord parentRecord) throws QException;
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public abstract String getChildTableName();
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public abstract String getForeignKeyFieldName();
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public abstract RelationshipType getRelationshipType();
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public List<QRecord> apply(List<QRecord> records)
+   {
+      try
+      {
+         List<QRecord>  rs               = new ArrayList<>();
+         List<QRecord>  childrenToInsert = new ArrayList<>();
+         QTableMetaData table            = getInsertInput().getTable();
+         QTableMetaData childTable       = getInsertInput().getInstance().getTable(getChildTableName());
+
+         ////////////////////////////////////////////////////////////////////////////////
+         // iterate over the inserted records, building a list child records to insert //
+         // for ones missing a value in the foreign key field.                         //
+         ////////////////////////////////////////////////////////////////////////////////
+         for(QRecord record : records)
+         {
+            if(record.getValue(getForeignKeyFieldName()) == null)
+            {
+               childrenToInsert.add(buildChildForRecord(record));
+            }
+         }
+
+         ///////////////////////////////////////////////////////////////////////////////////
+         // if there are no children to insert, then just return the original record list //
+         ///////////////////////////////////////////////////////////////////////////////////
+         if(childrenToInsert.isEmpty())
+         {
+            return (records);
+         }
+
+         /////////////////////////
+         // insert the children //
+         /////////////////////////
+         InsertInput insertInput = new InsertInput();
+         insertInput.setTableName(getChildTableName());
+         insertInput.setRecords(childrenToInsert);
+         insertInput.setTransaction(this.insertInput.getTransaction());
+         InsertOutput      insertOutput           = new InsertAction().execute(insertInput);
+         Iterator<QRecord> insertedRecordIterator = insertOutput.getRecords().iterator();
+
+         //////////////////////////////////////////////////////////////////////////////////////////////////////
+         // iterate over the original list of records again - for any that need a child (e.g., are missing   //
+         // foreign key), set their foreign key to a newly inserted child's key, and add them to be updated. //
+         //////////////////////////////////////////////////////////////////////////////////////////////////////
+         List<QRecord> recordsToUpdate = new ArrayList<>();
+         for(QRecord record : records)
+         {
+            Serializable primaryKey = record.getValue(table.getPrimaryKeyField());
+            if(record.getValue(getForeignKeyFieldName()) == null)
+            {
+               Serializable foreignKey = insertedRecordIterator.next().getValue(childTable.getPrimaryKeyField());
+               recordsToUpdate.add(new QRecord().withValue(table.getPrimaryKeyField(), primaryKey).withValue(getForeignKeyFieldName(), foreignKey));
+               record.setValue(getForeignKeyFieldName(), foreignKey);
+               rs.add(record);
+            }
+            else
+            {
+               rs.add(record);
+            }
+         }
+
+         ////////////////////////////////////////////////////////////////////////////
+         // update the originally inserted records to reference their new children //
+         ////////////////////////////////////////////////////////////////////////////
+         UpdateInput updateInput = new UpdateInput();
+         updateInput.setTableName(getInsertInput().getTableName());
+         updateInput.setRecords(recordsToUpdate);
+         updateInput.setTransaction(this.insertInput.getTransaction());
+         new UpdateAction().execute(updateInput);
+
+         return (rs);
+      }
+      catch(Exception e)
+      {
+         throw new RuntimeException("Error inserting new child records for new parent records", e);
+      }
+   }
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/QCodeLoader.java

@@ -0,0 +1,257 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.actions.customizers;
+
+
+import java.util.Optional;
+import java.util.function.Function;
+import com.kingsrook.qqq.backend.core.actions.automation.RecordAutomationHandler;
+import com.kingsrook.qqq.backend.core.actions.processes.BackendStep;
+import com.kingsrook.qqq.backend.core.actions.values.QCustomPossibleValueProvider;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.logging.QLogger;
+import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
+import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeType;
+import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSource;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.TableAutomationAction;
+import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
+
+
+/*******************************************************************************
+ ** Utility to load code for running QQQ customizers.
+ *******************************************************************************/
+public class QCodeLoader
+{
+   private static final QLogger LOG = QLogger.getLogger(QCodeLoader.class);
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public static <T, R> Optional<Function<T, R>> getTableCustomizerFunction(QTableMetaData table, String customizerName)
+   {
+      Optional<QCodeReference> codeReference = table.getCustomizer(customizerName);
+      if(codeReference.isPresent())
+      {
+         return (Optional.ofNullable(QCodeLoader.getFunction(codeReference.get())));
+      }
+      return (Optional.empty());
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public static <T> Optional<T> getTableCustomizer(Class<T> expectedClass, QTableMetaData table, String customizerName)
+   {
+      Optional<QCodeReference> codeReference = table.getCustomizer(customizerName);
+      if(codeReference.isPresent())
+      {
+         return (Optional.ofNullable(QCodeLoader.getAdHoc(expectedClass, codeReference.get())));
+      }
+      return (Optional.empty());
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @SuppressWarnings("unchecked")
+   public static <T, R> Function<T, R> getFunction(QCodeReference codeReference)
+   {
+      if(codeReference == null)
+      {
+         return (null);
+      }
+
+      if(!codeReference.getCodeType().equals(QCodeType.JAVA))
+      {
+         ///////////////////////////////////////////////////////////////////////////////////////
+         // todo - 1) support more languages, 2) wrap them w/ java Functions here, 3) profit! //
+         ///////////////////////////////////////////////////////////////////////////////////////
+         throw (new IllegalArgumentException("Only JAVA customizers are supported at this time."));
+      }
+
+      try
+      {
+         Class<?> customizerClass = Class.forName(codeReference.getName());
+         return ((Function<T, R>) customizerClass.getConstructor().newInstance());
+      }
+      catch(Exception e)
+      {
+         LOG.error("Error initializing customizer", logPair("codeReference", codeReference), e);
+
+         //////////////////////////////////////////////////////////////////////////////////////////////////////////
+         // return null here - under the assumption that during normal run-time operations, we'll never hit here //
+         // as we'll want to validate all functions in the instance validator at startup time (and IT will throw //
+         // if it finds an invalid code reference                                                                //
+         //////////////////////////////////////////////////////////////////////////////////////////////////////////
+         return (null);
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @SuppressWarnings("unchecked")
+   public static <T extends BackendStep> T getBackendStep(Class<T> expectedType, QCodeReference codeReference)
+   {
+      if(codeReference == null)
+      {
+         return (null);
+      }
+
+      if(!codeReference.getCodeType().equals(QCodeType.JAVA))
+      {
+         ///////////////////////////////////////////////////////////////////////////////////////
+         // todo - 1) support more languages, 2) wrap them w/ java Functions here, 3) profit! //
+         ///////////////////////////////////////////////////////////////////////////////////////
+         throw (new IllegalArgumentException("Only JAVA BackendSteps are supported at this time."));
+      }
+
+      try
+      {
+         Class<?> customizerClass = Class.forName(codeReference.getName());
+         return ((T) customizerClass.getConstructor().newInstance());
+      }
+      catch(Exception e)
+      {
+         LOG.error("Error initializing customizer", logPair("codeReference", codeReference), e);
+
+         //////////////////////////////////////////////////////////////////////////////////////////////////////////
+         // return null here - under the assumption that during normal run-time operations, we'll never hit here //
+         // as we'll want to validate all functions in the instance validator at startup time (and IT will throw //
+         // if it finds an invalid code reference                                                                //
+         //////////////////////////////////////////////////////////////////////////////////////////////////////////
+         return (null);
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @SuppressWarnings("unchecked")
+   public static <T> T getAdHoc(Class<T> expectedType, QCodeReference codeReference)
+   {
+      if(codeReference == null)
+      {
+         return (null);
+      }
+
+      if(!codeReference.getCodeType().equals(QCodeType.JAVA))
+      {
+         ///////////////////////////////////////////////////////////////////////////////////////
+         // todo - 1) support more languages, 2) wrap them w/ java Functions here, 3) profit! //
+         ///////////////////////////////////////////////////////////////////////////////////////
+         throw (new IllegalArgumentException("Only JAVA code references are supported at this time."));
+      }
+
+      try
+      {
+         Class<?> customizerClass = Class.forName(codeReference.getName());
+         return ((T) customizerClass.getConstructor().newInstance());
+      }
+      catch(Exception e)
+      {
+         LOG.error("Error initializing customizer", logPair("codeReference", codeReference), e);
+
+         //////////////////////////////////////////////////////////////////////////////////////////////////////////
+         // return null here - under the assumption that during normal run-time operations, we'll never hit here //
+         // as we'll want to validate all functions in the instance validator at startup time (and IT will throw //
+         // if it finds an invalid code reference                                                                //
+         //////////////////////////////////////////////////////////////////////////////////////////////////////////
+         return (null);
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public static RecordAutomationHandler getRecordAutomationHandler(TableAutomationAction action) throws QException
+   {
+      try
+      {
+         QCodeReference codeReference = action.getCodeReference();
+         if(!codeReference.getCodeType().equals(QCodeType.JAVA))
+         {
+            ///////////////////////////////////////////////////////////////////////////////////////
+            // todo - 1) support more languages, 2) wrap them w/ java Functions here, 3) profit! //
+            ///////////////////////////////////////////////////////////////////////////////////////
+            throw (new IllegalArgumentException("Only JAVA customizers are supported at this time."));
+         }
+
+         Class<?> codeClass  = Class.forName(codeReference.getName());
+         Object   codeObject = codeClass.getConstructor().newInstance();
+         if(!(codeObject instanceof RecordAutomationHandler recordAutomationHandler))
+         {
+            throw (new QException("The supplied code [" + codeClass.getName() + "] is not an instance of RecordAutomationHandler"));
+         }
+         return (recordAutomationHandler);
+      }
+      catch(QException qe)
+      {
+         throw (qe);
+      }
+      catch(Exception e)
+      {
+         throw (new QException("Error getting record automation handler for action [" + action.getName() + "]", e));
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public static QCustomPossibleValueProvider getCustomPossibleValueProvider(QPossibleValueSource possibleValueSource) throws QException
+   {
+      try
+      {
+         Class<?> codeClass  = Class.forName(possibleValueSource.getCustomCodeReference().getName());
+         Object   codeObject = codeClass.getConstructor().newInstance();
+         if(!(codeObject instanceof QCustomPossibleValueProvider customPossibleValueProvider))
+         {
+            throw (new QException("The supplied code [" + codeClass.getName() + "] is not an instance of QCustomPossibleValueProvider"));
+         }
+         return (customPossibleValueProvider);
+      }
+      catch(QException qe)
+      {
+         throw (qe);
+      }
+      catch(Exception e)
+      {
+         throw (new QException("Error getting custom possible value provider for PVS [" + possibleValueSource.getName() + "]", e));
+      }
+   }
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/TableCustomizer.java

@@ -0,0 +1,94 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.actions.customizers;
+
+
+import java.util.function.Consumer;
+
+
+/*******************************************************************************
+ ** Object used by TableCustomizers enum (and similar enums in backend modules)
+ ** to assist with definition and validation of Customizers applied to tables.
+ *******************************************************************************/
+public class TableCustomizer
+{
+   private final String           role;
+   private final Class<?>         expectedType;
+   private final Consumer<Object> validationFunction;
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public TableCustomizer(String role, Class<?> expectedType, Consumer<Object> validationFunction)
+   {
+      this.role = role;
+      this.expectedType = expectedType;
+      this.validationFunction = validationFunction;
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public TableCustomizer(String role, Class<?> expectedType)
+   {
+      this.role = role;
+      this.expectedType = expectedType;
+      this.validationFunction = null;
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for role
+    **
+    *******************************************************************************/
+   public String getRole()
+   {
+      return role;
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for expectedType
+    **
+    *******************************************************************************/
+   public Class<?> getExpectedType()
+   {
+      return expectedType;
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for validationFunction
+    **
+    *******************************************************************************/
+   public Consumer<Object> getValidationFunction()
+   {
+      return validationFunction;
+   }
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/TableCustomizers.java

@@ -0,0 +1,99 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.actions.customizers;
+
+
+/*******************************************************************************
+ ** Enum definition of possible table customizers - "roles" for custom code that
+ ** can be applied to tables.
+ **
+ ** Works with TableCustomizer (singular version of this name) objects, during
+ ** instance validation, to provide validation of the referenced code (and to
+ ** make such validation from sub-backend-modules possible in the future).
+ **
+ ** The idea of the 3rd argument here is to provide a way that we can enforce
+ ** the type-parameters for the custom code.  E.g., if it's a Function - how
+ ** can we check at run-time that the type-params are correct?  We couldn't find
+ ** how to do this "reflectively", so we can instead try to run the custom code,
+ ** passing it objects of the type that this customizer expects, and a validation
+ ** error will raise upon ClassCastException...  This maybe could improve!
+ *******************************************************************************/
+public enum TableCustomizers
+{
+   POST_QUERY_RECORD(new TableCustomizer("postQueryRecord", AbstractPostQueryCustomizer.class)),
+   POST_INSERT_RECORD(new TableCustomizer("postInsertRecord", AbstractPostInsertCustomizer.class));
+
+
+   private final TableCustomizer tableCustomizer;
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   TableCustomizers(TableCustomizer tableCustomizer)
+   {
+      this.tableCustomizer = tableCustomizer;
+   }
+
+
+
+   /*******************************************************************************
+    ** Get the TableCustomer for a given role (e.g., the role used in meta-data, not
+    ** the enum-constant name).
+    *******************************************************************************/
+   public static TableCustomizers forRole(String name)
+   {
+      for(TableCustomizers value : values())
+      {
+         if(value.tableCustomizer.getRole().equals(name))
+         {
+            return (value);
+         }
+      }
+
+      return (null);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for tableCustomizer
+    **
+    *******************************************************************************/
+   public TableCustomizer getTableCustomizer()
+   {
+      return tableCustomizer;
+   }
+
+
+
+   /*******************************************************************************
+    ** get the role from the tableCustomizer
+    **
+    *******************************************************************************/
+   public String getRole()
+   {
+      return (tableCustomizer.getRole());
+   }
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/dashboard/AbstractHTMLWidgetRenderer.java

@@ -0,0 +1,422 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.actions.dashboard;
+
+
+import java.io.Serializable;
+import java.net.URLEncoder;
+import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import java.util.stream.Collectors;
+import com.kingsrook.qqq.backend.core.actions.dashboard.widgets.AbstractWidgetRenderer;
+import com.kingsrook.qqq.backend.core.actions.values.QValueFormatter;
+import com.kingsrook.qqq.backend.core.context.QContext;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.actions.AbstractActionInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
+import com.kingsrook.qqq.backend.core.model.actions.widgets.RenderWidgetInput;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.DisplayFormat;
+import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
+import com.kingsrook.qqq.backend.core.utils.JsonUtils;
+import com.kingsrook.qqq.backend.core.utils.StringUtils;
+
+
+/*******************************************************************************
+ ** Base class for rendering qqq HTML dashboard widgets
+ **
+ *******************************************************************************/
+public abstract class AbstractHTMLWidgetRenderer extends AbstractWidgetRenderer
+{
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public static String openTopLevelBulletList()
+   {
+      return ("""
+         <div style="padding-left: 2rem;">
+            <ul>""");
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public static String closeTopLevelBulletList()
+   {
+      return ("""
+            </ul>
+         </div>""");
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   protected String bulletItalics(String text)
+   {
+      return ("<li><i>" + text + "</i></li>");
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   protected String bulletLink(String href, String text)
+   {
+      return ("<li><a href=\"" + href + "\">" + text + "</a></li>");
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   protected String bulletNameLink(String name, String href, String text)
+   {
+      return (bulletNameValue(name, "<a href=\"" + href + "\">" + text + "</a>"));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   protected String bulletNameValue(String name, String value)
+   {
+      return ("<li><b>" + name + "</b> &nbsp; " + Objects.requireNonNullElse(value, "--") + "</li>");
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public static String linkTableBulkLoad(RenderWidgetInput input, String tableName) throws QException
+   {
+      String tablePath = QContext.getQInstance().getTablePath(tableName);
+      return (tablePath + "/" + tableName + ".bulkInsert");
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public static String linkTableBulkLoadChildren(RenderWidgetInput input, String tableName) throws QException
+   {
+      String tablePath = QContext.getQInstance().getTablePath(tableName);
+      if(tablePath == null)
+      {
+         return (null);
+      }
+
+      return ("#/launchProcess=" + tableName + ".bulkInsert");
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public static String linkTableCreate(RenderWidgetInput input, String tableName) throws QException
+   {
+      String tablePath = QContext.getQInstance().getTablePath(tableName);
+      return (tablePath + "/create");
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public static String linkTableCreateWithDefaultValues(RenderWidgetInput input, String tableName, Map<String, Serializable> defaultValues) throws QException
+   {
+      String tablePath = QContext.getQInstance().getTablePath(tableName);
+      return (tablePath + "/create?defaultValues=" + URLEncoder.encode(JsonUtils.toJson(defaultValues), Charset.defaultCharset()));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public static String getCountLink(RenderWidgetInput input, String tableName, QQueryFilter filter, int count) throws QException
+   {
+      String totalString = QValueFormatter.formatValue(DisplayFormat.COMMAS, count);
+      String tablePath   = QContext.getQInstance().getTablePath(tableName);
+      if(tablePath == null || filter == null)
+      {
+         return (totalString);
+      }
+      return ("<a href='" + tablePath + "?filter=" + JsonUtils.toJson(filter) + "'>" + totalString + "</a>");
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public static void addTableFilterToListIfPermissed(RenderWidgetInput input, String tableName, List<String> urls, QQueryFilter filter) throws QException
+   {
+      String tablePath = QContext.getQInstance().getTablePath(tableName);
+      if(tablePath == null)
+      {
+         return;
+      }
+
+      urls.add(tablePath + "?filter=" + JsonUtils.toJson(filter));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public static String linkTableFilterUnencoded(RenderWidgetInput input, String tableName, QQueryFilter filter) throws QException
+   {
+      String tablePath = QContext.getQInstance().getTablePath(tableName);
+      if(tablePath == null)
+      {
+         return (null);
+      }
+
+      return (tablePath + "?filter=" + JsonUtils.toJson(filter));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public static String linkTableFilter(RenderWidgetInput input, String tableName, QQueryFilter filter) throws QException
+   {
+      String tablePath = QContext.getQInstance().getTablePath(tableName);
+      if(tablePath == null)
+      {
+         return (null);
+      }
+
+      return (tablePath + "?filter=" + URLEncoder.encode(JsonUtils.toJson(filter), Charset.defaultCharset()));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public static String aHrefTableFilterNoOfRecords(RenderWidgetInput input, String tableName, QQueryFilter filter, Integer noOfRecords, String singularLabel, String pluralLabel) throws QException
+   {
+      return (aHrefTableFilterNoOfRecords(input, tableName, filter, noOfRecords, singularLabel, pluralLabel, false));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public static String aHrefTableFilterNoOfRecords(RenderWidgetInput input, String tableName, QQueryFilter filter, Integer noOfRecords, String singularLabel, String pluralLabel, boolean onlyLinkCount) throws QException
+   {
+      String plural      = StringUtils.plural(noOfRecords, singularLabel, pluralLabel);
+      String countString = QValueFormatter.formatValue(DisplayFormat.COMMAS, noOfRecords);
+      String displayText = StringUtils.hasContent(plural) ? (" " + plural) : "";
+      String tablePath   = QContext.getQInstance().getTablePath(tableName);
+      if(tablePath == null)
+      {
+         return (countString + displayText);
+      }
+
+      String href = linkTableFilter(input, tableName, filter);
+      if(onlyLinkCount)
+      {
+         return ("<a href=\"" + href + "\">" + countString + "</a>" + displayText);
+      }
+      else
+      {
+         return ("<a href=\"" + href + "\">" + countString + displayText + "</a>");
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public static String aHrefViewRecord(RenderWidgetInput input, String tableName, Serializable id, String linkText) throws QException
+   {
+      String tablePath = QContext.getQInstance().getTablePath(tableName);
+      if(tablePath == null)
+      {
+         return (linkText);
+      }
+
+      return ("<a href=\"" + linkRecordView(input, tableName, id) + "\">" + linkText + "</a>");
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public static String linkRecordEdit(AbstractActionInput input, String tableName, Serializable recordId) throws QException
+   {
+      String tablePath = QContext.getQInstance().getTablePath(tableName);
+      return (tablePath + "/" + recordId + "/edit");
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public static String linkRecordView(AbstractActionInput input, String tableName, Serializable recordId) throws QException
+   {
+      String tablePath = QContext.getQInstance().getTablePath(tableName);
+      if(tablePath == null)
+      {
+         return (null);
+      }
+
+      return (tablePath + "/" + recordId);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public static String linkProcessForFilter(AbstractActionInput input, String processName, QQueryFilter filter) throws QException
+   {
+      QProcessMetaData process = QContext.getQInstance().getProcess(processName);
+      if(process == null)
+      {
+         return (null);
+      }
+      String tableName = process.getTableName();
+      if(tableName == null)
+      {
+         return (null);
+      }
+
+      String tablePath = QContext.getQInstance().getTablePath(tableName);
+      return (tablePath + "/" + processName + "?recordsParam=filterJSON&filterJSON=" + URLEncoder.encode(JsonUtils.toJson(filter), StandardCharsets.UTF_8));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public static String linkProcessForRecord(AbstractActionInput input, String processName, Serializable recordId) throws QException
+   {
+      QProcessMetaData process   = QContext.getQInstance().getProcess(processName);
+      String           tableName = process.getTableName();
+      String           tablePath = QContext.getQInstance().getTablePath(tableName);
+
+      return (tablePath + "/" + recordId + "/" + processName);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public static String linkTableCreateChild(RenderWidgetInput input, String childTableName, Map<String, Serializable> defaultValues) throws QException
+   {
+      return (linkTableCreateChild(input, childTableName, defaultValues, defaultValues.keySet()));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public static String aHrefTableCreateChild(RenderWidgetInput input, String childTableName, Map<String, Serializable> defaultValues) throws QException
+   {
+      return (aHrefTableCreateChild(input, childTableName, defaultValues, defaultValues.keySet()));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public static String linkTableCreateChild(RenderWidgetInput input, String childTableName, Map<String, Serializable> defaultValues, Set<String> disabledFields) throws QException
+   {
+      String tablePath = QContext.getQInstance().getTablePath(childTableName);
+      if(tablePath == null)
+      {
+         return (null);
+      }
+
+      Map<String, Integer> disabledFieldsMap = disabledFields.stream().collect(Collectors.toMap(k -> k, k -> 1));
+
+      return ("#/createChild=" + childTableName
+         + "/defaultValues=" + URLEncoder.encode(JsonUtils.toJson(defaultValues), StandardCharsets.UTF_8).replaceAll("\\+", "%20")
+         + "/disabledFields=" + URLEncoder.encode(JsonUtils.toJson(disabledFieldsMap), StandardCharsets.UTF_8).replaceAll("\\+", "%20"));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public static String getChipElement(String icon, String label, String color) throws QException
+   {
+      color = color != null ? color : "info";
+      color = StringUtils.ucFirst(color);
+
+      String html = "<span style='display: flex;'>";
+      html += "<div style='overflow: hidden; flex: none; display: flex; align-content: flex-start; align-items: center; height: 24px; padding-right: 8px; font-size: 13px; font-weight: 500; border: 1px solid; border-radius: 16px; color: " + color + "'>";
+      if(icon != null)
+      {
+         html += "<span style='font-size: 16px; padding: 5px' class='material-icons-round notranslate MuiIcon-root MuiIcon-fontSizeInherit MuiChip-icon MuiChip-iconSmall MuiChip-iconColor" + color + "'>" + icon + "</span>";
+      }
+      html += "<span class='MuiChip-label MuiChip-labelSmall'>" + label + "</span></div></span>";
+      return (html);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public static String aHrefTableCreateChild(RenderWidgetInput input, String childTableName, Map<String, Serializable> defaultValues, Set<String> disabledFields) throws QException
+   {
+      String tablePath = QContext.getQInstance().getTablePath(childTableName);
+      if(tablePath == null)
+      {
+         return (null);
+      }
+
+      return ("<a href=\"" + linkTableCreateChild(input, childTableName, defaultValues, defaultValues.keySet()) + "\">Create new</a>");
+   }
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/dashboard/RenderWidgetAction.java

@@ -0,0 +1,62 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.actions.dashboard;
+
+
+import java.io.Serializable;
+import java.util.Map;
+import com.kingsrook.qqq.backend.core.actions.ActionHelper;
+import com.kingsrook.qqq.backend.core.actions.customizers.QCodeLoader;
+import com.kingsrook.qqq.backend.core.actions.dashboard.widgets.AbstractWidgetRenderer;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.actions.widgets.RenderWidgetInput;
+import com.kingsrook.qqq.backend.core.model.actions.widgets.RenderWidgetOutput;
+import com.kingsrook.qqq.backend.core.utils.ValueUtils;
+
+
+/*******************************************************************************
+ ** Class for loading widget implementation code and rendering of widgets
+ **
+ *******************************************************************************/
+public class RenderWidgetAction
+{
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public RenderWidgetOutput execute(RenderWidgetInput input) throws QException
+   {
+      ActionHelper.validateSession(input);
+
+      AbstractWidgetRenderer widgetRenderer = QCodeLoader.getAdHoc(AbstractWidgetRenderer.class, input.getWidgetMetaData().getCodeReference());
+
+      ///////////////////////////////////////////////////////////////
+      // move default values from meta data into this render input //
+      ///////////////////////////////////////////////////////////////
+      for(Map.Entry<String, Serializable> entry : input.getWidgetMetaData().getDefaultValues().entrySet())
+      {
+         input.addQueryParam(entry.getKey(), ValueUtils.getValueAsString(entry.getValue()));
+      }
+
+      return (widgetRenderer.render(input));
+   }
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/dashboard/widgets/AbstractWidgetRenderer.java

@@ -0,0 +1,164 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.actions.dashboard.widgets;
+
+
+import java.time.ZoneId;
+import java.time.format.DateTimeFormatter;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import com.kingsrook.qqq.backend.core.actions.values.QValueFormatter;
+import com.kingsrook.qqq.backend.core.actions.values.SearchPossibleValueSourceAction;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
+import com.kingsrook.qqq.backend.core.model.actions.values.SearchPossibleValueSourceInput;
+import com.kingsrook.qqq.backend.core.model.actions.values.SearchPossibleValueSourceOutput;
+import com.kingsrook.qqq.backend.core.model.actions.widgets.RenderWidgetInput;
+import com.kingsrook.qqq.backend.core.model.actions.widgets.RenderWidgetOutput;
+import com.kingsrook.qqq.backend.core.model.dashboard.widgets.QWidgetData;
+import com.kingsrook.qqq.backend.core.model.metadata.dashboard.QWidgetMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.dashboard.WidgetDropdownData;
+import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValue;
+import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSource;
+import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
+import com.kingsrook.qqq.backend.core.utils.StringUtils;
+
+
+/*******************************************************************************
+ ** Base class for rendering qqq dashboard widgets
+ **
+ *******************************************************************************/
+public abstract class AbstractWidgetRenderer
+{
+   public static final QValueFormatter   valueFormatter = new QValueFormatter();
+   public static final DateTimeFormatter dateFormatter  = DateTimeFormatter.ofPattern("yyyy-MM-dd").withZone(ZoneId.systemDefault());
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public abstract RenderWidgetOutput render(RenderWidgetInput input) throws QException;
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   protected boolean setupDropdowns(RenderWidgetInput input, QWidgetMetaData metaData, QWidgetData widgetData) throws QException
+   {
+      List<List<Map<String, String>>> pvsData                   = new ArrayList<>();
+      List<String>                    pvsLabels                 = new ArrayList<>();
+      List<String>                    pvsNames                  = new ArrayList<>();
+      List<String>                    missingRequiredSelections = new ArrayList<>();
+      for(WidgetDropdownData dropdownData : CollectionUtils.nonNullList(metaData.getDropdowns()))
+      {
+         String               possibleValueSourceName = dropdownData.getPossibleValueSourceName();
+         QPossibleValueSource possibleValueSource     = input.getInstance().getPossibleValueSource(possibleValueSourceName);
+
+         /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+         // this looks complicated, but is just look for a label in the dropdown data and if found use it,                                                                                  //
+         // otherwise look for label in PVS and if found use that, otherwise just use the PVS name                                                                                          //
+         /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+         String pvsLabel = dropdownData.getLabel() != null ? dropdownData.getLabel() : (possibleValueSource.getLabel() != null ? possibleValueSource.getLabel() : possibleValueSourceName);
+         pvsLabels.add(pvsLabel);
+         pvsNames.add(possibleValueSourceName);
+
+         SearchPossibleValueSourceInput pvsInput = new SearchPossibleValueSourceInput();
+         pvsInput.setPossibleValueSourceName(possibleValueSourceName);
+
+         if(dropdownData.getForeignKeyFieldName() != null)
+         {
+            ////////////////////////////////////////
+            // look for an id in the query params //
+            ////////////////////////////////////////
+            Integer id = null;
+            if(input.getQueryParams() != null && input.getQueryParams().containsKey("id") && StringUtils.hasContent(input.getQueryParams().get("id")))
+            {
+               id = Integer.parseInt(input.getQueryParams().get("id"));
+            }
+            if(id != null)
+            {
+               pvsInput.setDefaultQueryFilter(new QQueryFilter().withCriteria(
+                  new QFilterCriteria(
+                     dropdownData.getForeignKeyFieldName(),
+                     QCriteriaOperator.EQUALS,
+                     id)));
+            }
+         }
+
+         SearchPossibleValueSourceOutput output = new SearchPossibleValueSourceAction().execute(pvsInput);
+
+         List<Map<String, String>> dropdownOptionList = new ArrayList<>();
+         pvsData.add(dropdownOptionList);
+
+         //////////////////////////////////////////
+         // sort results, dedupe, and add to map //
+         //////////////////////////////////////////
+         Set<String> exists = new HashSet<>();
+         output.getResults().removeIf(pvs -> !exists.add(pvs.getLabel()));
+         for(QPossibleValue<?> possibleValue : output.getResults())
+         {
+            dropdownOptionList.add(Map.of(
+               "id", String.valueOf(possibleValue.getId()),
+               "label", possibleValue.getLabel()
+            ));
+         }
+
+         ////////////////////////////////////////////////////////////////////////////////////////////////////////////
+         // because we know the dropdowns and what the field names will be when something is selected, we can make //
+         // sure that something has been selected, and if not, display a message that a selection needs made       //
+         ////////////////////////////////////////////////////////////////////////////////////////////////////////////
+         if(dropdownData.getIsRequired())
+         {
+            if(!input.getQueryParams().containsKey(possibleValueSourceName) || !StringUtils.hasContent(input.getQueryParams().get(possibleValueSourceName)))
+            {
+               missingRequiredSelections.add(pvsLabel);
+            }
+         }
+      }
+
+      widgetData.setDropdownNameList(pvsNames);
+      widgetData.setDropdownLabelList(pvsLabels);
+      widgetData.setDropdownDataList(pvsData);
+
+      ////////////////////////////////////////////////////////////////////////////////
+      // if there are any missing required dropdowns, build up a message to display //
+      ////////////////////////////////////////////////////////////////////////////////
+      if(missingRequiredSelections.size() > 0)
+      {
+         StringBuilder sb = new StringBuilder("Please select a ").append(StringUtils.joinWithCommasAndAnd(missingRequiredSelections));
+         sb.append(" from the ").append(StringUtils.plural(missingRequiredSelections.size(), "dropdown", "dropdowns")).append(" above.");
+         widgetData.setDropdownNeedsSelectedText(sb.toString());
+         return (false);
+      }
+      else
+      {
+         return (true);
+      }
+   }
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/dashboard/widgets/ChildRecordListRenderer.java

@@ -0,0 +1,236 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.actions.dashboard.widgets;
+
+
+import java.io.Serializable;
+import java.net.URLEncoder;
+import java.nio.charset.Charset;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import com.kingsrook.qqq.backend.core.actions.tables.GetAction;
+import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.exceptions.QNotFoundException;
+import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetOutput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput;
+import com.kingsrook.qqq.backend.core.model.actions.widgets.RenderWidgetInput;
+import com.kingsrook.qqq.backend.core.model.actions.widgets.RenderWidgetOutput;
+import com.kingsrook.qqq.backend.core.model.dashboard.widgets.ChildRecordListData;
+import com.kingsrook.qqq.backend.core.model.dashboard.widgets.WidgetType;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
+import com.kingsrook.qqq.backend.core.model.metadata.dashboard.AbstractWidgetMetaDataBuilder;
+import com.kingsrook.qqq.backend.core.model.metadata.dashboard.QWidgetMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinOn;
+import com.kingsrook.qqq.backend.core.model.metadata.joins.QJoinMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+import com.kingsrook.qqq.backend.core.utils.JsonUtils;
+import com.kingsrook.qqq.backend.core.utils.StringUtils;
+import com.kingsrook.qqq.backend.core.utils.ValueUtils;
+import org.apache.commons.lang.BooleanUtils;
+
+
+/*******************************************************************************
+ ** Generic widget for display a list of child records.
+ *******************************************************************************/
+public class ChildRecordListRenderer extends AbstractWidgetRenderer
+{
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public static Builder widgetMetaDataBuilder(QJoinMetaData join)
+   {
+      return (new Builder(new QWidgetMetaData()
+         .withName(join.getName())
+         .withIsCard(true)
+         .withCodeReference(new QCodeReference(ChildRecordListRenderer.class, null))
+         .withType(WidgetType.CHILD_RECORD_LIST.getType())
+         .withDefaultValue("joinName", join.getName())));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public static class Builder extends AbstractWidgetMetaDataBuilder
+   {
+
+      /*******************************************************************************
+       ** Constructor
+       **
+       *******************************************************************************/
+      public Builder(QWidgetMetaData widgetMetaData)
+      {
+         super(widgetMetaData);
+      }
+
+
+
+      /*******************************************************************************
+       **
+       *******************************************************************************/
+      public Builder withName(String name)
+      {
+         widgetMetaData.setName(name);
+         return (this);
+      }
+
+
+
+      /*******************************************************************************
+       **
+       *******************************************************************************/
+      public Builder withLabel(String label)
+      {
+         widgetMetaData.setLabel(label);
+         return (this);
+      }
+
+
+
+      /*******************************************************************************
+       **
+       *******************************************************************************/
+      public Builder withMaxRows(Integer maxRows)
+      {
+         widgetMetaData.withDefaultValue("maxRows", maxRows);
+         return (this);
+      }
+
+
+
+      /*******************************************************************************
+       **
+       *******************************************************************************/
+      public Builder withCanAddChildRecord(boolean b)
+      {
+         widgetMetaData.withDefaultValue("canAddChildRecord", true);
+         return (this);
+      }
+
+
+
+      /*******************************************************************************
+       **
+       *******************************************************************************/
+      public Builder withDisabledFieldsForNewChildRecords(Set<String> disabledFieldsForNewChildRecords)
+      {
+         widgetMetaData.withDefaultValue("disabledFieldsForNewChildRecords", new HashSet<>(disabledFieldsForNewChildRecords));
+         return (this);
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public RenderWidgetOutput render(RenderWidgetInput input) throws QException
+   {
+      String        widgetLabel = input.getQueryParams().get("widgetLabel");
+      String        joinName    = input.getQueryParams().get("joinName");
+      QJoinMetaData join        = input.getInstance().getJoin(joinName);
+      String        id          = input.getQueryParams().get("id");
+
+      Integer maxRows = null;
+      if(StringUtils.hasContent(input.getQueryParams().get("maxRows")))
+      {
+         maxRows = ValueUtils.getValueAsInteger(input.getQueryParams().get("maxRows"));
+      }
+
+      ////////////////////////////////////////////////////////
+      // fetch the record that we're getting children for.  //
+      // e.g., the left-side of the join, with the input id //
+      ////////////////////////////////////////////////////////
+      GetInput getInput = new GetInput();
+      getInput.setTableName(join.getLeftTable());
+      getInput.setPrimaryKey(id);
+      GetOutput getOutput = new GetAction().execute(getInput);
+      QRecord   record    = getOutput.getRecord();
+
+      if(record == null)
+      {
+         QTableMetaData table = input.getInstance().getTable(join.getLeftTable());
+         throw (new QNotFoundException("Could not find " + (table == null ? "" : table.getLabel()) + " with primary key " + id));
+      }
+
+      ////////////////////////////////////////////////////////////////////
+      // set up the query - for the table on the right side of the join //
+      ////////////////////////////////////////////////////////////////////
+      QQueryFilter filter = new QQueryFilter();
+      for(JoinOn joinOn : join.getJoinOns())
+      {
+         filter.addCriteria(new QFilterCriteria(joinOn.getRightField(), QCriteriaOperator.EQUALS, List.of(record.getValue(joinOn.getLeftField()))));
+      }
+      filter.setOrderBys(join.getOrderBys());
+
+      QueryInput queryInput = new QueryInput();
+      queryInput.setTableName(join.getRightTable());
+      queryInput.setShouldTranslatePossibleValues(true);
+      queryInput.setShouldGenerateDisplayValues(true);
+      queryInput.setFilter(filter);
+      queryInput.setLimit(maxRows);
+      QueryOutput queryOutput = new QueryAction().execute(queryInput);
+
+      QTableMetaData table       = input.getInstance().getTable(join.getRightTable());
+      String         tablePath   = input.getInstance().getTablePath(table.getName());
+      String         viewAllLink = tablePath == null ? null : (tablePath + "?filter=" + URLEncoder.encode(JsonUtils.toJson(filter), Charset.defaultCharset()));
+
+      ChildRecordListData widgetData = new ChildRecordListData(widgetLabel, queryOutput, table, tablePath, viewAllLink);
+
+      if(BooleanUtils.isTrue(ValueUtils.getValueAsBoolean(input.getQueryParams().get("canAddChildRecord"))))
+      {
+         widgetData.setCanAddChildRecord(true);
+
+         //////////////////////////////////////////////////////////
+         // new child records must have values from the join-ons //
+         //////////////////////////////////////////////////////////
+         Map<String, Serializable> defaultValuesForNewChildRecords = new HashMap<>();
+         for(JoinOn joinOn : join.getJoinOns())
+         {
+            defaultValuesForNewChildRecords.put(joinOn.getRightField(), record.getValue(joinOn.getLeftField()));
+         }
+         widgetData.setDefaultValuesForNewChildRecords(defaultValuesForNewChildRecords);
+
+         Map<String, Serializable> widgetValues = input.getWidgetMetaData().getDefaultValues();
+         if(widgetValues.containsKey("disabledFieldsForNewChildRecords"))
+         {
+            widgetData.setDisabledFieldsForNewChildRecords((Set<String>) widgetValues.get("disabledFieldsForNewChildRecords"));
+         }
+      }
+
+      return (new RenderWidgetOutput(widgetData));
+   }
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/dashboard/widgets/DateTimeGroupBy.java

@@ -0,0 +1,249 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2023.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.actions.dashboard.widgets;
+
+
+import java.time.DayOfWeek;
+import java.time.Instant;
+import java.time.ZoneId;
+import java.time.ZonedDateTime;
+import java.time.format.DateTimeFormatter;
+import java.time.temporal.ChronoField;
+import java.time.temporal.ChronoUnit;
+import java.time.temporal.IsoFields;
+import java.time.temporal.TemporalAdjusters;
+import com.kingsrook.qqq.backend.core.utils.StringUtils;
+import com.kingsrook.qqq.backend.core.utils.ValueUtils;
+
+
+/*******************************************************************************
+ ** Enum to define various "levels" of group-by for on dashboards that want to
+ ** group records by, e.g., year, or month, or week, or day, or hour.
+ *******************************************************************************/
+public enum DateTimeGroupBy
+{
+   /////////////////////////////////////////////////////////////////////////////////////////////////////////////
+   // note - double %'s on the time format strings here, because this is a java-format string, which will get //
+   // its '%s' replaced with a column name, and so then those %'s for the date_format need escaped as %%.     //
+   // See https://www.w3schools.com/sql/func_mysql_date_format.asp for DATE_FORMAT args                       //
+   /////////////////////////////////////////////////////////////////////////////////////////////////////////////
+   YEAR("%%Y", MillisPer.YEAR, 1, ChronoUnit.YEARS, DateTimeFormatter.ofPattern("yyyy"), DateTimeFormatter.ofPattern("yyyy")),
+   MONTH("%%Y-%%m", 2 * MillisPer.MONTH, 1, ChronoUnit.MONTHS, DateTimeFormatter.ofPattern("yyyy-MM"), DateTimeFormatter.ofPattern("MMM'.' yyyy")),
+   WEEK("%%XW%%V", 35 * MillisPer.DAY, 7, ChronoUnit.DAYS, DateTimeFormatter.ofPattern("YYYY'W'ww"), DateTimeFormatter.ofPattern("YYYY'W'w")),
+   DAY("%%Y-%%m-%%d", 36 * MillisPer.HOUR, 1, ChronoUnit.DAYS, DateTimeFormatter.ofPattern("yyyy-MM-dd"), DateTimeFormatter.ofPattern("EEE'.' M'/'d")),
+   HOUR("%%Y-%%m-%%dT%%H", 0, 1, ChronoUnit.HOURS, DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH"), DateTimeFormatter.ofPattern("h a"));
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public interface MillisPer
+   {
+      long HOUR  = 60 * 60 * 1000;
+      long DAY   = 24 * HOUR;
+      long WEEK  = 7 * DAY;
+      long MONTH = 30 * DAY;
+      long YEAR  = 365 * DAY;
+   }
+
+
+
+   private final String            sqlDateFormat;
+   private final long              millisThreshold;
+   private final int               noOfChronoUnitsToAdd;
+   private final ChronoUnit        chronoUnitToAdd;
+   private final DateTimeFormatter selectedStringFormatter;
+   private final DateTimeFormatter humanStringFormatter;
+
+
+
+   /*******************************************************************************
+    ** Constructor
+    **
+    *******************************************************************************/
+   DateTimeGroupBy(String sqlDateFormat, long millisThreshold, int noOfChronoUnitsToAdd, ChronoUnit chronoUnitToAdd, DateTimeFormatter selectedStringFormatter, DateTimeFormatter humanStringFormatter)
+   {
+      this.sqlDateFormat = sqlDateFormat;
+      this.millisThreshold = millisThreshold;
+      this.noOfChronoUnitsToAdd = noOfChronoUnitsToAdd;
+      this.chronoUnitToAdd = chronoUnitToAdd;
+      this.selectedStringFormatter = selectedStringFormatter;
+      this.humanStringFormatter = humanStringFormatter;
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public String getSqlExpression()
+   {
+      ZoneId sessionOrInstanceZoneId = ValueUtils.getSessionOrInstanceZoneId();
+      String targetTimezone          = sessionOrInstanceZoneId.toString();
+
+      if("Z".equals(targetTimezone) || !StringUtils.hasContent(targetTimezone))
+      {
+         targetTimezone = "UTC";
+      }
+
+      //////////////////////////////////////////////////////////////////////////////////////////////////////
+      // if we only had a timezone offset (not a zone name/id), then the zoneId's toString will look like //
+      // UTC-05:00.  MySQL doesn't want that, so, strip away the leading UTC, to just get -05:00          //
+      //////////////////////////////////////////////////////////////////////////////////////////////////////
+      if((targetTimezone.startsWith("UTC-") || targetTimezone.startsWith("UTC+")) && targetTimezone.length() > 5)
+      {
+         targetTimezone = targetTimezone.substring(3);
+      }
+
+      return "DATE_FORMAT(CONVERT_TZ(%s, 'UTC', '" + targetTimezone + "'), '" + sqlDateFormat + "')";
+
+      /*
+      if(this == WEEK)
+      {
+         return "YEARWEEK(CONVERT_TZ(%s, 'UTC', '" + targetTimezone + "'), 6)";
+      }
+      else
+      {
+         return "DATE_FORMAT(CONVERT_TZ(%s, 'UTC', '" + targetTimezone + "'), '" + sqlDateFormat + "')";
+      }
+      */
+   }
+
+
+
+   /*******************************************************************************
+    ** get an instance of this enum, based on start & end instants - look at the #
+    ** of millis between them, and return the first enum value w/ a millisThreshold
+    ** under that difference.  Default to HOUR.
+    *******************************************************************************/
+   public static DateTimeGroupBy selectFromStartAndEndTimes(Instant start, Instant end)
+   {
+      long millisBetween = end.toEpochMilli() - start.toEpochMilli();
+      for(DateTimeGroupBy value : DateTimeGroupBy.values())
+      {
+         if(millisBetween > value.millisThreshold)
+         {
+            return (value);
+         }
+      }
+
+      return (HOUR);
+   }
+
+
+
+   /*******************************************************************************
+    ** Make an Instant into a string that will match what came out of the database's
+    ** DATE_FORMAT() function
+    *******************************************************************************/
+   public String makeSelectedString(Instant time)
+   {
+      ZonedDateTime zoned = time.atZone(ValueUtils.getSessionOrInstanceZoneId());
+
+      if(this == WEEK)
+      {
+         ////////////////////////////////////////////////////////////////////////////////////////////////////
+         // so, it seems like database is returning, e.g., W00-W52, but java is doing W1-W53...            //
+         // which, apparently we can compensate for by adding a week?  not sure, but results seemed right. //
+         ////////////////////////////////////////////////////////////////////////////////////////////////////
+         zoned = zoned.plusDays(7);
+         int weekYear = zoned.get(IsoFields.WEEK_BASED_YEAR);
+         int week     = zoned.get(IsoFields.WEEK_OF_WEEK_BASED_YEAR);
+         return (String.format("%04dW%02d", weekYear, week));
+      }
+
+      return (selectedStringFormatter.format(zoned));
+   }
+
+
+
+   /*******************************************************************************
+    ** Make a string to show to a user
+    *******************************************************************************/
+   public String makeHumanString(Instant instant)
+   {
+      ZonedDateTime zoned = instant.atZone(ValueUtils.getSessionOrInstanceZoneId());
+      if(this.equals(WEEK))
+      {
+         DateTimeFormatter dateTimeFormatter = DateTimeFormatter.ofPattern("M'/'d");
+
+         while(zoned.get(ChronoField.DAY_OF_WEEK) != DayOfWeek.SUNDAY.getValue())
+         {
+            ////////////////////////////////////////
+            // go backwards until sunday is found //
+            ////////////////////////////////////////
+            zoned = zoned.minus(1, ChronoUnit.DAYS);
+         }
+
+         return (dateTimeFormatter.format(zoned) + "-" + dateTimeFormatter.format(zoned.plusDays(6)));
+
+         /*
+         int               weekOfYear        = zoned.get(ChronoField.ALIGNED_WEEK_OF_YEAR);
+         ZonedDateTime     sunday            = zoned.with(IsoFields.WEEK_OF_WEEK_BASED_YEAR, weekOfYear).with(TemporalAdjusters.previousOrSame(DayOfWeek.SUNDAY));
+         ZonedDateTime     saturday          = sunday.with(TemporalAdjusters.next(DayOfWeek.SATURDAY));
+         DateTimeFormatter dateTimeFormatter = DateTimeFormatter.ofPattern("M'/'d");
+
+         return (dateTimeFormatter.format(sunday) + "-" + dateTimeFormatter.format(saturday));
+         */
+      }
+
+      return (humanStringFormatter.format(zoned));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @SuppressWarnings("checkstyle:indentation")
+   public Instant roundDown(Instant instant)
+   {
+      ZonedDateTime zoned = instant.atZone(ValueUtils.getSessionOrInstanceZoneId());
+      return switch(this)
+         {
+            case YEAR -> zoned.with(TemporalAdjusters.firstDayOfYear()).truncatedTo(ChronoUnit.DAYS).toInstant();
+            case MONTH -> zoned.with(TemporalAdjusters.firstDayOfMonth()).truncatedTo(ChronoUnit.DAYS).toInstant();
+            case WEEK ->
+            {
+               while(zoned.get(ChronoField.DAY_OF_WEEK) != DayOfWeek.SUNDAY.getValue())
+               {
+                  zoned = zoned.minusDays(1);
+               }
+               yield (zoned.truncatedTo(ChronoUnit.DAYS).toInstant());
+            }
+            case DAY -> zoned.truncatedTo(ChronoUnit.DAYS).toInstant();
+            case HOUR -> zoned.truncatedTo(ChronoUnit.HOURS).toInstant();
+         };
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public Instant increment(Instant instant)
+   {
+      ZonedDateTime zoned = instant.atZone(ValueUtils.getSessionOrInstanceZoneId());
+      return (zoned.plus(noOfChronoUnitsToAdd, chronoUnitToAdd).toInstant());
+   }
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/dashboard/widgets/DefaultWidgetRenderer.java

@@ -0,0 +1,92 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2023.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.actions.dashboard.widgets;
+
+
+import java.util.Map;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.actions.widgets.RenderWidgetInput;
+import com.kingsrook.qqq.backend.core.model.actions.widgets.RenderWidgetOutput;
+import com.kingsrook.qqq.backend.core.model.dashboard.widgets.QWidgetData;
+
+
+/*******************************************************************************
+ **
+ *******************************************************************************/
+public class DefaultWidgetRenderer extends AbstractWidgetRenderer
+{
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public RenderWidgetOutput render(RenderWidgetInput input) throws QException
+   {
+      return new RenderWidgetOutput(new DefaultWidgetData(input));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public static class DefaultWidgetData extends QWidgetData
+   {
+      private final String              type;
+      private final Map<String, String> queryParams;
+
+
+
+      /*******************************************************************************
+       ** Constructor
+       **
+       *******************************************************************************/
+      public DefaultWidgetData(RenderWidgetInput renderWidgetInput)
+      {
+         this.type = renderWidgetInput.getWidgetMetaData().getType();
+         this.queryParams = renderWidgetInput.getQueryParams();
+      }
+
+
+
+      /*******************************************************************************
+       **
+       *******************************************************************************/
+      @Override
+      public String getType()
+      {
+         return (type);
+      }
+
+
+
+      /*******************************************************************************
+       ** Getter for queryParams
+       **
+       *******************************************************************************/
+      public Map<String, String> getQueryParams()
+      {
+         return queryParams;
+      }
+   }
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/dashboard/widgets/DividerWidgetRenderer.java

@@ -0,0 +1,47 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.actions.dashboard.widgets;
+
+
+import com.kingsrook.qqq.backend.core.actions.ActionHelper;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.actions.widgets.RenderWidgetInput;
+import com.kingsrook.qqq.backend.core.model.actions.widgets.RenderWidgetOutput;
+import com.kingsrook.qqq.backend.core.model.dashboard.widgets.DividerWidgetData;
+
+
+/*******************************************************************************
+ ** Generic widget for showing a divider
+ *******************************************************************************/
+public class DividerWidgetRenderer extends AbstractWidgetRenderer
+{
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public RenderWidgetOutput render(RenderWidgetInput input) throws QException
+   {
+      ActionHelper.validateSession(input);
+      return (new RenderWidgetOutput(new DividerWidgetData()));
+   }
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/dashboard/widgets/NoCodeWidgetRenderer.java

@@ -0,0 +1,153 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2023.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.actions.dashboard.widgets;
+
+
+import java.io.Serializable;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.logging.QLogger;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
+import com.kingsrook.qqq.backend.core.model.actions.widgets.RenderWidgetInput;
+import com.kingsrook.qqq.backend.core.model.actions.widgets.RenderWidgetOutput;
+import com.kingsrook.qqq.backend.core.model.dashboard.widgets.RawHTML;
+import com.kingsrook.qqq.backend.core.model.metadata.dashboard.nocode.AbstractWidgetOutput;
+import com.kingsrook.qqq.backend.core.model.metadata.dashboard.nocode.AbstractWidgetValueSource;
+import com.kingsrook.qqq.backend.core.model.metadata.dashboard.nocode.QNoCodeWidgetMetaData;
+import com.kingsrook.qqq.backend.core.modules.backend.implementations.utils.BackendQueryFilterUtils;
+import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
+import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
+
+
+/*******************************************************************************
+ **
+ *******************************************************************************/
+public class NoCodeWidgetRenderer extends AbstractWidgetRenderer
+{
+   private static final QLogger LOG = QLogger.getLogger(NoCodeWidgetRenderer.class);
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public RenderWidgetOutput render(RenderWidgetInput input) throws QException
+   {
+      QNoCodeWidgetMetaData widgetMetaData = (QNoCodeWidgetMetaData) input.getWidgetMetaData();
+
+      Map<String, Object> context = initContext(input);
+      context.putAll(input.getQueryParams());
+
+      ///////////////////////////////////////////////
+      // populate context by evaluating all values //
+      ///////////////////////////////////////////////
+      for(AbstractWidgetValueSource valueSource : widgetMetaData.getValues())
+      {
+         try
+         {
+            LOG.trace("Computing: " + valueSource.getType() + " named " + valueSource.getName() + "...");
+            Object value = valueSource.evaluate(context, input);
+            LOG.trace("Computed: " + valueSource.getName() + " = " + value);
+            context.put(valueSource.getName(), value);
+            context.put(valueSource.getName() + ".source", valueSource);
+         }
+         catch(Exception e)
+         {
+            LOG.warn("Error evaluating widget value source", e, logPair("widgetName", input.getWidgetMetaData().getName()), logPair("valueSourceName", valueSource.getName()));
+         }
+      }
+
+      /////////////////////////////////////////////
+      // build content by evaluating all outputs //
+      /////////////////////////////////////////////
+      List<AbstractWidgetOutput> outputs = widgetMetaData.getOutputs();
+      String                     content = renderOutputs(context, outputs);
+
+      return (new RenderWidgetOutput(new RawHTML(widgetMetaData.getLabel(), content)));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public Map<String, Object> initContext(RenderWidgetInput input)
+   {
+      Map<String, Object> context = new HashMap<>();
+      context.put("utils", new NoCodeWidgetVelocityUtils(context, input));
+      context.put("input", input);
+      return context;
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public String renderOutputs(Map<String, Object> context, List<AbstractWidgetOutput> outputs) throws QException
+   {
+      StringBuilder content = new StringBuilder();
+      for(AbstractWidgetOutput output : CollectionUtils.nonNullList(outputs))
+      {
+         boolean conditionPassed = true;
+         if(output.getCondition() != null)
+         {
+            conditionPassed = evaluateCondition(output.getCondition(), context);
+         }
+
+         if(conditionPassed)
+         {
+            String render = output.render(context);
+            content.append(render);
+            LOG.trace("Condition passed, rendered: " + render);
+         }
+         else
+         {
+            LOG.trace("Condition failed - not rendering this output.");
+         }
+      }
+      return (content.toString());
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private boolean evaluateCondition(QFilterCriteria condition, Map<String, Object> context)
+   {
+      try
+      {
+         Object value = context.get(condition.getFieldName());
+         return (BackendQueryFilterUtils.doesCriteriaMatch(condition, condition.getFieldName(), (Serializable) value));
+      }
+      catch(Exception e)
+      {
+         LOG.warn("Error evaluating condition: " + condition, e);
+         return (false);
+      }
+   }
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/dashboard/widgets/NoCodeWidgetVelocityUtils.java

@@ -0,0 +1,339 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2023.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.actions.dashboard.widgets;
+
+
+import java.io.Serializable;
+import java.math.BigDecimal;
+import java.math.RoundingMode;
+import java.time.Instant;
+import java.time.ZoneId;
+import java.util.Map;
+import com.kingsrook.qqq.backend.core.actions.dashboard.AbstractHTMLWidgetRenderer;
+import com.kingsrook.qqq.backend.core.actions.values.QValueFormatter;
+import com.kingsrook.qqq.backend.core.context.QContext;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.logging.QLogger;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
+import com.kingsrook.qqq.backend.core.model.actions.widgets.RenderWidgetInput;
+import com.kingsrook.qqq.backend.core.model.metadata.dashboard.nocode.WidgetCount;
+import com.kingsrook.qqq.backend.core.utils.StringUtils;
+import com.kingsrook.qqq.backend.core.utils.ValueUtils;
+
+
+/*******************************************************************************
+ **
+ *******************************************************************************/
+public class NoCodeWidgetVelocityUtils
+{
+   private static final QLogger LOG = QLogger.getLogger(NoCodeWidgetVelocityUtils.class);
+
+   private Map<String, Object> context;
+   private RenderWidgetInput   input;
+
+
+
+   /*******************************************************************************
+    ** Constructor
+    **
+    *******************************************************************************/
+   public NoCodeWidgetVelocityUtils(Map<String, Object> context, RenderWidgetInput input)
+   {
+      this.context = context;
+      this.input = input;
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public String helpIcon()
+   {
+      return ("""
+         <span class="material-icons-round notranslate MuiIcon-root MuiIcon-fontSizeInherit" style="color: blue; position: relative; top: 6px;" aria-hidden="true">help_outline</span>
+         """);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public String errorIcon()
+   {
+      return ("""
+         <span class="material-icons-round notranslate MuiIcon-root MuiIcon-fontSizeInherit" style="color: red; position: relative; top: 6px;" aria-hidden="true">error_outline</span>
+         """);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public String warningIcon()
+   {
+      return ("""
+         <span class="material-icons-round notranslate MuiIcon-root MuiIcon-fontSizeInherit" style="color: orange; position: relative; top: 6px;" aria-hidden="true">warning</span>
+         """);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public String checkIcon()
+   {
+      return ("""
+         <span class="material-icons-round notranslate MuiIcon-root MuiIcon-fontSizeInherit" style="color: green; position: relative; top: 6px;" aria-hidden="true">check</span>
+         """);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public String pendingIcon()
+   {
+      return ("""
+         <span class="material-icons-round notranslate MuiIcon-root MuiIcon-fontSizeInherit" style="color: #0062ff; position: relative; top: 6px;" aria-hidden="true">pending</span>
+         """);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public String spanColorGreen()
+   {
+      return ("""
+         <span style="color: green;">
+         """);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public String spanColorOrange()
+   {
+      return ("""
+         <span style="color: orange;">
+         """);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public String spanColorRed()
+   {
+      return ("""
+         <span style="color: red;">
+         """);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public String plural(Integer size, String ifOne, String ifNotOne)
+   {
+      return StringUtils.plural(size, ifOne, ifNotOne);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public String formatDateTime(Instant i)
+   {
+      if(i == null)
+      {
+         return ("");
+      }
+      return QValueFormatter.formatDateTimeWithZone(i.atZone(ZoneId.of(QContext.getQInstance().getDefaultTimeZoneId())));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public String formatSecondsAsDuration(Integer seconds)
+   {
+      StringBuilder rs = new StringBuilder();
+
+      if(seconds == null)
+      {
+         return ("");
+      }
+
+      int secondsPerDay = 24 * 60 * 60;
+      if(seconds >= secondsPerDay)
+      {
+         int days = seconds / (secondsPerDay);
+         seconds = seconds % secondsPerDay;
+         rs.append(days).append(StringUtils.plural(days, " day", " days")).append(" ");
+      }
+
+      int secondsPerHour = 60 * 60;
+      if(seconds >= secondsPerHour)
+      {
+         int hours = seconds / (secondsPerHour);
+         seconds = seconds % secondsPerHour;
+         rs.append(hours).append(StringUtils.plural(hours, " hour", " hours")).append(" ");
+      }
+
+      int secondsPerMinute = 60;
+      if(seconds >= secondsPerMinute)
+      {
+         int minutes = seconds / (secondsPerMinute);
+         seconds = seconds % secondsPerMinute;
+         rs.append(minutes).append(StringUtils.plural(minutes, " minute", " minutes")).append(" ");
+      }
+
+      if(seconds > 0 || rs.length() == 0)
+      {
+         rs.append(seconds).append(StringUtils.plural(seconds, " second", " seconds")).append(" ");
+      }
+
+      if(rs.length() > 0)
+      {
+         rs.deleteCharAt(rs.length() - 1);
+      }
+
+      return (rs.toString());
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public String formatSecondsAsRoundedDuration(Integer seconds)
+   {
+      StringBuilder rs = new StringBuilder();
+
+      if(seconds == null)
+      {
+         return ("");
+      }
+
+      int secondsPerDay = 24 * 60 * 60;
+      if(seconds >= secondsPerDay)
+      {
+         int days = seconds / (secondsPerDay);
+         return (days + StringUtils.plural(days, " day", " days"));
+      }
+
+      int secondsPerHour = 60 * 60;
+      if(seconds >= secondsPerHour)
+      {
+         int hours = seconds / (secondsPerHour);
+         return (hours + StringUtils.plural(hours, " hour", " hours"));
+      }
+
+      int secondsPerMinute = 60;
+      if(seconds >= secondsPerMinute)
+      {
+         int minutes = seconds / (secondsPerMinute);
+         return (minutes + StringUtils.plural(minutes, " minute", " minutes"));
+      }
+
+      if(seconds > 0 || rs.length() == 0)
+      {
+         return (seconds + StringUtils.plural(seconds, " second", " seconds"));
+      }
+
+      return ("");
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public String tableCountFilterLink(String countVariableName, String singular, String plural) throws QException
+   {
+      try
+      {
+         WidgetCount  widgetCount = (WidgetCount) context.get(countVariableName + ".source");
+         Integer      count       = ValueUtils.getValueAsInteger(context.get(countVariableName));
+         QQueryFilter filter      = widgetCount.getEffectiveFilter(input);
+         return (AbstractHTMLWidgetRenderer.aHrefTableFilterNoOfRecords(null, widgetCount.getTableName(), filter, count, singular, plural));
+      }
+      catch(Exception e)
+      {
+         LOG.warn("Error rendering widget link", e);
+         return ("");
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public String format(String displayFormat, Serializable value)
+   {
+      return (QValueFormatter.formatValue(displayFormat, value));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public String round(BigDecimal input, int digits)
+   {
+      return String.valueOf(input.setScale(digits, RoundingMode.HALF_UP));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public Object ifElse(Object ifObject, Object elseObject)
+   {
+      if(StringUtils.hasContent(ValueUtils.getValueAsString(ifObject)))
+      {
+         return (ifObject);
+      }
+      else if(StringUtils.hasContent(ValueUtils.getValueAsString(elseObject)))
+      {
+         return (elseObject);
+      }
+
+      return ("");
+   }
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/dashboard/widgets/ParentWidgetRenderer.java

@@ -0,0 +1,71 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.actions.dashboard.widgets;
+
+
+import com.kingsrook.qqq.backend.core.actions.ActionHelper;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.actions.widgets.RenderWidgetInput;
+import com.kingsrook.qqq.backend.core.model.actions.widgets.RenderWidgetOutput;
+import com.kingsrook.qqq.backend.core.model.dashboard.widgets.ParentWidgetData;
+import com.kingsrook.qqq.backend.core.model.metadata.dashboard.ParentWidgetMetaData;
+
+
+/*******************************************************************************
+ ** Generic widget for display a parent widget with children of possible values,
+ ** child widgets, and child actions
+ *******************************************************************************/
+public class ParentWidgetRenderer extends AbstractWidgetRenderer
+{
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public RenderWidgetOutput render(RenderWidgetInput input) throws QException
+   {
+      ActionHelper.validateSession(input);
+
+      try
+      {
+         ParentWidgetMetaData metaData   = (ParentWidgetMetaData) input.getWidgetMetaData();
+         ParentWidgetData     widgetData = new ParentWidgetData();
+
+         /////////////////////////////////////////////////////////////
+         // handle any PVSs creating dropdown data for the frontend //
+         /////////////////////////////////////////////////////////////
+         boolean dropdownsValid = setupDropdowns(input, metaData, widgetData);
+
+         if(dropdownsValid)
+         {
+            widgetData.setChildWidgetNameList(metaData.getChildWidgetNameList());
+         }
+
+         return (new RenderWidgetOutput(widgetData));
+      }
+      catch(Exception e)
+      {
+         throw (new QException("Error rendering parent widget", e));
+      }
+   }
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/dashboard/widgets/ProcessWidgetRenderer.java

@@ -0,0 +1,73 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.actions.dashboard.widgets;
+
+
+import java.util.HashMap;
+import com.kingsrook.qqq.backend.core.actions.ActionHelper;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.actions.widgets.RenderWidgetInput;
+import com.kingsrook.qqq.backend.core.model.actions.widgets.RenderWidgetOutput;
+import com.kingsrook.qqq.backend.core.model.dashboard.widgets.ProcessWidgetData;
+import com.kingsrook.qqq.backend.core.model.metadata.dashboard.QWidgetMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
+
+
+/*******************************************************************************
+ ** Generic widget for displaying a process as a widget
+ *******************************************************************************/
+public class ProcessWidgetRenderer extends AbstractWidgetRenderer
+{
+   public static final String WIDGET_PROCESS_NAME = "processName";
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public RenderWidgetOutput render(RenderWidgetInput input) throws QException
+   {
+      ActionHelper.validateSession(input);
+
+      try
+      {
+         ProcessWidgetData data = new ProcessWidgetData();
+         if(input.getWidgetMetaData() instanceof QWidgetMetaData widgetMetaData)
+         {
+            setupDropdowns(input, widgetMetaData, data);
+
+            String           processName     = (String) widgetMetaData.getDefaultValues().get(WIDGET_PROCESS_NAME);
+            QProcessMetaData processMetaData = input.getInstance().getProcess(processName);
+            data.setProcessMetaData(processMetaData);
+
+            data.setDefaultValues(new HashMap<>(input.getQueryParams()));
+         }
+         return (new RenderWidgetOutput(data));
+      }
+      catch(Exception e)
+      {
+         throw (new QException("Error rendering process widget", e));
+      }
+   }
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/dashboard/widgets/QuickSightChartRenderer.java

@@ -0,0 +1,104 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.actions.dashboard.widgets;
+
+
+import com.kingsrook.qqq.backend.core.actions.ActionHelper;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.actions.widgets.RenderWidgetInput;
+import com.kingsrook.qqq.backend.core.model.actions.widgets.RenderWidgetOutput;
+import com.kingsrook.qqq.backend.core.model.dashboard.widgets.QWidgetData;
+import com.kingsrook.qqq.backend.core.model.dashboard.widgets.QuickSightChart;
+import com.kingsrook.qqq.backend.core.model.metadata.dashboard.QuickSightChartMetaData;
+import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
+import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
+import software.amazon.awssdk.regions.Region;
+import software.amazon.awssdk.services.quicksight.QuickSightClient;
+import software.amazon.awssdk.services.quicksight.model.GenerateEmbedUrlForRegisteredUserRequest;
+import software.amazon.awssdk.services.quicksight.model.GenerateEmbedUrlForRegisteredUserResponse;
+import software.amazon.awssdk.services.quicksight.model.RegisteredUserDashboardEmbeddingConfiguration;
+import software.amazon.awssdk.services.quicksight.model.RegisteredUserEmbeddingExperienceConfiguration;
+
+
+/*******************************************************************************
+ ** Widget implementation for amazon QuickSight charts
+ **
+ *******************************************************************************/
+public class QuickSightChartRenderer extends AbstractWidgetRenderer
+{
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public RenderWidgetOutput render(RenderWidgetInput input) throws QException
+   {
+      ActionHelper.validateSession(input);
+
+      try
+      {
+         QuickSightChartMetaData quickSightMetaData = (QuickSightChartMetaData) input.getWidgetMetaData();
+         QuickSightClient        quickSightClient   = getQuickSightClient(quickSightMetaData);
+
+         final RegisteredUserEmbeddingExperienceConfiguration experienceConfiguration = RegisteredUserEmbeddingExperienceConfiguration.builder()
+            .dashboard(
+               RegisteredUserDashboardEmbeddingConfiguration.builder()
+                  .initialDashboardId(quickSightMetaData.getDashboardId())
+                  .build())
+            .build();
+
+         final GenerateEmbedUrlForRegisteredUserRequest generateEmbedUrlForRegisteredUserRequest = GenerateEmbedUrlForRegisteredUserRequest.builder()
+            .awsAccountId(quickSightMetaData.getAccountId())
+            .userArn(quickSightMetaData.getUserArn())
+            .experienceConfiguration(experienceConfiguration)
+            .build();
+
+         final GenerateEmbedUrlForRegisteredUserResponse generateEmbedUrlForRegisteredUserResponse = quickSightClient.generateEmbedUrlForRegisteredUser(generateEmbedUrlForRegisteredUserRequest);
+
+         String      embedUrl = generateEmbedUrlForRegisteredUserResponse.embedUrl();
+         QWidgetData widget   = new QuickSightChart(input.getWidgetMetaData().getName(), quickSightMetaData.getLabel(), embedUrl);
+         return (new RenderWidgetOutput(widget));
+      }
+      catch(Exception e)
+      {
+         throw (new QException("Error rendering widget", e));
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private QuickSightClient getQuickSightClient(QuickSightChartMetaData metaData)
+   {
+      AwsBasicCredentials awsCredentials = AwsBasicCredentials.create(metaData.getAccessKey(), metaData.getSecretKey());
+
+      QuickSightClient amazonQuickSightClient = QuickSightClient.builder()
+         .credentialsProvider(StaticCredentialsProvider.create(awsCredentials))
+         .region(Region.of(metaData.getRegion()))
+         .build();
+
+      return (amazonQuickSightClient);
+   }
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/dashboard/widgets/USMapWidgetRenderer.java

@@ -0,0 +1,62 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.actions.dashboard.widgets;
+
+
+import java.math.BigDecimal;
+import java.util.List;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.actions.widgets.RenderWidgetInput;
+import com.kingsrook.qqq.backend.core.model.actions.widgets.RenderWidgetOutput;
+import com.kingsrook.qqq.backend.core.model.dashboard.widgets.USMapWidgetData;
+
+
+/*******************************************************************************
+ ** Generic widget for display a map of the us
+ *******************************************************************************/
+public class USMapWidgetRenderer extends AbstractWidgetRenderer
+{
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public RenderWidgetOutput render(RenderWidgetInput input) throws QException
+   {
+      return (new RenderWidgetOutput(
+         new USMapWidgetData()
+            .withHeight("250px")
+            .withMapMarkerList(generateMapMarkerList())
+      ));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   protected List<USMapWidgetData.MapMarker> generateMapMarkerList() throws QException
+   {
+      return (List.of(new USMapWidgetData.MapMarker("maryville", new BigDecimal("38.725278"), new BigDecimal("-89.957778"))));
+   }
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/interfaces/AggregateInterface.java

@@ -0,0 +1,40 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.actions.interfaces;
+
+
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateOutput;
+
+
+/*******************************************************************************
+ ** Interface for the Aggregate action.
+ **
+ *******************************************************************************/
+public interface AggregateInterface
+{
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   AggregateOutput execute(AggregateInput aggregateInput) throws QException;
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/interfaces/GetInterface.java

@@ -0,0 +1,73 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.actions.interfaces;
+
+
+import java.util.HashSet;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetOutput;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.UniqueKey;
+
+
+/*******************************************************************************
+ ** Interface for the Get action.
+ **
+ *******************************************************************************/
+public interface GetInterface
+{
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   GetOutput execute(GetInput getInput) throws QException;
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   default void validateInput(GetInput getInput) throws QException
+   {
+      if(getInput.getPrimaryKey() != null & getInput.getUniqueKey() != null)
+      {
+         throw new QException("A GetInput may not contain both a primary key [" + getInput.getPrimaryKey() + "] and unique key [" + getInput.getUniqueKey() + "]");
+      }
+
+      if(getInput.getUniqueKey() != null)
+      {
+         QTableMetaData table      = getInput.getTable();
+         boolean        foundMatch = false;
+         for(UniqueKey uniqueKey : table.getUniqueKeys())
+         {
+            if(new HashSet<>(uniqueKey.getFieldNames()).equals(getInput.getUniqueKey().keySet()))
+            {
+               foundMatch = true;
+               break;
+            }
+         }
+
+         if(!foundMatch)
+         {
+            throw new QException("Table [" + table.getName() + "] does not have a unique key defined on fields: " + getInput.getUniqueKey().keySet().stream().sorted().toList());
+         }
+      }
+   }
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/interfaces/InsertInterface.java

@@ -22,7 +22,6 @@
 package com.kingsrook.qqq.backend.core.actions.interfaces;
 
 
-import com.kingsrook.qqq.backend.core.actions.QBackendTransaction;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertOutput;
@@ -32,19 +31,11 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertOutput;
  ** Interface for the Insert action.
  **
  *******************************************************************************/
-public interface InsertInterface
+public interface InsertInterface extends QActionInterface
 {
    /*******************************************************************************
     **
     *******************************************************************************/
    InsertOutput execute(InsertInput insertInput) throws QException;
 
-   /*******************************************************************************
-    **
-    *******************************************************************************/
-   default QBackendTransaction openTransaction(InsertInput insertInput) throws QException
-   {
-      return (new QBackendTransaction());
-   }
-
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/interfaces/QActionInterface.java

@@ -0,0 +1,44 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.actions.interfaces;
+
+
+import com.kingsrook.qqq.backend.core.actions.QBackendTransaction;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.actions.AbstractTableActionInput;
+
+
+/*******************************************************************************
+ **
+ *******************************************************************************/
+public interface QActionInterface
+{
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   default QBackendTransaction openTransaction(AbstractTableActionInput input) throws QException
+   {
+      return (new QBackendTransaction());
+   }
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/metadata/MetaDataAction.java

@@ -27,17 +27,26 @@ import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
 import com.kingsrook.qqq.backend.core.actions.ActionHelper;
+import com.kingsrook.qqq.backend.core.actions.permissions.PermissionCheckResult;
+import com.kingsrook.qqq.backend.core.actions.permissions.PermissionsHelper;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.model.actions.metadata.MetaDataInput;
 import com.kingsrook.qqq.backend.core.model.actions.metadata.MetaDataOutput;
+import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.dashboard.QWidgetMetaDataInterface;
 import com.kingsrook.qqq.backend.core.model.metadata.frontend.AppTreeNode;
 import com.kingsrook.qqq.backend.core.model.metadata.frontend.QFrontendAppMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.frontend.QFrontendProcessMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.frontend.QFrontendReportMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.frontend.QFrontendTableMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.frontend.QFrontendWidgetMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppChildMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.permissions.MetaDataWithPermissionRules;
 import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
 
 
 /*******************************************************************************
@@ -64,8 +73,18 @@ public class MetaDataAction
       Map<String, QFrontendTableMetaData> tables = new LinkedHashMap<>();
       for(Map.Entry<String, QTableMetaData> entry : metaDataInput.getInstance().getTables().entrySet())
       {
-         tables.put(entry.getKey(), new QFrontendTableMetaData(entry.getValue(), false));
-         treeNodes.put(entry.getKey(), new AppTreeNode(entry.getValue()));
+         String         tableName = entry.getKey();
+         QTableMetaData table     = entry.getValue();
+
+         PermissionCheckResult permissionResult = PermissionsHelper.getPermissionCheckResult(metaDataInput, table);
+         if(permissionResult.equals(PermissionCheckResult.DENY_HIDE))
+         {
+            continue;
+         }
+
+         QBackendMetaData backendForTable = metaDataInput.getInstance().getBackendForTable(tableName);
+         tables.put(tableName, new QFrontendTableMetaData(metaDataInput, backendForTable, table, false));
+         treeNodes.put(tableName, new AppTreeNode(table));
       }
       metaDataOutput.setTables(tables);
 
@@ -75,23 +94,92 @@ public class MetaDataAction
       Map<String, QFrontendProcessMetaData> processes = new LinkedHashMap<>();
       for(Map.Entry<String, QProcessMetaData> entry : metaDataInput.getInstance().getProcesses().entrySet())
       {
-         processes.put(entry.getKey(), new QFrontendProcessMetaData(entry.getValue(), false));
-         treeNodes.put(entry.getKey(), new AppTreeNode(entry.getValue()));
+         String           processName = entry.getKey();
+         QProcessMetaData process     = entry.getValue();
+
+         PermissionCheckResult permissionResult = PermissionsHelper.getPermissionCheckResult(metaDataInput, process);
+         if(permissionResult.equals(PermissionCheckResult.DENY_HIDE))
+         {
+            continue;
+         }
+
+         processes.put(processName, new QFrontendProcessMetaData(metaDataInput, process, false));
+         treeNodes.put(processName, new AppTreeNode(process));
       }
       metaDataOutput.setProcesses(processes);
 
+      //////////////////////////////////////
+      // map reports to frontend metadata //
+      //////////////////////////////////////
+      Map<String, QFrontendReportMetaData> reports = new LinkedHashMap<>();
+      for(Map.Entry<String, QReportMetaData> entry : metaDataInput.getInstance().getReports().entrySet())
+      {
+         String          reportName = entry.getKey();
+         QReportMetaData report     = entry.getValue();
+
+         PermissionCheckResult permissionResult = PermissionsHelper.getPermissionCheckResult(metaDataInput, report);
+         if(permissionResult.equals(PermissionCheckResult.DENY_HIDE))
+         {
+            continue;
+         }
+
+         reports.put(reportName, new QFrontendReportMetaData(metaDataInput, report, false));
+         treeNodes.put(reportName, new AppTreeNode(report));
+      }
+      metaDataOutput.setReports(reports);
+
+      //////////////////////////////////////
+      // map widgets to frontend metadata //
+      //////////////////////////////////////
+      Map<String, QFrontendWidgetMetaData> widgets = new LinkedHashMap<>();
+      for(Map.Entry<String, QWidgetMetaDataInterface> entry : metaDataInput.getInstance().getWidgets().entrySet())
+      {
+         String                   widgetName = entry.getKey();
+         QWidgetMetaDataInterface widget     = entry.getValue();
+
+         PermissionCheckResult permissionResult = PermissionsHelper.getPermissionCheckResult(metaDataInput, widget);
+         if(permissionResult.equals(PermissionCheckResult.DENY_HIDE))
+         {
+            continue;
+         }
+
+         widgets.put(widgetName, new QFrontendWidgetMetaData(metaDataInput, widget));
+      }
+      metaDataOutput.setWidgets(widgets);
+
       ///////////////////////////////////
       // map apps to frontend metadata //
       ///////////////////////////////////
       Map<String, QFrontendAppMetaData> apps = new LinkedHashMap<>();
       for(Map.Entry<String, QAppMetaData> entry : metaDataInput.getInstance().getApps().entrySet())
       {
-         apps.put(entry.getKey(), new QFrontendAppMetaData(entry.getValue()));
-         treeNodes.put(entry.getKey(), new AppTreeNode(entry.getValue()));
+         String       appName = entry.getKey();
+         QAppMetaData app     = entry.getValue();
 
-         for(QAppChildMetaData child : entry.getValue().getChildren())
+         PermissionCheckResult permissionResult = PermissionsHelper.getPermissionCheckResult(metaDataInput, app);
+         if(permissionResult.equals(PermissionCheckResult.DENY_HIDE))
          {
-            apps.get(entry.getKey()).addChild(new AppTreeNode(child));
+            continue;
+         }
+
+         apps.put(appName, new QFrontendAppMetaData(app, metaDataOutput));
+         treeNodes.put(appName, new AppTreeNode(app));
+
+         if(CollectionUtils.nullSafeHasContents(app.getChildren()))
+         {
+            for(QAppChildMetaData child : app.getChildren())
+            {
+               if(child instanceof MetaDataWithPermissionRules metaDataWithPermissionRules)
+               {
+                  PermissionCheckResult childPermissionResult = PermissionsHelper.getPermissionCheckResult(metaDataInput, metaDataWithPermissionRules);
+                  if(childPermissionResult.equals(PermissionCheckResult.DENY_HIDE))
+                  {
+                     continue;
+                  }
+               }
+
+               apps.get(appName).addChild(new AppTreeNode(child));
+            }
          }
       }
       metaDataOutput.setApps(apps);
@@ -104,11 +192,21 @@ public class MetaDataAction
       {
          if(appMetaData.getParentAppName() == null)
          {
-            buildAppTree(treeNodes, appTree, appMetaData);
+            buildAppTree(metaDataInput, treeNodes, appTree, appMetaData);
          }
       }
       metaDataOutput.setAppTree(appTree);
 
+      ////////////////////////////////////
+      // add branding metadata if found //
+      ////////////////////////////////////
+      if(metaDataInput.getInstance().getBranding() != null)
+      {
+         metaDataOutput.setBranding(metaDataInput.getInstance().getBranding());
+      }
+
+      metaDataOutput.setEnvironmentValues(metaDataInput.getInstance().getEnvironmentValues());
+
       // todo post-customization - can do whatever w/ the result if you want?
 
       return metaDataOutput;
@@ -119,7 +217,7 @@ public class MetaDataAction
    /*******************************************************************************
     **
     *******************************************************************************/
-   private void buildAppTree(Map<String, AppTreeNode> treeNodes, List<AppTreeNode> nodeList, QAppChildMetaData childMetaData)
+   private void buildAppTree(MetaDataInput metaDataInput, Map<String, AppTreeNode> treeNodes, List<AppTreeNode> nodeList, QAppChildMetaData childMetaData)
    {
       AppTreeNode treeNode = treeNodes.get(childMetaData.getName());
       if(treeNode == null)
@@ -134,7 +232,16 @@ public class MetaDataAction
          {
             for(QAppChildMetaData child : app.getChildren())
             {
-               buildAppTree(treeNodes, treeNode.getChildren(), child);
+               if(child instanceof MetaDataWithPermissionRules metaDataWithPermissionRules)
+               {
+                  PermissionCheckResult permissionResult = PermissionsHelper.getPermissionCheckResult(metaDataInput, metaDataWithPermissionRules);
+                  if(permissionResult.equals(PermissionCheckResult.DENY_HIDE))
+                  {
+                     continue;
+                  }
+               }
+
+               buildAppTree(metaDataInput, treeNodes, treeNode.getChildren(), child);
             }
          }
       }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/metadata/ProcessMetaDataAction.java

@@ -52,7 +52,7 @@ public class ProcessMetaDataAction
       {
          throw (new QNotFoundException("Process [" + processMetaDataInput.getProcessName() + "] was not found."));
       }
-      processMetaDataOutput.setProcess(new QFrontendProcessMetaData(process, true));
+      processMetaDataOutput.setProcess(new QFrontendProcessMetaData(processMetaDataInput, process, true));
 
       // todo post-customization - can do whatever w/ the result if you want
 

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/metadata/TableMetaDataAction.java

@@ -27,6 +27,7 @@ import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.exceptions.QNotFoundException;
 import com.kingsrook.qqq.backend.core.model.actions.metadata.TableMetaDataInput;
 import com.kingsrook.qqq.backend.core.model.actions.metadata.TableMetaDataOutput;
+import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.frontend.QFrontendTableMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 
@@ -52,7 +53,8 @@ public class TableMetaDataAction
       {
          throw (new QNotFoundException("Table [" + tableMetaDataInput.getTableName() + "] was not found."));
       }
-      tableMetaDataOutput.setTable(new QFrontendTableMetaData(table, true));
+      QBackendMetaData backendForTable = tableMetaDataInput.getInstance().getBackendForTable(table.getName());
+      tableMetaDataOutput.setTable(new QFrontendTableMetaData(tableMetaDataInput, backendForTable, table, true));
 
       // todo post-customization - can do whatever w/ the result if you want
 

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/permissions/AvailablePermission.java

@@ -0,0 +1,204 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2023.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.actions.permissions;
+
+
+import java.util.Objects;
+import com.kingsrook.qqq.backend.core.model.data.QField;
+import com.kingsrook.qqq.backend.core.model.data.QRecordEntity;
+
+
+/*******************************************************************************
+ **
+ *******************************************************************************/
+public class AvailablePermission extends QRecordEntity
+{
+   public static final String TABLE_NAME = "availablePermission";
+
+   @QField(label = "Permission Name")
+   private String name;
+
+   @QField(label = "Object")
+   private String objectName;
+
+   @QField()
+   private String objectType;
+
+   @QField()
+   private String permissionType;
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public boolean equals(Object o)
+   {
+      if(this == o)
+      {
+         return true;
+      }
+      if(o == null || getClass() != o.getClass())
+      {
+         return false;
+      }
+      AvailablePermission that = (AvailablePermission) o;
+      return Objects.equals(name, that.name) && Objects.equals(objectName, that.objectName) && Objects.equals(objectType, that.objectType) && Objects.equals(permissionType, that.permissionType);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public int hashCode()
+   {
+      return Objects.hash(name, objectName, objectType, permissionType);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for name
+    *******************************************************************************/
+   public String getName()
+   {
+      return (this.name);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for name
+    *******************************************************************************/
+   public void setName(String name)
+   {
+      this.name = name;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for name
+    *******************************************************************************/
+   public AvailablePermission withName(String name)
+   {
+      this.name = name;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for objectType
+    *******************************************************************************/
+   public String getObjectType()
+   {
+      return (this.objectType);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for objectType
+    *******************************************************************************/
+   public void setObjectType(String objectType)
+   {
+      this.objectType = objectType;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for objectType
+    *******************************************************************************/
+   public AvailablePermission withObjectType(String objectType)
+   {
+      this.objectType = objectType;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for permissionType
+    *******************************************************************************/
+   public String getPermissionType()
+   {
+      return (this.permissionType);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for permissionType
+    *******************************************************************************/
+   public void setPermissionType(String permissionType)
+   {
+      this.permissionType = permissionType;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for permissionType
+    *******************************************************************************/
+   public AvailablePermission withPermissionType(String permissionType)
+   {
+      this.permissionType = permissionType;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for objectName
+    *******************************************************************************/
+   public String getObjectName()
+   {
+      return (this.objectName);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for objectName
+    *******************************************************************************/
+   public void setObjectName(String objectName)
+   {
+      this.objectName = objectName;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for objectName
+    *******************************************************************************/
+   public AvailablePermission withObjectName(String objectName)
+   {
+      this.objectName = objectName;
+      return (this);
+   }
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/permissions/BulkTableActionProcessPermissionChecker.java

@@ -0,0 +1,67 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2023.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.actions.permissions;
+
+
+import com.kingsrook.qqq.backend.core.exceptions.QPermissionDeniedException;
+import com.kingsrook.qqq.backend.core.logging.QLogger;
+import com.kingsrook.qqq.backend.core.model.actions.AbstractActionInput;
+import com.kingsrook.qqq.backend.core.model.actions.AbstractTableActionInput;
+import com.kingsrook.qqq.backend.core.model.metadata.permissions.MetaDataWithPermissionRules;
+
+
+/*******************************************************************************
+ **
+ *******************************************************************************/
+public class BulkTableActionProcessPermissionChecker implements CustomPermissionChecker
+{
+   private static final QLogger LOG = QLogger.getLogger(BulkTableActionProcessPermissionChecker.class);
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public void checkPermissionsThrowing(AbstractActionInput actionInput, MetaDataWithPermissionRules metaDataWithPermissionRules) throws QPermissionDeniedException
+   {
+      String processName = metaDataWithPermissionRules.getName();
+      if(processName != null && processName.indexOf('.') > -1)
+      {
+         String[] parts          = processName.split("\\.", 2);
+         String   tableName      = parts[0];
+         String   bulkActionName = parts[1];
+
+         AbstractTableActionInput tableActionInput = new AbstractTableActionInput();
+         tableActionInput.setTableName(tableName);
+
+         switch(bulkActionName)
+         {
+            case "bulkInsert" -> PermissionsHelper.checkTablePermissionThrowing(tableActionInput, TablePermissionSubType.INSERT);
+            case "bulkEdit" -> PermissionsHelper.checkTablePermissionThrowing(tableActionInput, TablePermissionSubType.EDIT);
+            case "bulkDelete" -> PermissionsHelper.checkTablePermissionThrowing(tableActionInput, TablePermissionSubType.DELETE);
+            default -> LOG.warn("Unexpected bulk action name when checking permissions for process: " + processName);
+         }
+      }
+   }
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/permissions/CustomPermissionChecker.java

@@ -0,0 +1,41 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2023.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.actions.permissions;
+
+
+import com.kingsrook.qqq.backend.core.exceptions.QPermissionDeniedException;
+import com.kingsrook.qqq.backend.core.model.actions.AbstractActionInput;
+import com.kingsrook.qqq.backend.core.model.metadata.permissions.MetaDataWithPermissionRules;
+
+
+/*******************************************************************************
+ **
+ *******************************************************************************/
+public interface CustomPermissionChecker
+{
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   void checkPermissionsThrowing(AbstractActionInput actionInput, MetaDataWithPermissionRules metaDataWithPermissionRules) throws QPermissionDeniedException;
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/permissions/PermissionCheckResult.java

@@ -0,0 +1,33 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2023.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.actions.permissions;
+
+
+/*******************************************************************************
+ **
+ *******************************************************************************/
+public enum PermissionCheckResult
+{
+   ALLOW,
+   DENY_HIDE,
+   DENY_DISABLE;
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/permissions/PermissionSubType.java

@@ -0,0 +1,36 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2023.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.actions.permissions;
+
+
+/*******************************************************************************
+ **
+ *******************************************************************************/
+sealed interface PermissionSubType permits PrivatePermissionSubType, TablePermissionSubType
+{
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   String getPermissionSuffix();
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/permissions/PermissionsHelper.java

@@ -0,0 +1,602 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2023.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.actions.permissions;
+
+
+import java.io.Serializable;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.LinkedHashSet;
+import java.util.Map;
+import java.util.stream.Collectors;
+import com.kingsrook.qqq.backend.core.actions.customizers.QCodeLoader;
+import com.kingsrook.qqq.backend.core.context.QContext;
+import com.kingsrook.qqq.backend.core.exceptions.QPermissionDeniedException;
+import com.kingsrook.qqq.backend.core.logging.QLogger;
+import com.kingsrook.qqq.backend.core.model.actions.AbstractActionInput;
+import com.kingsrook.qqq.backend.core.model.actions.AbstractTableActionInput;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.dashboard.QWidgetMetaDataInterface;
+import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.permissions.DenyBehavior;
+import com.kingsrook.qqq.backend.core.model.metadata.permissions.MetaDataWithName;
+import com.kingsrook.qqq.backend.core.model.metadata.permissions.MetaDataWithPermissionRules;
+import com.kingsrook.qqq.backend.core.model.metadata.permissions.QPermissionRules;
+import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+import com.kingsrook.qqq.backend.core.model.session.QSession;
+import com.kingsrook.qqq.backend.core.utils.StringUtils;
+
+
+/*******************************************************************************
+ **
+ *******************************************************************************/
+public class PermissionsHelper
+{
+   private static final QLogger LOG = QLogger.getLogger(PermissionsHelper.class);
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public static void checkTablePermissionThrowing(AbstractTableActionInput tableActionInput, TablePermissionSubType permissionSubType) throws QPermissionDeniedException
+   {
+      checkTablePermissionThrowing(tableActionInput, tableActionInput.getTableName(), permissionSubType);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private static void checkTablePermissionThrowing(AbstractActionInput actionInput, String tableName, TablePermissionSubType permissionSubType) throws QPermissionDeniedException
+   {
+      warnAboutPermissionSubTypeForTables(permissionSubType);
+      QTableMetaData table = QContext.getQInstance().getTable(tableName);
+
+      commonCheckPermissionThrowing(getEffectivePermissionRules(table, QContext.getQInstance()), permissionSubType, table.getName());
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public static String getTablePermissionName(String tableName, TablePermissionSubType permissionSubType)
+   {
+      QInstance        qInstance          = QContext.getQInstance();
+      QPermissionRules rules              = getEffectivePermissionRules(qInstance.getTable(tableName), qInstance);
+      String           permissionBaseName = getEffectivePermissionBaseName(rules, tableName);
+      return (getPermissionName(permissionBaseName, permissionSubType));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public static boolean hasTablePermission(AbstractActionInput actionInput, String tableName, TablePermissionSubType permissionSubType)
+   {
+      try
+      {
+         checkTablePermissionThrowing(actionInput, tableName, permissionSubType);
+         return (true);
+      }
+      catch(QPermissionDeniedException e)
+      {
+         return (false);
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public static PermissionCheckResult getPermissionCheckResult(AbstractActionInput actionInput, MetaDataWithPermissionRules metaDataWithPermissionRules)
+   {
+      QPermissionRules rules              = getEffectivePermissionRules(metaDataWithPermissionRules, QContext.getQInstance());
+      String           permissionBaseName = getEffectivePermissionBaseName(rules, metaDataWithPermissionRules.getName());
+
+      switch(rules.getLevel())
+      {
+         case NOT_PROTECTED:
+         {
+            /////////////////////////////////////////////////
+            // if the entity isn't protected, always ALLOW //
+            /////////////////////////////////////////////////
+            return PermissionCheckResult.ALLOW;
+         }
+         case HAS_ACCESS_PERMISSION:
+         {
+            ////////////////////////////////////////////////////////////////////////
+            // if the entity just has a 'has access', then check for 'has access' //
+            ////////////////////////////////////////////////////////////////////////
+            return getPermissionCheckResult(actionInput, rules, permissionBaseName, metaDataWithPermissionRules, PrivatePermissionSubType.HAS_ACCESS);
+         }
+         case READ_WRITE_PERMISSIONS:
+         {
+            ////////////////////////////////////////////////////////////////
+            // if the table is configured w/ read/write, check for either //
+            ////////////////////////////////////////////////////////////////
+            if(metaDataWithPermissionRules instanceof QTableMetaData)
+            {
+               return getPermissionCheckResult(actionInput, rules, permissionBaseName, metaDataWithPermissionRules, PrivatePermissionSubType.READ, PrivatePermissionSubType.WRITE);
+            }
+            return getPermissionCheckResult(actionInput, rules, permissionBaseName, metaDataWithPermissionRules, PrivatePermissionSubType.HAS_ACCESS);
+         }
+         case READ_INSERT_EDIT_DELETE_PERMISSIONS:
+         {
+            //////////////////////////////////////////////////////////////////////////
+            // if the table is configured w/ read/insert/edit/delete, check for any //
+            //////////////////////////////////////////////////////////////////////////
+            if(metaDataWithPermissionRules instanceof QTableMetaData)
+            {
+               return getPermissionCheckResult(actionInput, rules, permissionBaseName, metaDataWithPermissionRules, TablePermissionSubType.READ, TablePermissionSubType.INSERT, TablePermissionSubType.EDIT, TablePermissionSubType.DELETE);
+            }
+            return getPermissionCheckResult(actionInput, rules, permissionBaseName, metaDataWithPermissionRules, PrivatePermissionSubType.HAS_ACCESS);
+         }
+         default:
+         {
+            return getPermissionDeniedCheckResult(rules);
+         }
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public static void checkProcessPermissionThrowing(AbstractActionInput actionInput, String processName) throws QPermissionDeniedException
+   {
+      checkProcessPermissionThrowing(actionInput, processName, Collections.emptyMap());
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public static void checkProcessPermissionThrowing(AbstractActionInput actionInput, String processName, Map<String, Serializable> processValues) throws QPermissionDeniedException
+   {
+      QProcessMetaData process                  = QContext.getQInstance().getProcess(processName);
+      QPermissionRules effectivePermissionRules = getEffectivePermissionRules(process, QContext.getQInstance());
+
+      if(effectivePermissionRules.getCustomPermissionChecker() != null)
+      {
+         /////////////////////////////////////
+         // todo - avoid stack overflows... //
+         /////////////////////////////////////
+         CustomPermissionChecker customPermissionChecker = QCodeLoader.getAdHoc(CustomPermissionChecker.class, effectivePermissionRules.getCustomPermissionChecker());
+         customPermissionChecker.checkPermissionsThrowing(actionInput, process);
+         return;
+      }
+
+      commonCheckPermissionThrowing(effectivePermissionRules, PrivatePermissionSubType.HAS_ACCESS, process.getName());
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public static boolean hasProcessPermission(AbstractActionInput actionInput, String processName)
+   {
+      try
+      {
+         checkProcessPermissionThrowing(actionInput, processName);
+         return (true);
+      }
+      catch(QPermissionDeniedException e)
+      {
+         return (false);
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public static void checkAppPermissionThrowing(AbstractActionInput actionInput, String appName) throws QPermissionDeniedException
+   {
+      QAppMetaData app = QContext.getQInstance().getApp(appName);
+      commonCheckPermissionThrowing(getEffectivePermissionRules(app, QContext.getQInstance()), PrivatePermissionSubType.HAS_ACCESS, app.getName());
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public static boolean hasAppPermission(AbstractActionInput actionInput, String appName)
+   {
+      try
+      {
+         checkAppPermissionThrowing(actionInput, appName);
+         return (true);
+      }
+      catch(QPermissionDeniedException e)
+      {
+         return (false);
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public static void checkReportPermissionThrowing(AbstractActionInput actionInput, String reportName) throws QPermissionDeniedException
+   {
+      QReportMetaData report = QContext.getQInstance().getReport(reportName);
+      commonCheckPermissionThrowing(getEffectivePermissionRules(report, QContext.getQInstance()), PrivatePermissionSubType.HAS_ACCESS, report.getName());
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public static boolean hasReportPermission(AbstractActionInput actionInput, String reportName)
+   {
+      try
+      {
+         checkReportPermissionThrowing(actionInput, reportName);
+         return (true);
+      }
+      catch(QPermissionDeniedException e)
+      {
+         return (false);
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public static void checkWidgetPermissionThrowing(AbstractActionInput actionInput, String widgetName) throws QPermissionDeniedException
+   {
+      QWidgetMetaDataInterface widget = QContext.getQInstance().getWidget(widgetName);
+      commonCheckPermissionThrowing(getEffectivePermissionRules(widget, QContext.getQInstance()), PrivatePermissionSubType.HAS_ACCESS, widget.getName());
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public static boolean hasWidgetPermission(AbstractActionInput actionInput, String widgetName)
+   {
+      try
+      {
+         checkWidgetPermissionThrowing(actionInput, widgetName);
+         return (true);
+      }
+      catch(QPermissionDeniedException e)
+      {
+         return (false);
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public static Collection<String> getAllAvailablePermissionNames(QInstance instance)
+   {
+      return (getAllAvailablePermissions(instance).stream()
+         .map(AvailablePermission::getName)
+         .collect(Collectors.toCollection(LinkedHashSet::new)));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public static Collection<AvailablePermission> getAllAvailablePermissions(QInstance instance)
+   {
+      Collection<AvailablePermission> rs = new LinkedHashSet<>();
+
+      for(QTableMetaData tableMetaData : instance.getTables().values())
+      {
+         if(tableMetaData.getIsHidden())
+         {
+            continue;
+         }
+
+         QPermissionRules rules    = getEffectivePermissionRules(tableMetaData, instance);
+         String           baseName = getEffectivePermissionBaseName(rules, tableMetaData.getName());
+
+         for(TablePermissionSubType permissionSubType : TablePermissionSubType.values())
+         {
+            addEffectiveAvailablePermission(rules, permissionSubType, rs, baseName, tableMetaData, "Table");
+         }
+      }
+
+      for(QProcessMetaData processMetaData : instance.getProcesses().values())
+      {
+         if(processMetaData.getIsHidden())
+         {
+            continue;
+         }
+
+         QPermissionRules rules    = getEffectivePermissionRules(processMetaData, instance);
+         String           baseName = getEffectivePermissionBaseName(rules, processMetaData.getName());
+         addEffectiveAvailablePermission(rules, PrivatePermissionSubType.HAS_ACCESS, rs, baseName, processMetaData, "Process");
+      }
+
+      for(QAppMetaData appMetaData : instance.getApps().values())
+      {
+         QPermissionRules rules    = getEffectivePermissionRules(appMetaData, instance);
+         String           baseName = getEffectivePermissionBaseName(rules, appMetaData.getName());
+         addEffectiveAvailablePermission(rules, PrivatePermissionSubType.HAS_ACCESS, rs, baseName, appMetaData, "App");
+      }
+
+      for(QReportMetaData reportMetaData : instance.getReports().values())
+      {
+         QPermissionRules rules    = getEffectivePermissionRules(reportMetaData, instance);
+         String           baseName = getEffectivePermissionBaseName(rules, reportMetaData.getName());
+         addEffectiveAvailablePermission(rules, PrivatePermissionSubType.HAS_ACCESS, rs, baseName, reportMetaData, "Report");
+      }
+
+      for(QWidgetMetaDataInterface widgetMetaData : instance.getWidgets().values())
+      {
+         QPermissionRules rules    = getEffectivePermissionRules(widgetMetaData, instance);
+         String           baseName = getEffectivePermissionBaseName(rules, widgetMetaData.getName());
+         addEffectiveAvailablePermission(rules, PrivatePermissionSubType.HAS_ACCESS, rs, baseName, widgetMetaData, "Widget");
+      }
+
+      return (rs);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private static void addEffectiveAvailablePermission(QPermissionRules rules, PermissionSubType permissionSubType, Collection<AvailablePermission> rs, String baseName, MetaDataWithName metaDataWithName, String objectType)
+   {
+      PermissionSubType effectivePermissionSubType = getEffectivePermissionSubType(rules, permissionSubType);
+      if(effectivePermissionSubType != null)
+      {
+         rs.add(new AvailablePermission()
+            .withName(getPermissionName(baseName, effectivePermissionSubType))
+            .withObjectName(metaDataWithName.getLabel())
+            .withPermissionType(effectivePermissionSubType.toString())
+            .withObjectType(objectType));
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public static QPermissionRules getEffectivePermissionRules(MetaDataWithPermissionRules metaDataWithPermissionRules, QInstance instance)
+   {
+      if(metaDataWithPermissionRules.getPermissionRules() == null)
+      {
+         LOG.warn("Null permission rules on meta data object [" + metaDataWithPermissionRules.getClass().getSimpleName() + "][" + metaDataWithPermissionRules.getName() + "] - does the instance need enriched?  Returning instance default rules.");
+         return (instance.getDefaultPermissionRules());
+      }
+      return (metaDataWithPermissionRules.getPermissionRules());
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   static boolean hasPermission(QSession session, String permissionBaseName, PermissionSubType permissionSubType)
+   {
+      if(permissionSubType == null)
+      {
+         return (true);
+      }
+
+      String permissionName = getPermissionName(permissionBaseName, permissionSubType);
+      return (session.hasPermission(permissionName));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   static PermissionCheckResult getPermissionCheckResult(AbstractActionInput actionInput, QPermissionRules rules, String permissionBaseName, MetaDataWithPermissionRules metaDataWithPermissionRules, PermissionSubType... permissionSubTypes)
+   {
+      for(PermissionSubType permissionSubType : permissionSubTypes)
+      {
+         PermissionSubType effectivePermissionSubType = getEffectivePermissionSubType(rules, permissionSubType);
+
+         if(rules.getCustomPermissionChecker() != null)
+         {
+            try
+            {
+               CustomPermissionChecker customPermissionChecker = QCodeLoader.getAdHoc(CustomPermissionChecker.class, rules.getCustomPermissionChecker());
+               customPermissionChecker.checkPermissionsThrowing(actionInput, metaDataWithPermissionRules);
+               return (PermissionCheckResult.ALLOW);
+            }
+            catch(QPermissionDeniedException e)
+            {
+               return (getPermissionDeniedCheckResult(rules));
+            }
+         }
+
+         if(hasPermission(QContext.getQSession(), permissionBaseName, effectivePermissionSubType))
+         {
+            return (PermissionCheckResult.ALLOW);
+         }
+      }
+
+      return (getPermissionDeniedCheckResult(rules));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   static String getEffectivePermissionBaseName(QPermissionRules rules, String standardName)
+   {
+      if(rules != null && StringUtils.hasContent(rules.getPermissionBaseName()))
+      {
+         return (rules.getPermissionBaseName());
+      }
+
+      return (standardName);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @SuppressWarnings("checkstyle:indentation")
+   static PermissionSubType getEffectivePermissionSubType(QPermissionRules rules, PermissionSubType originalPermissionSubType)
+   {
+      if(rules == null || rules.getLevel() == null)
+      {
+         return (originalPermissionSubType);
+      }
+
+      ////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      // if the original permission sub-type is "hasAccess" - then this is a check for a process/report/widget. //
+      // in that case - never return the table-level read/write/insert/edit/delete options                      //
+      ////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      if(PrivatePermissionSubType.HAS_ACCESS.equals(originalPermissionSubType))
+      {
+         return switch(rules.getLevel())
+            {
+               case NOT_PROTECTED -> null;
+               default -> PrivatePermissionSubType.HAS_ACCESS;
+            };
+      }
+      else
+      {
+         ////////////////////////////////////////////////////////////////////////////////////////////////////////
+         // else, this is a table check - so - based on the rules being used for this table, map the requested //
+         // permission sub-type to what we expect to be set for the table                                      //
+         ////////////////////////////////////////////////////////////////////////////////////////////////////////
+         return switch(rules.getLevel())
+            {
+               case NOT_PROTECTED -> null;
+               case HAS_ACCESS_PERMISSION -> PrivatePermissionSubType.HAS_ACCESS;
+               case READ_WRITE_PERMISSIONS ->
+               {
+                  if(PrivatePermissionSubType.READ.equals(originalPermissionSubType) || PrivatePermissionSubType.WRITE.equals(originalPermissionSubType))
+                  {
+                     yield (originalPermissionSubType);
+                  }
+                  else if(TablePermissionSubType.INSERT.equals(originalPermissionSubType) || TablePermissionSubType.EDIT.equals(originalPermissionSubType) || TablePermissionSubType.DELETE.equals(originalPermissionSubType))
+                  {
+                     yield (PrivatePermissionSubType.WRITE);
+                  }
+                  else if(TablePermissionSubType.READ.equals(originalPermissionSubType))
+                  {
+                     yield (PrivatePermissionSubType.READ);
+                  }
+                  else
+                  {
+                     throw new IllegalStateException("Unexpected permissionSubType: " + originalPermissionSubType);
+                  }
+               }
+               case READ_INSERT_EDIT_DELETE_PERMISSIONS -> originalPermissionSubType;
+            };
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private static void commonCheckPermissionThrowing(QPermissionRules rules, PermissionSubType permissionSubType, String name) throws QPermissionDeniedException
+   {
+      PermissionSubType effectivePermissionSubType = getEffectivePermissionSubType(rules, permissionSubType);
+      String            permissionBaseName         = getEffectivePermissionBaseName(rules, name);
+
+      if(effectivePermissionSubType == null)
+      {
+         return;
+      }
+
+      if(!hasPermission(QContext.getQSession(), permissionBaseName, effectivePermissionSubType))
+      {
+         // LOG.debug("Throwing permission denied for: " + getPermissionName(permissionBaseName, effectivePermissionSubType) + " for " + QContext.getQSession().getUser());
+         throw (new QPermissionDeniedException("Permission denied."));
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private static String getPermissionName(String permissionBaseName, PermissionSubType permissionSubType)
+   {
+      return permissionBaseName + "." + permissionSubType.getPermissionSuffix();
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private static PermissionCheckResult getPermissionDeniedCheckResult(QPermissionRules rules)
+   {
+      if(rules == null || rules.getDenyBehavior() == null || rules.getDenyBehavior().equals(DenyBehavior.HIDDEN))
+      {
+         return (PermissionCheckResult.DENY_HIDE);
+      }
+      else
+      {
+         return (PermissionCheckResult.DENY_DISABLE);
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private static void warnAboutPermissionSubTypeForTables(PermissionSubType permissionSubType)
+   {
+      if(permissionSubType == null)
+      {
+         return;
+      }
+
+      if(permissionSubType == PrivatePermissionSubType.HAS_ACCESS)
+      {
+         LOG.warn("PermissionSubType.HAS_ACCESS should not be checked for a table");
+      }
+   }
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/permissions/PrivatePermissionSubType.java

@@ -0,0 +1,56 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2023.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.actions.permissions;
+
+
+/*******************************************************************************
+ **
+ *******************************************************************************/
+enum PrivatePermissionSubType implements PermissionSubType
+{
+   HAS_ACCESS("hasAccess"), // for processes, reports, etc - basically, not tables.
+   READ("read"), // for a table in read/write mode - or - for read (query, get, count) on a table in full-mode
+   WRITE("write");
+
+   private final String permissionSuffix;
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   PrivatePermissionSubType(String permissionSuffix)
+   {
+      this.permissionSuffix = permissionSuffix;
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for permissionSuffix
+    *******************************************************************************/
+   public String getPermissionSuffix()
+   {
+      return (this.permissionSuffix);
+   }
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/permissions/ReportProcessPermissionChecker.java

@@ -0,0 +1,53 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2023.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.actions.permissions;
+
+
+import com.kingsrook.qqq.backend.core.exceptions.QPermissionDeniedException;
+import com.kingsrook.qqq.backend.core.model.actions.AbstractActionInput;
+import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput;
+import com.kingsrook.qqq.backend.core.model.metadata.permissions.MetaDataWithPermissionRules;
+
+
+/*******************************************************************************
+ **
+ *******************************************************************************/
+public class ReportProcessPermissionChecker implements CustomPermissionChecker
+{
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public void checkPermissionsThrowing(AbstractActionInput actionInput, MetaDataWithPermissionRules metaDataWithPermissionRules) throws QPermissionDeniedException
+   {
+      if(actionInput instanceof RunProcessInput runProcessInput)
+      {
+         String reportName = runProcessInput.getValueString("reportName");
+         if(reportName != null)
+         {
+            PermissionsHelper.checkReportPermissionThrowing(actionInput, reportName);
+         }
+      }
+   }
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/permissions/TablePermissionSubType.java

@@ -0,0 +1,57 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2023.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.actions.permissions;
+
+
+/*******************************************************************************
+ **
+ *******************************************************************************/
+public enum TablePermissionSubType implements PermissionSubType
+{
+   READ("read"), // for a table in read/write mode - or - for read (query, get, count) on a table in full-mode
+   INSERT("insert"), // for table-insert.
+   EDIT("edit"), // for table-edit.
+   DELETE("delete"); // for table-delete.
+
+   private final String permissionSuffix;
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   TablePermissionSubType(String permissionSuffix)
+   {
+      this.permissionSuffix = permissionSuffix;
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for permissionSuffix
+    *******************************************************************************/
+   public String getPermissionSuffix()
+   {
+      return (this.permissionSuffix);
+   }
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/processes/QProcessCallback.java

@@ -39,10 +39,16 @@ public interface QProcessCallback
    /*******************************************************************************
     ** Get the filter query for this callback.
     *******************************************************************************/
-   QQueryFilter getQueryFilter();
+   default QQueryFilter getQueryFilter()
+   {
+      return (null);
+   }
 
    /*******************************************************************************
     ** Get the field values for this callback.
     *******************************************************************************/
-   Map<String, Serializable> getFieldValues(List<QFieldMetaData> fields);
+   default Map<String, Serializable> getFieldValues(List<QFieldMetaData> fields)
+   {
+      return (null);
+   }
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/processes/RunBackendStepAction.java

@@ -31,6 +31,7 @@ import com.kingsrook.qqq.backend.core.actions.ActionHelper;
 import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.exceptions.QUserFacingException;
+import com.kingsrook.qqq.backend.core.logging.QLogger;
 import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
 import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
@@ -42,8 +43,6 @@ import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionInputMet
 import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.processes.QStepMetaData;
 import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
 
 
 /*******************************************************************************
@@ -52,7 +51,7 @@ import org.apache.logging.log4j.Logger;
  *******************************************************************************/
 public class RunBackendStepAction
 {
-   private static final Logger LOG = LogManager.getLogger(RunBackendStepAction.class);
+   private static final QLogger LOG = QLogger.getLogger(RunBackendStepAction.class);
 
 
 
@@ -107,7 +106,7 @@ public class RunBackendStepAction
          return;
       }
 
-      List<QFieldMetaData> fieldsToGet = new ArrayList<>();
+      List<QFieldMetaData> fieldsToGet           = new ArrayList<>();
       List<QFieldMetaData> requiredFieldsMissing = new ArrayList<>();
       for(QFieldMetaData field : inputMetaData.getFieldList())
       {
@@ -175,8 +174,7 @@ public class RunBackendStepAction
       {
          if(CollectionUtils.nullSafeIsEmpty(runBackendStepInput.getRecords()))
          {
-            QueryInput queryInput = new QueryInput(runBackendStepInput.getInstance());
-            queryInput.setSession(runBackendStepInput.getSession());
+            QueryInput queryInput = new QueryInput();
             queryInput.setTableName(inputMetaData.getRecordListMetaData().getTableName());
 
             // todo - handle this being async (e.g., http)
@@ -215,7 +213,7 @@ public class RunBackendStepAction
          Object   codeObject = codeClass.getConstructor().newInstance();
          if(!(codeObject instanceof BackendStep backendStepCodeObject))
          {
-            throw (new QException("The supplied code [" + codeClass.getName() + "] is not an instance of FunctionBody"));
+            throw (new QException("The supplied code [" + codeClass.getName() + "] is not an instance of BackendStep"));
          }
 
          backendStepCodeObject.run(runBackendStepInput, runBackendStepOutput);

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/processes/RunProcessAction.java

@@ -23,28 +23,49 @@ package com.kingsrook.qqq.backend.core.actions.processes;
 
 
 import java.io.Serializable;
+import java.time.Instant;
+import java.time.temporal.ChronoUnit;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
 import java.util.UUID;
 import com.kingsrook.qqq.backend.core.actions.ActionHelper;
+import com.kingsrook.qqq.backend.core.actions.dashboard.widgets.NoCodeWidgetRenderer;
+import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
+import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
+import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.logging.QLogger;
 import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessState;
 import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
 import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
 import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput;
 import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessOutput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.processes.NoCodeWidgetFrontendComponentMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendComponentMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendStepMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.processes.QStepMetaData;
+import com.kingsrook.qqq.backend.core.processes.implementations.basepull.BasepullConfiguration;
 import com.kingsrook.qqq.backend.core.state.InMemoryStateProvider;
 import com.kingsrook.qqq.backend.core.state.StateProviderInterface;
 import com.kingsrook.qqq.backend.core.state.StateType;
 import com.kingsrook.qqq.backend.core.state.UUIDAndTypeStateKey;
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
+import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
+import com.kingsrook.qqq.backend.core.utils.StringUtils;
+import com.kingsrook.qqq.backend.core.utils.ValueUtils;
+import org.apache.commons.lang.BooleanUtils;
 
 
 /*******************************************************************************
@@ -53,7 +74,17 @@ import org.apache.logging.log4j.Logger;
  *******************************************************************************/
 public class RunProcessAction
 {
-   private static final Logger LOG = LogManager.getLogger(RunProcessAction.class);
+   private static final QLogger LOG = QLogger.getLogger(RunProcessAction.class);
+
+   public static final String BASEPULL_THIS_RUNTIME_KEY = "basepullThisRuntimeKey";
+   public static final String BASEPULL_LAST_RUNTIME_KEY = "basepullLastRuntimeKey";
+   public static final String BASEPULL_TIMESTAMP_FIELD  = "basepullTimestampField";
+
+   ////////////////////////////////////////////////////////////////////////////////////////////////
+   // indicator that the timestamp field should be updated - e.g., the execute step is finished. //
+   ////////////////////////////////////////////////////////////////////////////////////////////////
+   public static final String BASEPULL_READY_TO_UPDATE_TIMESTAMP_FIELD = "basepullReadyToUpdateTimestamp";
+   public static final String BASEPULL_DID_QUERY_USING_TIMESTAMP_FIELD = "basepullDidQueryUsingTimestamp";
 
 
 
@@ -82,16 +113,41 @@ public class RunProcessAction
       runProcessOutput.setProcessUUID(runProcessInput.getProcessUUID());
 
       UUIDAndTypeStateKey stateKey     = new UUIDAndTypeStateKey(UUID.fromString(runProcessInput.getProcessUUID()), StateType.PROCESS_STATUS);
-      ProcessState        processState = primeProcessState(runProcessInput, stateKey);
+      ProcessState        processState = primeProcessState(runProcessInput, stateKey, process);
+
+      /////////////////////////////////////////////////////////
+      // if process is 'basepull' style, keep track of 'now' //
+      /////////////////////////////////////////////////////////
+      BasepullConfiguration basepullConfiguration = process.getBasepullConfiguration();
+      if(basepullConfiguration != null)
+      {
+         ///////////////////////////////////////
+         // get the stored basepull timestamp //
+         ///////////////////////////////////////
+         persistLastRunTime(runProcessInput, process, basepullConfiguration);
+      }
 
-      // todo - custom routing
-      List<QStepMetaData> stepList = getAvailableStepList(process, runProcessInput);
       try
       {
+         String lastStepName = runProcessInput.getStartAfterStep();
+
          STEP_LOOP:
-         for(QStepMetaData step : stepList)
+         while(true)
          {
-            if(step instanceof QFrontendStepMetaData)
+            ///////////////////////////////////////////////////////////////////////////////////////////////////////
+            // always refresh the step list - as any step that runs can modify it (in the process state).        //
+            // this is why we don't do a loop over the step list - as we'd get ConcurrentModificationExceptions. //
+            ///////////////////////////////////////////////////////////////////////////////////////////////////////
+            List<QStepMetaData> stepList = getAvailableStepList(processState, process, lastStepName);
+            if(stepList.isEmpty())
+            {
+               break;
+            }
+
+            QStepMetaData step = stepList.get(0);
+            lastStepName = step.getName();
+
+            if(step instanceof QFrontendStepMetaData frontendStep)
             {
                ////////////////////////////////////////////////////////////////
                // Handle what to do with frontend steps, per request setting //
@@ -100,13 +156,15 @@ public class RunProcessAction
                {
                   case BREAK ->
                   {
-                     LOG.info("Breaking process [" + process.getName() + "] at frontend step (as requested by caller): " + step.getName());
+                     LOG.trace("Breaking process [" + process.getName() + "] at frontend step (as requested by caller): " + step.getName());
+                     processFrontendStepFieldDefaultValues(processState, frontendStep);
+                     processFrontendComponents(processState, frontendStep);
                      processState.setNextStepName(step.getName());
                      break STEP_LOOP;
                   }
                   case SKIP ->
                   {
-                     LOG.info("Skipping frontend step [" + step.getName() + "] in process [" + process.getName() + "] (as requested by caller)");
+                     LOG.trace("Skipping frontend step [" + step.getName() + "] in process [" + process.getName() + "] (as requested by caller)");
 
                      //////////////////////////////////////////////////////////////////////
                      // much less error prone in case this code changes in the future... //
@@ -116,7 +174,7 @@ public class RunProcessAction
                   }
                   case FAIL ->
                   {
-                     LOG.info("Throwing error for frontend step [" + step.getName() + "] in process [" + process.getName() + "] (as requested by caller)");
+                     LOG.trace("Throwing error for frontend step [" + step.getName() + "] in process [" + process.getName() + "] (as requested by caller)");
                      throw (new QException("Failing process at step " + step.getName() + " (as requested, to fail on frontend steps)"));
                   }
                   default -> throw new IllegalStateException("Unexpected value: " + runProcessInput.getFrontendStepBehavior());
@@ -127,6 +185,7 @@ public class RunProcessAction
                ///////////////////////
                // Run backend steps //
                ///////////////////////
+               LOG.debug("Running backend step [" + step.getName() + "] in process [" + process.getName() + "]");
                runBackendStep(runProcessInput, process, runProcessOutput, stateKey, backendStepMetaData, process, processState);
             }
             else
@@ -137,6 +196,18 @@ public class RunProcessAction
                throw (new QException("Unsure how to run a step of type: " + step.getClass().getName()));
             }
          }
+
+         ///////////////////////////////////////////////////////////////////////////
+         // if 'basepull' style process, update the stored basepull timestamp     //
+         // but only when we've been signaled to do so - i.e., only if we did our //
+         // query using the timestamp field, and only after an Execute step runs. //
+         ///////////////////////////////////////////////////////////////////////////
+         if(basepullConfiguration != null
+            && BooleanUtils.isTrue(ValueUtils.getValueAsBoolean(runProcessInput.getValue(BASEPULL_DID_QUERY_USING_TIMESTAMP_FIELD)))
+            && BooleanUtils.isTrue(ValueUtils.getValueAsBoolean(runProcessInput.getValue(BASEPULL_READY_TO_UPDATE_TIMESTAMP_FIELD))))
+         {
+            storeLastRunTime(runProcessInput, process, basepullConfiguration);
+         }
       }
       catch(QException qe)
       {
@@ -165,11 +236,47 @@ public class RunProcessAction
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private void processFrontendComponents(ProcessState processState, QFrontendStepMetaData frontendStep) throws QException
+   {
+      for(QFrontendComponentMetaData component : CollectionUtils.nonNullList(frontendStep.getComponents()))
+      {
+         if(component instanceof NoCodeWidgetFrontendComponentMetaData noCodeWidgetComponent)
+         {
+            NoCodeWidgetRenderer noCodeWidgetRenderer = new NoCodeWidgetRenderer();
+            Map<String, Object>  context              = noCodeWidgetRenderer.initContext(null);
+            context.putAll(processState.getValues());
+            String html = noCodeWidgetRenderer.renderOutputs(context, noCodeWidgetComponent.getOutputs());
+            processState.getValues().put(frontendStep.getName() + ".html", html);
+         }
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private void processFrontendStepFieldDefaultValues(ProcessState processState, QFrontendStepMetaData step)
+   {
+      for(QFieldMetaData formField : CollectionUtils.mergeLists(step.getFormFields(), step.getInputFields(), step.getViewFields(), step.getOutputFields()))
+      {
+         if(formField.getDefaultValue() != null && processState.getValues().get(formField.getName()) == null)
+         {
+            processState.getValues().put(formField.getName(), formField.getDefaultValue());
+         }
+      }
+   }
+
+
+
    /*******************************************************************************
     ** When we start running a process (or resuming it), get data in the RunProcessRequest
     ** either from the state provider (if they're found, for a resume).
     *******************************************************************************/
-   ProcessState primeProcessState(RunProcessInput runProcessInput, UUIDAndTypeStateKey stateKey) throws QException
+   ProcessState primeProcessState(RunProcessInput runProcessInput, UUIDAndTypeStateKey stateKey, QProcessMetaData process) throws QException
    {
       Optional<ProcessState> optionalProcessState = loadState(stateKey);
       if(optionalProcessState.isEmpty())
@@ -177,11 +284,14 @@ public class RunProcessAction
          if(runProcessInput.getStartAfterStep() == null)
          {
             ///////////////////////////////////////////////////////////////////////////////////
-            // this is fine - it means its our first time running in the backend.            //
+            // This condition (no state in state-provider, and no start-after-step) means    //
+            // that we're starting a new process!  Init the process state here, then         //
             // Go ahead and store the state that we have (e.g., w/ initial records & values) //
             ///////////////////////////////////////////////////////////////////////////////////
-            storeState(stateKey, runProcessInput.getProcessState());
-            optionalProcessState = Optional.of(runProcessInput.getProcessState());
+            ProcessState processState = runProcessInput.getProcessState();
+            processState.setStepList(process.getStepList().stream().map(QStepMetaData::getName).toList());
+            storeState(stateKey, processState);
+            optionalProcessState = Optional.of(processState);
          }
          else
          {
@@ -227,13 +337,33 @@ public class RunProcessAction
     *******************************************************************************/
    private void runBackendStep(RunProcessInput runProcessInput, QProcessMetaData process, RunProcessOutput runProcessOutput, UUIDAndTypeStateKey stateKey, QBackendStepMetaData backendStep, QProcessMetaData qProcessMetaData, ProcessState processState) throws Exception
    {
-      RunBackendStepInput runBackendStepInput = new RunBackendStepInput(runProcessInput.getInstance(), processState);
+      RunBackendStepInput runBackendStepInput = new RunBackendStepInput(processState);
       runBackendStepInput.setProcessName(process.getName());
       runBackendStepInput.setStepName(backendStep.getName());
-      runBackendStepInput.setTableName(process.getTableName());
-      runBackendStepInput.setSession(runProcessInput.getSession());
       runBackendStepInput.setCallback(runProcessInput.getCallback());
+      runBackendStepInput.setFrontendStepBehavior(runProcessInput.getFrontendStepBehavior());
       runBackendStepInput.setAsyncJobCallback(runProcessInput.getAsyncJobCallback());
+
+      runBackendStepInput.setTableName(process.getTableName());
+      if(!StringUtils.hasContent(runBackendStepInput.getTableName()))
+      {
+         ////////////////////////////////////////////////////////////////
+         // help support generic (e.g., not tied-to-a-table) processes //
+         ////////////////////////////////////////////////////////////////
+         if(runProcessInput.getValue("tableName") != null)
+         {
+            runBackendStepInput.setTableName(ValueUtils.getValueAsString(runProcessInput.getValue("tableName")));
+         }
+      }
+
+      ///////////////////////////////////////////////////////////////
+      // if 'basepull' values are in the inputs, add to step input //
+      ///////////////////////////////////////////////////////////////
+      if(runProcessInput.getValues().containsKey(BASEPULL_LAST_RUNTIME_KEY))
+      {
+         runBackendStepInput.setBasepullLastRunTime((Instant) runProcessInput.getValues().get(BASEPULL_LAST_RUNTIME_KEY));
+      }
+
       RunBackendStepOutput lastFunctionResult = new RunBackendStepAction().execute(runBackendStepInput);
       storeState(stateKey, lastFunctionResult.getProcessState());
 
@@ -249,41 +379,63 @@ public class RunProcessAction
    /*******************************************************************************
     ** Get the list of steps which are eligible to run.
     *******************************************************************************/
-   private List<QStepMetaData> getAvailableStepList(QProcessMetaData process, RunProcessInput runProcessInput)
+   private List<QStepMetaData> getAvailableStepList(ProcessState processState, QProcessMetaData process, String lastStep) throws QException
    {
-      if(runProcessInput.getStartAfterStep() == null)
+      if(lastStep == null)
       {
-         /////////////////////////////////////////////////////////////////////////////
-         // if the caller did not supply a 'startAfterStep', then use the full list //
-         /////////////////////////////////////////////////////////////////////////////
-         return (process.getStepList());
+         ///////////////////////////////////////////////////////////////////////
+         // if the caller did not supply a 'lastStep', then use the full list //
+         ///////////////////////////////////////////////////////////////////////
+         return (stepNamesToSteps(process, processState.getStepList()));
       }
       else
       {
-         ////////////////////////////////////////////////////////////////////////////////
-         // else, loop until the startAfterStep is found, and return the ones after it //
-         ////////////////////////////////////////////////////////////////////////////////
-         boolean             foundStartAfterStep = false;
-         List<QStepMetaData> rs                  = new ArrayList<>();
+         ////////////////////////////////////////////////////////////////////////////
+         // else, loop until the 'lastStep' is found, and return the ones after it //
+         ////////////////////////////////////////////////////////////////////////////
+         boolean      foundLastStep  = false;
+         List<String> validStepNames = new ArrayList<>();
 
-         for(QStepMetaData step : process.getStepList())
+         for(String stepName : processState.getStepList())
          {
-            if(foundStartAfterStep)
+            if(foundLastStep)
             {
-               rs.add(step);
+               validStepNames.add(stepName);
             }
 
-            if(step.getName().equals(runProcessInput.getStartAfterStep()))
+            if(stepName.equals(lastStep))
             {
-               foundStartAfterStep = true;
+               foundLastStep = true;
             }
          }
-         return (rs);
+         return (stepNamesToSteps(process, validStepNames));
       }
    }
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private List<QStepMetaData> stepNamesToSteps(QProcessMetaData process, List<String> stepNames) throws QException
+   {
+      List<QStepMetaData> result = new ArrayList<>();
+
+      for(String stepName : stepNames)
+      {
+         QStepMetaData step = process.getStep(stepName);
+         if(step == null)
+         {
+            throw (new QException("Could not find a step named [" + stepName + "] in this process."));
+         }
+         result.add(step);
+      }
+
+      return (result);
+   }
+
+
+
    /*******************************************************************************
     ** Load an instance of the appropriate state provider
     **
@@ -329,4 +481,125 @@ public class RunProcessAction
       return (getStateProvider().get(ProcessState.class, stateKey));
    }
 
+
+
+   /*******************************************************************************
+    ** Insert or update the last runtime value for this basepull into the backend.
+    *******************************************************************************/
+   protected void storeLastRunTime(RunProcessInput runProcessInput, QProcessMetaData process, BasepullConfiguration basepullConfiguration) throws QException
+   {
+      String basepullTableName            = basepullConfiguration.getTableName();
+      String basepullKeyFieldName         = basepullConfiguration.getKeyField();
+      String basepullLastRunTimeFieldName = basepullConfiguration.getLastRunTimeFieldName();
+      String basepullKeyValue             = (basepullConfiguration.getKeyValue() != null) ? basepullConfiguration.getKeyValue() : process.getName();
+
+      ///////////////////////////////////////
+      // get the stored basepull timestamp //
+      ///////////////////////////////////////
+      QueryInput queryInput = new QueryInput();
+      queryInput.setTableName(basepullTableName);
+      queryInput.setFilter(new QQueryFilter().withCriteria(
+         new QFilterCriteria()
+            .withFieldName(basepullKeyFieldName)
+            .withOperator(QCriteriaOperator.EQUALS)
+            .withValues(List.of(basepullKeyValue))));
+      QueryOutput queryOutput = new QueryAction().execute(queryInput);
+
+      //////////////////////////////////////////
+      // get the runtime for this process run //
+      //////////////////////////////////////////
+      Instant newRunTime = (Instant) runProcessInput.getValues().get(BASEPULL_THIS_RUNTIME_KEY);
+
+      /////////////////////////////////////////////////
+      // update if found, otherwise insert new value //
+      /////////////////////////////////////////////////
+      if(CollectionUtils.nullSafeHasContents(queryOutput.getRecords()))
+      {
+         ///////////////////////////////////////////////////////////////////////////////
+         // update the basepull table with 'now' (which is before original query ran) //
+         ///////////////////////////////////////////////////////////////////////////////
+         QRecord basepullRecord = queryOutput.getRecords().get(0);
+         basepullRecord.setValue(basepullLastRunTimeFieldName, newRunTime);
+
+         ////////////
+         // update //
+         ////////////
+         UpdateInput updateInput = new UpdateInput();
+         updateInput.setTableName(basepullTableName);
+         updateInput.setRecords(List.of(basepullRecord));
+         new UpdateAction().execute(updateInput);
+      }
+      else
+      {
+         QRecord basepullRecord = new QRecord()
+            .withValue(basepullKeyFieldName, basepullKeyValue)
+            .withValue(basepullLastRunTimeFieldName, newRunTime);
+
+         ////////////////////////////////
+         // insert new basepull record //
+         ////////////////////////////////
+         InsertInput insertInput = new InsertInput();
+         insertInput.setTableName(basepullTableName);
+         insertInput.setRecords(List.of(basepullRecord));
+         new InsertAction().execute(insertInput);
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** Lookup the last runtime for this basepull, and set it (plus now) in the process's
+    ** values.
+    *******************************************************************************/
+   protected void persistLastRunTime(RunProcessInput runProcessInput, QProcessMetaData process, BasepullConfiguration basepullConfiguration) throws QException
+   {
+      ////////////////////////////////////////////////////////
+      // if these values were already computed, don't re-do //
+      ////////////////////////////////////////////////////////
+      if(runProcessInput.getValue(BASEPULL_THIS_RUNTIME_KEY) != null)
+      {
+         return;
+      }
+
+      /////////////////////////////////////////////////////////////////////////////////////////////////
+      // store 'now', which will be used to update basepull record if process completes successfully //
+      /////////////////////////////////////////////////////////////////////////////////////////////////
+      Instant now = Instant.now();
+      runProcessInput.getValues().put(BASEPULL_THIS_RUNTIME_KEY, now);
+
+      String  basepullTableName                    = basepullConfiguration.getTableName();
+      String  basepullKeyFieldName                 = basepullConfiguration.getKeyField();
+      String  basepullLastRunTimeFieldName         = basepullConfiguration.getLastRunTimeFieldName();
+      Integer basepullHoursBackForInitialTimestamp = basepullConfiguration.getHoursBackForInitialTimestamp();
+      String  basepullKeyValue                     = (basepullConfiguration.getKeyValue() != null) ? basepullConfiguration.getKeyValue() : process.getName();
+
+      ///////////////////////////////////////
+      // get the stored basepull timestamp //
+      ///////////////////////////////////////
+      QueryInput queryInput = new QueryInput();
+      queryInput.setTableName(basepullTableName);
+      queryInput.setFilter(new QQueryFilter().withCriteria(
+         new QFilterCriteria()
+            .withFieldName(basepullKeyFieldName)
+            .withOperator(QCriteriaOperator.EQUALS)
+            .withValues(List.of(basepullKeyValue))));
+      QueryOutput queryOutput = new QueryAction().execute(queryInput);
+
+      ///////////////////////////////////////////////////////////////////////////////////////////////////
+      // get the stored time, if not, default to 'now' unless a number of hours to offset was provided //
+      ///////////////////////////////////////////////////////////////////////////////////////////////////
+      Instant lastRunTime = now;
+      if(CollectionUtils.nullSafeHasContents(queryOutput.getRecords()))
+      {
+         QRecord basepullRecord = queryOutput.getRecords().get(0);
+         lastRunTime = ValueUtils.getValueAsInstant(basepullRecord.getValue(basepullLastRunTimeFieldName));
+      }
+      else if(basepullHoursBackForInitialTimestamp != null)
+      {
+         lastRunTime = lastRunTime.minus(basepullHoursBackForInitialTimestamp, ChronoUnit.HOURS);
+      }
+
+      runProcessInput.getValues().put(BASEPULL_LAST_RUNTIME_KEY, lastRunTime);
+      runProcessInput.getValues().put(BASEPULL_TIMESTAMP_FIELD, basepullConfiguration.getTimestampField());
+   }
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/queues/GetQueueSize.java

@@ -0,0 +1,84 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2023.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.actions.queues;
+
+
+import java.util.List;
+import com.amazonaws.auth.AWSStaticCredentialsProvider;
+import com.amazonaws.auth.BasicAWSCredentials;
+import com.amazonaws.services.sqs.AmazonSQS;
+import com.amazonaws.services.sqs.AmazonSQSClientBuilder;
+import com.amazonaws.services.sqs.model.GetQueueAttributesResult;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.logging.QLogger;
+import com.kingsrook.qqq.backend.core.model.metadata.queues.QQueueMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.queues.QQueueProviderMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.queues.SQSQueueProviderMetaData;
+import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
+
+
+/*******************************************************************************
+ **
+ *******************************************************************************/
+public class GetQueueSize
+{
+   private static final QLogger LOG = QLogger.getLogger(GetQueueSize.class);
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public Integer getQueueSize(QQueueProviderMetaData queueProviderMetaData, QQueueMetaData queueMetaData) throws QException
+   {
+      try
+      {
+         //////////////////////////////////////////////////////////////////
+         // todo - handle other queue provider types, somewhere, somehow //
+         //////////////////////////////////////////////////////////////////
+         SQSQueueProviderMetaData queueProvider = (SQSQueueProviderMetaData) queueProviderMetaData;
+
+         BasicAWSCredentials credentials = new BasicAWSCredentials(queueProvider.getAccessKey(), queueProvider.getSecretKey());
+         final AmazonSQS sqs = AmazonSQSClientBuilder.standard()
+            .withRegion(queueProvider.getRegion())
+            .withCredentials(new AWSStaticCredentialsProvider(credentials))
+            .build();
+
+         String queueUrl = queueProvider.getBaseURL();
+         if(!queueUrl.endsWith("/"))
+         {
+            queueUrl += "/";
+         }
+         queueUrl += queueMetaData.getQueueName();
+
+         GetQueueAttributesResult queueAttributes             = sqs.getQueueAttributes(queueUrl, List.of("ApproximateNumberOfMessages"));
+         String                   approximateNumberOfMessages = queueAttributes.getAttributes().get("ApproximateNumberOfMessages");
+         return (Integer.parseInt(approximateNumberOfMessages));
+      }
+      catch(Exception e)
+      {
+         LOG.warn("Error getting queue size", e, logPair("queueName", queueMetaData == null ? "null" : queueMetaData.getName()));
+         throw (new QException("Error getting queue size", e));
+      }
+   }
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/queues/SQSQueuePoller.java

@@ -0,0 +1,221 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.actions.queues;
+
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.function.Supplier;
+import com.amazonaws.auth.AWSStaticCredentialsProvider;
+import com.amazonaws.auth.BasicAWSCredentials;
+import com.amazonaws.services.sqs.AmazonSQS;
+import com.amazonaws.services.sqs.AmazonSQSClientBuilder;
+import com.amazonaws.services.sqs.model.DeleteMessageBatchRequest;
+import com.amazonaws.services.sqs.model.DeleteMessageBatchRequestEntry;
+import com.amazonaws.services.sqs.model.Message;
+import com.amazonaws.services.sqs.model.ReceiveMessageRequest;
+import com.amazonaws.services.sqs.model.ReceiveMessageResult;
+import com.kingsrook.qqq.backend.core.actions.processes.RunProcessAction;
+import com.kingsrook.qqq.backend.core.context.QContext;
+import com.kingsrook.qqq.backend.core.logging.QLogger;
+import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput;
+import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessOutput;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.queues.QQueueMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.queues.SQSQueueProviderMetaData;
+import com.kingsrook.qqq.backend.core.model.session.QSession;
+
+
+/*******************************************************************************
+ ** Class to poll an SQS queue, and run process code for each message found.
+ *******************************************************************************/
+public class SQSQueuePoller implements Runnable
+{
+   private static final QLogger LOG = QLogger.getLogger(SQSQueuePoller.class);
+
+   ///////////////////////////////////////////////
+   // todo - move these 2 to a "QBaseRunnable"? //
+   ///////////////////////////////////////////////
+   private QInstance          qInstance;
+   private Supplier<QSession> sessionSupplier;
+
+   private SQSQueueProviderMetaData queueProviderMetaData;
+   private QQueueMetaData           queueMetaData;
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public void run()
+   {
+      QContext.init(qInstance, sessionSupplier.get());
+
+      String originalThreadName = Thread.currentThread().getName();
+      Thread.currentThread().setName("SQSPoller>" + queueMetaData.getName());
+      LOG.debug("Running " + this.getClass().getSimpleName() + "[" + queueMetaData.getName() + "]");
+
+      try
+      {
+         BasicAWSCredentials credentials = new BasicAWSCredentials(queueProviderMetaData.getAccessKey(), queueProviderMetaData.getSecretKey());
+         final AmazonSQS sqs = AmazonSQSClientBuilder.standard()
+            .withRegion(queueProviderMetaData.getRegion())
+            .withCredentials(new AWSStaticCredentialsProvider(credentials))
+            .build();
+
+         String queueUrl = queueProviderMetaData.getBaseURL();
+         if(!queueUrl.endsWith("/"))
+         {
+            queueUrl += "/";
+         }
+         queueUrl += queueMetaData.getQueueName();
+
+         while(true)
+         {
+            ///////////////////////////////
+            // fetch a batch of messages //
+            ///////////////////////////////
+            ReceiveMessageRequest receiveMessageRequest = new ReceiveMessageRequest();
+            receiveMessageRequest.setQueueUrl(queueUrl);
+            receiveMessageRequest.setMaxNumberOfMessages(10);
+            receiveMessageRequest.setWaitTimeSeconds(20); // help urge SQS to query multiple servers and find more messages
+            ReceiveMessageResult receiveMessageResult = sqs.receiveMessage(receiveMessageRequest);
+            if(receiveMessageResult.getMessages().isEmpty())
+            {
+               LOG.debug("0 messages received.  Breaking.");
+               break;
+            }
+            LOG.debug(receiveMessageResult.getMessages().size() + " messages received.  Processing.");
+
+            //////////////////////////////////////////////////////////////////////////////////////////////////////////////
+            // extract data from the messages into list of bodies to pass into process, and list of delete-batch-inputs //
+            //////////////////////////////////////////////////////////////////////////////////////////////////////////////
+            List<DeleteMessageBatchRequestEntry> deleteRequestEntries = new ArrayList<>();
+            ArrayList<String>                    bodies               = new ArrayList<>();
+            int                                  i                    = 0;
+            for(Message message : receiveMessageResult.getMessages())
+            {
+               bodies.add(message.getBody());
+               deleteRequestEntries.add(new DeleteMessageBatchRequestEntry(String.valueOf(i++), message.getReceiptHandle()));
+            }
+
+            /////////////////////////////////////////////////////////////////////////////////////
+            // run the process, in a try-catch, so even if it fails, our loop keeps going.     //
+            // the messages in a failed process will get re-delivered, to try-again, up to the //
+            // number of times configured in AWS                                               //
+            /////////////////////////////////////////////////////////////////////////////////////
+            try
+            {
+               RunProcessInput runProcessInput = new RunProcessInput();
+               runProcessInput.setProcessName(queueMetaData.getProcessName());
+               runProcessInput.setFrontendStepBehavior(RunProcessInput.FrontendStepBehavior.SKIP);
+               runProcessInput.addValue("bodies", bodies);
+
+               QContext.pushAction(runProcessInput);
+
+               RunProcessAction runProcessAction = new RunProcessAction();
+               RunProcessOutput runProcessOutput = runProcessAction.execute(runProcessInput);
+
+               ////////////////////////////////////////////////////////////////////////////////////////////
+               // if there was an exception returned by the process (e.g., thrown in backend step), then //
+               // warn and leave the messages for re-processing.                                         //
+               ////////////////////////////////////////////////////////////////////////////////////////////
+               if(runProcessOutput.getException().isPresent())
+               {
+                  LOG.warn("Exception returned by process when handling SQS Messages.  They will not be deleted from the queue.", runProcessOutput.getException().get());
+               }
+               else
+               {
+                  ///////////////////////////////////////////////
+                  // else, if no exception, do a batch delete. //
+                  ///////////////////////////////////////////////
+                  sqs.deleteMessageBatch(new DeleteMessageBatchRequest()
+                     .withQueueUrl(queueUrl)
+                     .withEntries(deleteRequestEntries));
+               }
+            }
+            catch(Exception e)
+            {
+               LOG.warn("Error receiving SQS Messages.", e);
+            }
+            finally
+            {
+               QContext.popAction();
+            }
+         }
+      }
+      catch(Exception e)
+      {
+         LOG.warn("Error running SQS Queue Poller", e);
+      }
+      finally
+      {
+         Thread.currentThread().setName(originalThreadName);
+         QContext.clear();
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for queueProviderMetaData
+    **
+    *******************************************************************************/
+   public void setQueueProviderMetaData(SQSQueueProviderMetaData queueProviderMetaData)
+   {
+      this.queueProviderMetaData = queueProviderMetaData;
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for queueMetaData
+    **
+    *******************************************************************************/
+   public void setQueueMetaData(QQueueMetaData queueMetaData)
+   {
+      this.queueMetaData = queueMetaData;
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for qInstance
+    **
+    *******************************************************************************/
+   public void setQInstance(QInstance qInstance)
+   {
+      this.qInstance = qInstance;
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for sessionSupplier
+    **
+    *******************************************************************************/
+   public void setSessionSupplier(Supplier<QSession> sessionSupplier)
+   {
+      this.sessionSupplier = sessionSupplier;
+   }
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/BufferedRecordPipe.java

@@ -0,0 +1,89 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.actions.reporting;
+
+
+import java.util.ArrayList;
+import java.util.List;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+
+
+/*******************************************************************************
+ ** Subclass of RecordPipe, which uses a buffer in the addRecord method, to avoid
+ ** sending single-records at a time through postRecordActions and to consumers.
+ *******************************************************************************/
+public class BufferedRecordPipe extends RecordPipe
+{
+   private List<QRecord> buffer     = new ArrayList<>();
+   private Integer       bufferSize = 100;
+
+
+
+   /*******************************************************************************
+    ** Constructor - uses default buffer size
+    **
+    *******************************************************************************/
+   public BufferedRecordPipe()
+   {
+   }
+
+
+
+   /*******************************************************************************
+    ** Constructor - customize buffer size.
+    **
+    *******************************************************************************/
+   public BufferedRecordPipe(Integer bufferSize)
+   {
+      this.bufferSize = bufferSize;
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public void addRecord(QRecord record)
+   {
+      buffer.add(record);
+      if(buffer.size() >= bufferSize)
+      {
+         addRecords(buffer);
+         buffer.clear();
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public void finalFlush()
+   {
+      if(!buffer.isEmpty())
+      {
+         addRecords(buffer);
+         buffer.clear();
+      }
+   }
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/CsvExportStreamer.java

@@ -27,24 +27,24 @@ import java.nio.charset.StandardCharsets;
 import java.util.List;
 import com.kingsrook.qqq.backend.core.adapters.QRecordToCsvAdapter;
 import com.kingsrook.qqq.backend.core.exceptions.QReportingException;
-import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportInput;
+import com.kingsrook.qqq.backend.core.logging.QLogger;
+import com.kingsrook.qqq.backend.core.model.actions.reporting.ExportInput;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
+import com.kingsrook.qqq.backend.core.utils.StringUtils;
 
 
 /*******************************************************************************
- ** CSV report format implementation
+ ** CSV export format implementation
  *******************************************************************************/
-public class CsvReportStreamer implements ReportStreamerInterface
+public class CsvExportStreamer implements ExportStreamerInterface
 {
-   private static final Logger LOG = LogManager.getLogger(CsvReportStreamer.class);
+   private static final QLogger LOG = QLogger.getLogger(CsvExportStreamer.class);
 
    private final QRecordToCsvAdapter qRecordToCsvAdapter;
 
-   private ReportInput          reportInput;
+   private ExportInput          exportInput;
    private QTableMetaData       table;
    private List<QFieldMetaData> fields;
    private OutputStream         outputStream;
@@ -54,7 +54,7 @@ public class CsvReportStreamer implements ReportStreamerInterface
    /*******************************************************************************
     **
     *******************************************************************************/
-   public CsvReportStreamer()
+   public CsvExportStreamer()
    {
       qRecordToCsvAdapter = new QRecordToCsvAdapter();
    }
@@ -65,14 +65,14 @@ public class CsvReportStreamer implements ReportStreamerInterface
     **
     *******************************************************************************/
    @Override
-   public void start(ReportInput reportInput, List<QFieldMetaData> fields) throws QReportingException
+   public void start(ExportInput exportInput, List<QFieldMetaData> fields, String label) throws QReportingException
    {
-      this.reportInput = reportInput;
+      this.exportInput = exportInput;
       this.fields = fields;
-      table = reportInput.getTable();
-      outputStream = this.reportInput.getReportOutputStream();
+      table = exportInput.getTable();
+      outputStream = this.exportInput.getReportOutputStream();
 
-      writeReportHeaderRow();
+      writeTitleAndHeader();
    }
 
 
@@ -80,20 +80,29 @@ public class CsvReportStreamer implements ReportStreamerInterface
    /*******************************************************************************
     **
     *******************************************************************************/
-   private void writeReportHeaderRow() throws QReportingException
+   private void writeTitleAndHeader() throws QReportingException
    {
       try
       {
-         int col = 0;
-         for(QFieldMetaData column : fields)
+         if(StringUtils.hasContent(exportInput.getTitleRow()))
          {
-            if(col++ > 0)
-            {
-               outputStream.write(',');
-            }
-            outputStream.write(('"' + column.getLabel() + '"').getBytes(StandardCharsets.UTF_8));
+            outputStream.write((exportInput.getTitleRow() + "\n").getBytes(StandardCharsets.UTF_8));
          }
-         outputStream.write('\n');
+
+         if(exportInput.getIncludeHeaderRow())
+         {
+            int col = 0;
+            for(QFieldMetaData column : fields)
+            {
+               if(col++ > 0)
+               {
+                  outputStream.write(',');
+               }
+               outputStream.write(('"' + column.getLabel() + '"').getBytes(StandardCharsets.UTF_8));
+            }
+            outputStream.write('\n');
+         }
+
          outputStream.flush();
       }
       catch(Exception e)
@@ -108,20 +117,28 @@ public class CsvReportStreamer implements ReportStreamerInterface
     **
     *******************************************************************************/
    @Override
-   public int takeRecordsFromPipe(RecordPipe recordPipe) throws QReportingException
+   public void addRecords(List<QRecord> qRecords) throws QReportingException
    {
-      List<QRecord> qRecords = recordPipe.consumeAvailableRecords();
       LOG.info("Consuming [" + qRecords.size() + "] records from the pipe");
 
+      for(QRecord qRecord : qRecords)
+      {
+         writeRecord(qRecord);
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private void writeRecord(QRecord qRecord) throws QReportingException
+   {
       try
       {
-         for(QRecord qRecord : qRecords)
-         {
-            String csv = qRecordToCsvAdapter.recordToCsv(table, qRecord, fields);
-            outputStream.write(csv.getBytes(StandardCharsets.UTF_8));
-            outputStream.flush(); // todo - less often?
-         }
-         return (qRecords.size());
+         String csv = qRecordToCsvAdapter.recordToCsv(table, qRecord, fields);
+         outputStream.write(csv.getBytes(StandardCharsets.UTF_8));
+         outputStream.flush(); // todo - less often?
       }
       catch(Exception e)
       {
@@ -131,6 +148,17 @@ public class CsvReportStreamer implements ReportStreamerInterface
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public void addTotalsRow(QRecord record) throws QReportingException
+   {
+      writeRecord(record);
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/ExcelExportStreamer.java

@@ -0,0 +1,343 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.actions.reporting;
+
+
+import java.io.OutputStream;
+import java.io.Serializable;
+import java.time.Instant;
+import java.time.LocalDate;
+import java.time.LocalDateTime;
+import java.time.ZoneId;
+import java.time.ZonedDateTime;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import com.kingsrook.qqq.backend.core.actions.reporting.excelformatting.ExcelStylerInterface;
+import com.kingsrook.qqq.backend.core.actions.reporting.excelformatting.PlainExcelStyler;
+import com.kingsrook.qqq.backend.core.exceptions.QReportingException;
+import com.kingsrook.qqq.backend.core.logging.QLogger;
+import com.kingsrook.qqq.backend.core.model.actions.reporting.ExportInput;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.DisplayFormat;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+import com.kingsrook.qqq.backend.core.utils.StringUtils;
+import com.kingsrook.qqq.backend.core.utils.ValueUtils;
+import org.dhatim.fastexcel.StyleSetter;
+import org.dhatim.fastexcel.Workbook;
+import org.dhatim.fastexcel.Worksheet;
+
+
+/*******************************************************************************
+ ** Excel export format implementation
+ *******************************************************************************/
+public class ExcelExportStreamer implements ExportStreamerInterface
+{
+   private static final QLogger LOG = QLogger.getLogger(ExcelExportStreamer.class);
+
+   private ExportInput          exportInput;
+   private QTableMetaData       table;
+   private List<QFieldMetaData> fields;
+   private OutputStream         outputStream;
+
+   private ExcelStylerInterface excelStylerInterface = new PlainExcelStyler();
+   private Map<String, String>  excelCellFormats;
+
+   private Workbook  workbook;
+   private Worksheet worksheet;
+   private int       row        = 0;
+   private int       sheetCount = 0;
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public ExcelExportStreamer()
+   {
+   }
+
+
+
+   /*******************************************************************************
+    ** display formats is a map of field name to Excel format strings (e.g., $#,##0.00)
+    *******************************************************************************/
+   @Override
+   public void setDisplayFormats(Map<String, String> displayFormats)
+   {
+      this.excelCellFormats = new HashMap<>();
+      for(Map.Entry<String, String> entry : displayFormats.entrySet())
+      {
+         String excelFormat = DisplayFormat.getExcelFormat(entry.getValue());
+         if(excelFormat != null)
+         {
+            excelCellFormats.put(entry.getKey(), excelFormat);
+         }
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** Starts a new worksheet in the current workbook.  Can be called multiple times.
+    *******************************************************************************/
+   @Override
+   public void start(ExportInput exportInput, List<QFieldMetaData> fields, String label) throws QReportingException
+   {
+      try
+      {
+         this.exportInput = exportInput;
+         this.fields = fields;
+         table = exportInput.getTable();
+         outputStream = this.exportInput.getReportOutputStream();
+         this.row = 0;
+         this.sheetCount++;
+
+         /////////////////////////////////////////////////////////////////////////////////////////////////////
+         // if this is the first call in here (e.g., the workbook hasn't been opened yet), then open it now //
+         /////////////////////////////////////////////////////////////////////////////////////////////////////
+         if(workbook == null)
+         {
+            String    appName  = "QQQ";
+            QInstance instance = exportInput.getInstance();
+            if(instance != null && instance.getBranding() != null && instance.getBranding().getCompanyName() != null)
+            {
+               appName = instance.getBranding().getCompanyName();
+            }
+            workbook = new Workbook(outputStream, appName, null);
+         }
+
+         /////////////////////////////////////////////////////////////////////////////////////
+         // if start is called a second time (e.g., and there's already an open worksheet), //
+         // finish that sheet, before a new one is created.                                 //
+         /////////////////////////////////////////////////////////////////////////////////////
+         if(worksheet != null)
+         {
+            worksheet.finish();
+         }
+
+         worksheet = workbook.newWorksheet(Objects.requireNonNullElse(label, "Sheet" + sheetCount));
+
+         writeTitleAndHeader();
+      }
+      catch(Exception e)
+      {
+         throw (new QReportingException("Error starting worksheet", e));
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private void writeTitleAndHeader() throws QReportingException
+   {
+      try
+      {
+         ///////////////
+         // title row //
+         ///////////////
+         if(StringUtils.hasContent(exportInput.getTitleRow()))
+         {
+            worksheet.value(row, 0, exportInput.getTitleRow());
+            worksheet.range(row, 0, row, fields.size() - 1).merge();
+
+            StyleSetter titleStyle = worksheet.range(row, 0, row, fields.size() - 1).style();
+            excelStylerInterface.styleTitleRow(titleStyle);
+            titleStyle.set();
+
+            row++;
+            worksheet.flush();
+         }
+
+         ////////////////
+         // header row //
+         ////////////////
+         if(exportInput.getIncludeHeaderRow())
+         {
+            int col = 0;
+            for(QFieldMetaData column : fields)
+            {
+               worksheet.value(row, col, column.getLabel());
+               col++;
+            }
+
+            StyleSetter headerStyle = worksheet.range(row, 0, row, fields.size() - 1).style();
+            excelStylerInterface.styleHeaderRow(headerStyle);
+            headerStyle.set();
+
+            row++;
+            worksheet.flush();
+         }
+      }
+      catch(Exception e)
+      {
+         throw (new QReportingException("Error starting Excel report"));
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public void addRecords(List<QRecord> qRecords) throws QReportingException
+   {
+      LOG.info("Consuming [" + qRecords.size() + "] records from the pipe");
+
+      try
+      {
+         for(QRecord qRecord : qRecords)
+         {
+            writeRecord(qRecord);
+
+            row++;
+            worksheet.flush(); // todo?  not at all?  or just sometimes?
+         }
+      }
+      catch(Exception e)
+      {
+         LOG.error("Exception generating excel file", e);
+         try
+         {
+            workbook.finish();
+            outputStream.close();
+         }
+         finally
+         {
+            throw (new QReportingException("Error generating Excel report", e));
+         }
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private void writeRecord(QRecord qRecord)
+   {
+      int col = 0;
+      for(QFieldMetaData field : fields)
+      {
+         Serializable value = qRecord.getValue(field.getName());
+
+         if(value != null)
+         {
+            if(value instanceof String s)
+            {
+               worksheet.value(row, col, s);
+            }
+            else if(value instanceof Number n)
+            {
+               worksheet.value(row, col, n);
+
+               if(excelCellFormats != null)
+               {
+                  String format = excelCellFormats.get(field.getName());
+                  if(format != null)
+                  {
+                     worksheet.style(row, col).format(format).set();
+                  }
+               }
+            }
+            else if(value instanceof Boolean b)
+            {
+               worksheet.value(row, col, b);
+            }
+            else if(value instanceof Date d)
+            {
+               worksheet.value(row, col, d);
+               worksheet.style(row, col).format("yyyy-MM-dd").set();
+            }
+            else if(value instanceof LocalDate d)
+            {
+               worksheet.value(row, col, d);
+               worksheet.style(row, col).format("yyyy-MM-dd").set();
+            }
+            else if(value instanceof LocalDateTime d)
+            {
+               worksheet.value(row, col, d);
+               worksheet.style(row, col).format("yyyy-MM-dd H:mm:ss").set();
+            }
+            else if(value instanceof ZonedDateTime d)
+            {
+               worksheet.value(row, col, d);
+               worksheet.style(row, col).format("yyyy-MM-dd H:mm:ss").set();
+            }
+            else if(value instanceof Instant i)
+            {
+               // todo - what would be a better zone to use here?
+               worksheet.value(row, col, i.atZone(ZoneId.systemDefault()));
+               worksheet.style(row, col).format("yyyy-MM-dd H:mm:ss").set();
+            }
+            else
+            {
+               worksheet.value(row, col, ValueUtils.getValueAsString(value));
+            }
+         }
+         col++;
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public void addTotalsRow(QRecord record)
+   {
+      writeRecord(record);
+
+      StyleSetter totalsRowStyle = worksheet.range(row, 0, row, fields.size() - 1).style();
+      excelStylerInterface.styleTotalsRow(totalsRowStyle);
+      totalsRowStyle.set();
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public void finish() throws QReportingException
+   {
+      try
+      {
+         if(workbook != null)
+         {
+            workbook.finish();
+         }
+      }
+      catch(Exception e)
+      {
+         throw (new QReportingException("Error finishing Excel report", e));
+      }
+   }
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/ExcelReportStreamer.java

@@ -1,214 +0,0 @@
-/*
- * QQQ - Low-code Application Framework for Engineers.
- * Copyright (C) 2021-2022.  Kingsrook, LLC
- * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
- * contact@kingsrook.com
- * https://github.com/Kingsrook/
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program.  If not, see <https://www.gnu.org/licenses/>.
- */
-
-package com.kingsrook.qqq.backend.core.actions.reporting;
-
-
-import java.io.OutputStream;
-import java.io.Serializable;
-import java.time.LocalDate;
-import java.time.LocalDateTime;
-import java.time.ZonedDateTime;
-import java.util.Date;
-import java.util.List;
-import com.kingsrook.qqq.backend.core.exceptions.QReportingException;
-import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportInput;
-import com.kingsrook.qqq.backend.core.model.data.QRecord;
-import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
-import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
-import com.kingsrook.qqq.backend.core.utils.ValueUtils;
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
-import org.dhatim.fastexcel.Workbook;
-import org.dhatim.fastexcel.Worksheet;
-
-
-/*******************************************************************************
- ** Excel report format implementation
- *******************************************************************************/
-public class ExcelReportStreamer implements ReportStreamerInterface
-{
-   private static final Logger LOG = LogManager.getLogger(ExcelReportStreamer.class);
-
-   private ReportInput          reportInput;
-   private QTableMetaData       table;
-   private List<QFieldMetaData> fields;
-   private OutputStream         outputStream;
-
-   private Workbook  workbook;
-   private Worksheet worksheet;
-   private int       row = 1;
-
-
-
-   /*******************************************************************************
-    **
-    *******************************************************************************/
-   public ExcelReportStreamer()
-   {
-   }
-
-
-
-   /*******************************************************************************
-    **
-    *******************************************************************************/
-   @Override
-   public void start(ReportInput reportInput, List<QFieldMetaData> fields) throws QReportingException
-   {
-      this.reportInput = reportInput;
-      this.fields = fields;
-      table = reportInput.getTable();
-      outputStream = this.reportInput.getReportOutputStream();
-
-      workbook = new Workbook(outputStream, "QQQ", null);
-      worksheet = workbook.newWorksheet("Sheet 1");
-
-      writeReportHeaderRow();
-   }
-
-
-
-   /*******************************************************************************
-    **
-    *******************************************************************************/
-   private void writeReportHeaderRow() throws QReportingException
-   {
-      try
-      {
-         int col = 0;
-         for(QFieldMetaData column : fields)
-         {
-            worksheet.value(0, col, column.getLabel());
-            col++;
-         }
-
-         worksheet.flush();
-      }
-      catch(Exception e)
-      {
-         throw (new QReportingException("Error starting Excel report"));
-      }
-   }
-
-
-
-   /*******************************************************************************
-    **
-    *******************************************************************************/
-   @Override
-   public int takeRecordsFromPipe(RecordPipe recordPipe) throws QReportingException
-   {
-      List<QRecord> qRecords = recordPipe.consumeAvailableRecords();
-      LOG.info("Consuming [" + qRecords.size() + "] records from the pipe");
-
-      try
-      {
-         for(QRecord qRecord : qRecords)
-         {
-            int col = 0;
-            for(QFieldMetaData column : fields)
-            {
-               Serializable value = qRecord.getValue(column.getName());
-               if(value != null)
-               {
-                  if(value instanceof String s)
-                  {
-                     worksheet.value(row, col, s);
-                  }
-                  else if(value instanceof Number n)
-                  {
-                     worksheet.value(row, col, n);
-                  }
-                  else if(value instanceof Boolean b)
-                  {
-                     worksheet.value(row, col, b);
-                  }
-                  else if(value instanceof Date d)
-                  {
-                     worksheet.value(row, col, d);
-                     worksheet.style(row, col).format("yyyy-MM-dd").set();
-                  }
-                  else if(value instanceof LocalDate d)
-                  {
-                     worksheet.value(row, col, d);
-                     worksheet.style(row, col).format("yyyy-MM-dd").set();
-                  }
-                  else if(value instanceof LocalDateTime d)
-                  {
-                     worksheet.value(row, col, d);
-                     worksheet.style(row, col).format("yyyy-MM-dd H:mm:ss").set();
-                  }
-                  else if(value instanceof ZonedDateTime d)
-                  {
-                     worksheet.value(row, col, d);
-                     worksheet.style(row, col).format("yyyy-MM-dd H:mm:ss").set();
-                  }
-                  else
-                  {
-                     worksheet.value(row, col, ValueUtils.getValueAsString(value));
-                  }
-               }
-               col++;
-            }
-
-            row++;
-            worksheet.flush(); // todo?  not at all?  or just sometimes?
-         }
-      }
-      catch(Exception e)
-      {
-         try
-         {
-            workbook.finish();
-            outputStream.close();
-         }
-         finally
-         {
-            throw (new QReportingException("Error generating Excel report", e));
-         }
-      }
-
-      return (qRecords.size());
-   }
-
-
-
-   /*******************************************************************************
-    **
-    *******************************************************************************/
-   @Override
-   public void finish() throws QReportingException
-   {
-      try
-      {
-         if(workbook != null)
-         {
-            workbook.finish();
-         }
-      }
-      catch(Exception e)
-      {
-         throw (new QReportingException("Error finishing Excel report", e));
-      }
-   }
-
-}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/ExportAction.java

@@ -31,29 +31,30 @@ import com.kingsrook.qqq.backend.core.actions.async.AsyncJobManager;
 import com.kingsrook.qqq.backend.core.actions.async.AsyncJobState;
 import com.kingsrook.qqq.backend.core.actions.async.AsyncJobStatus;
 import com.kingsrook.qqq.backend.core.actions.interfaces.CountInterface;
-import com.kingsrook.qqq.backend.core.actions.interfaces.QueryInterface;
+import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.exceptions.QReportingException;
 import com.kingsrook.qqq.backend.core.exceptions.QUserFacingException;
+import com.kingsrook.qqq.backend.core.logging.QLogger;
+import com.kingsrook.qqq.backend.core.model.actions.reporting.ExportInput;
+import com.kingsrook.qqq.backend.core.model.actions.reporting.ExportOutput;
 import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportFormat;
-import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportInput;
-import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportOutput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountOutput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleDispatcher;
 import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleInterface;
 import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
 import com.kingsrook.qqq.backend.core.utils.SleepUtils;
 import com.kingsrook.qqq.backend.core.utils.StringUtils;
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
 
 
 /*******************************************************************************
- ** Action to generate a report.
+ ** Action to generate an export from a table
  **
  ** At this time (future may change?), this action starts a new thread to run
  ** the query in the backend module.  As records are produced by the query,
@@ -63,9 +64,9 @@ import org.apache.logging.log4j.Logger;
  ** time the report outputStream can be closed.
  **
  *******************************************************************************/
-public class ReportAction
+public class ExportAction
 {
-   private static final Logger LOG = LogManager.getLogger(ReportAction.class);
+   private static final QLogger LOG = QLogger.getLogger(ExportAction.class);
 
    private boolean preExecuteRan       = false;
    private Integer countFromPreExecute = null;
@@ -82,21 +83,21 @@ public class ReportAction
     ** first, in their thread, to catch any validation errors before they start
     ** the thread (which they may abandon).
     *******************************************************************************/
-   public void preExecute(ReportInput reportInput) throws QException
+   public void preExecute(ExportInput exportInput) throws QException
    {
-      ActionHelper.validateSession(reportInput);
+      ActionHelper.validateSession(exportInput);
 
       QBackendModuleDispatcher qBackendModuleDispatcher = new QBackendModuleDispatcher();
-      QBackendModuleInterface  backendModule            = qBackendModuleDispatcher.getQBackendModule(reportInput.getBackend());
+      QBackendModuleInterface  backendModule            = qBackendModuleDispatcher.getQBackendModule(exportInput.getBackend());
 
       ///////////////////////////////////
       // verify field names (if given) //
       ///////////////////////////////////
-      if(CollectionUtils.nullSafeHasContents(reportInput.getFieldNames()))
+      if(CollectionUtils.nullSafeHasContents(exportInput.getFieldNames()))
       {
-         QTableMetaData table         = reportInput.getTable();
+         QTableMetaData table         = exportInput.getTable();
          List<String>   badFieldNames = new ArrayList<>();
-         for(String fieldName : reportInput.getFieldNames())
+         for(String fieldName : exportInput.getFieldNames())
          {
             try
             {
@@ -119,8 +120,8 @@ public class ReportAction
       ///////////////////////////////////////////////////////////////////////////////////////////////////////////
       // check if this report format has a max-rows limit -- if so, do a count to verify we're under the limit //
       ///////////////////////////////////////////////////////////////////////////////////////////////////////////
-      ReportFormat reportFormat = reportInput.getReportFormat();
-      verifyCountUnderMax(reportInput, backendModule, reportFormat);
+      ReportFormat reportFormat = exportInput.getReportFormat();
+      verifyCountUnderMax(exportInput, backendModule, reportFormat);
 
       preExecuteRan = true;
    }
@@ -130,47 +131,48 @@ public class ReportAction
    /*******************************************************************************
     ** Run the report.
     *******************************************************************************/
-   public ReportOutput execute(ReportInput reportInput) throws QException
+   public ExportOutput execute(ExportInput exportInput) throws QException
    {
       if(!preExecuteRan)
       {
          /////////////////////////////////////
          // ensure that pre-execute has ran //
          /////////////////////////////////////
-         preExecute(reportInput);
+         preExecute(exportInput);
       }
 
       QBackendModuleDispatcher qBackendModuleDispatcher = new QBackendModuleDispatcher();
-      QBackendModuleInterface  backendModule            = qBackendModuleDispatcher.getQBackendModule(reportInput.getBackend());
+      QBackendModuleInterface  backendModule            = qBackendModuleDispatcher.getQBackendModule(exportInput.getBackend());
 
       //////////////////////////
       // set up a query input //
       //////////////////////////
-      QueryInterface queryInterface = backendModule.getQueryInterface();
-      QueryInput     queryInput     = new QueryInput(reportInput.getInstance());
-      queryInput.setSession(reportInput.getSession());
-      queryInput.setTableName(reportInput.getTableName());
-      queryInput.setFilter(reportInput.getQueryFilter());
-      queryInput.setLimit(reportInput.getLimit());
+      QueryAction queryAction = new QueryAction();
+      QueryInput  queryInput  = new QueryInput();
+      queryInput.setTableName(exportInput.getTableName());
+      queryInput.setFilter(exportInput.getQueryFilter());
+      queryInput.setLimit(exportInput.getLimit());
+      queryInput.setShouldTranslatePossibleValues(true);
 
       /////////////////////////////////////////////////////////////////
       // tell this query that it needs to put its output into a pipe //
       /////////////////////////////////////////////////////////////////
-      RecordPipe recordPipe = new RecordPipe();
+      RecordPipe recordPipe = new BufferedRecordPipe(500);
       queryInput.setRecordPipe(recordPipe);
 
       ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
       // set up a report streamer, which will read rows from the pipe, and write formatted report rows to the output stream //
       ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-      ReportFormat            reportFormat   = reportInput.getReportFormat();
-      ReportStreamerInterface reportStreamer = reportFormat.newReportStreamer();
-      reportStreamer.start(reportInput, getFields(reportInput));
+      ReportFormat            reportFormat   = exportInput.getReportFormat();
+      ExportStreamerInterface reportStreamer = reportFormat.newReportStreamer();
+      List<QFieldMetaData>    fields         = getFields(exportInput);
+      reportStreamer.start(exportInput, fields, "Sheet 1");
 
       //////////////////////////////////////////
       // run the query action as an async job //
       //////////////////////////////////////////
       AsyncJobManager asyncJobManager = new AsyncJobManager();
-      String          queryJobUUID    = asyncJobManager.startJob("ReportAction>QueryAction", (status) -> (queryInterface.execute(queryInput)));
+      String          queryJobUUID    = asyncJobManager.startJob("ReportAction>QueryAction", (status) -> (queryAction.execute(queryInput)));
       LOG.info("Started query job [" + queryJobUUID + "] for report");
 
       AsyncJobState  queryJobState  = AsyncJobState.RUNNING;
@@ -207,8 +209,9 @@ public class ReportAction
             lastReceivedRecordsAt = System.currentTimeMillis();
             nextSleepMillis = INIT_SLEEP_MS;
 
-            int recordsConsumed = reportStreamer.takeRecordsFromPipe(recordPipe);
-            recordCount += recordsConsumed;
+            List<QRecord> records = recordPipe.consumeAvailableRecords();
+            processRecords(reportStreamer, fields, records);
+            recordCount += records.size();
 
             LOG.info(countFromPreExecute != null
                ? String.format("Processed %,d of %,d records so far", recordCount, countFromPreExecute)
@@ -235,8 +238,9 @@ public class ReportAction
       ///////////////////////////////////////////////////
       // send the final records to the report streamer //
       ///////////////////////////////////////////////////
-      int recordsConsumed = reportStreamer.takeRecordsFromPipe(recordPipe);
-      recordCount += recordsConsumed;
+      List<QRecord> records = recordPipe.consumeAvailableRecords();
+      processRecords(reportStreamer, fields, records);
+      recordCount += records.size();
 
       long reportEndTime = System.currentTimeMillis();
       LOG.info((countFromPreExecute != null
@@ -251,17 +255,17 @@ public class ReportAction
 
       try
       {
-         reportInput.getReportOutputStream().close();
+         exportInput.getReportOutputStream().close();
       }
       catch(Exception e)
       {
          throw (new QReportingException("Error completing report", e));
       }
 
-      ReportOutput reportOutput = new ReportOutput();
-      reportOutput.setRecordCount(recordCount);
+      ExportOutput exportOutput = new ExportOutput();
+      exportOutput.setRecordCount(recordCount);
 
-      return (reportOutput);
+      return (exportOutput);
    }
 
 
@@ -269,17 +273,56 @@ public class ReportAction
    /*******************************************************************************
     **
     *******************************************************************************/
-   private List<QFieldMetaData> getFields(ReportInput reportInput)
+   private static void processRecords(ExportStreamerInterface reportStreamer, List<QFieldMetaData> fields, List<QRecord> records) throws QReportingException
    {
-      QTableMetaData table = reportInput.getTable();
-      if(reportInput.getFieldNames() != null)
+      for(QFieldMetaData field : fields)
       {
-         return (reportInput.getFieldNames().stream().map(table::getField).toList());
+         if(field.getName().endsWith(":possibleValueLabel"))
+         {
+            String effectiveFieldName = field.getName().replace(":possibleValueLabel", "");
+            for(QRecord record : records)
+            {
+               String displayValue = record.getDisplayValue(effectiveFieldName);
+               record.setValue(field.getName(), displayValue);
+            }
+         }
+      }
+
+      reportStreamer.addRecords(records);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private List<QFieldMetaData> getFields(ExportInput exportInput)
+   {
+      List<QFieldMetaData> fieldList;
+      QTableMetaData       table = exportInput.getTable();
+      if(exportInput.getFieldNames() != null)
+      {
+         fieldList = exportInput.getFieldNames().stream().map(table::getField).toList();
       }
       else
       {
-         return (new ArrayList<>(table.getFields().values()));
+         fieldList = new ArrayList<>(table.getFields().values());
       }
+
+      //////////////////////////////////////////
+      // add fields for possible value labels //
+      //////////////////////////////////////////
+      List<QFieldMetaData> returnList = new ArrayList<>();
+      for(QFieldMetaData field : fieldList)
+      {
+         returnList.add(field);
+         if(StringUtils.hasContent(field.getPossibleValueSourceName()))
+         {
+            returnList.add(new QFieldMetaData(field.getName() + ":possibleValueLabel", QFieldType.STRING).withLabel(field.getLabel() + " Name"));
+         }
+      }
+
+      return (returnList);
    }
 
 
@@ -287,12 +330,12 @@ public class ReportAction
    /*******************************************************************************
     **
     *******************************************************************************/
-   private void verifyCountUnderMax(ReportInput reportInput, QBackendModuleInterface backendModule, ReportFormat reportFormat) throws QException
+   private void verifyCountUnderMax(ExportInput exportInput, QBackendModuleInterface backendModule, ReportFormat reportFormat) throws QException
    {
       if(reportFormat.getMaxCols() != null)
       {
-         List<QFieldMetaData> fields = getFields(reportInput);
-         if (fields.size() > reportFormat.getMaxCols())
+         List<QFieldMetaData> fields = getFields(exportInput);
+         if(fields.size() > reportFormat.getMaxCols())
          {
             throw (new QUserFacingException("The requested report would include more columns ("
                + String.format("%,d", fields.size()) + ") than the maximum allowed ("
@@ -302,13 +345,12 @@ public class ReportAction
 
       if(reportFormat.getMaxRows() != null)
       {
-         if(reportInput.getLimit() == null || reportInput.getLimit() > reportFormat.getMaxRows())
+         if(exportInput.getLimit() == null || exportInput.getLimit() > reportFormat.getMaxRows())
          {
             CountInterface countInterface = backendModule.getCountInterface();
-            CountInput     countInput     = new CountInput(reportInput.getInstance());
-            countInput.setSession(reportInput.getSession());
-            countInput.setTableName(reportInput.getTableName());
-            countInput.setFilter(reportInput.getQueryFilter());
+            CountInput     countInput     = new CountInput();
+            countInput.setTableName(exportInput.getTableName());
+            countInput.setFilter(exportInput.getQueryFilter());
             CountOutput countOutput = countInterface.execute(countInput);
             countFromPreExecute = countOutput.getCount();
             if(countFromPreExecute > reportFormat.getMaxRows())

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/ExportStreamerInterface.java

@@ -23,29 +23,46 @@ package com.kingsrook.qqq.backend.core.actions.reporting;
 
 
 import java.util.List;
+import java.util.Map;
 import com.kingsrook.qqq.backend.core.exceptions.QReportingException;
-import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportInput;
+import com.kingsrook.qqq.backend.core.model.actions.reporting.ExportInput;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
 
 
 /*******************************************************************************
- ** Interface for various report formats to implement.
+ ** Interface for various export formats to implement.
  *******************************************************************************/
-public interface ReportStreamerInterface
+public interface ExportStreamerInterface
 {
    /*******************************************************************************
     ** Called once, before any rows are available.  Meant to write a header, for example.
     *******************************************************************************/
-   void start(ReportInput reportInput, List<QFieldMetaData> fields) throws QReportingException;
+   void start(ExportInput exportInput, List<QFieldMetaData> fields, String label) throws QReportingException;
 
    /*******************************************************************************
     ** Called as records flow into the pipe.
     ******************************************************************************/
-   int takeRecordsFromPipe(RecordPipe recordPipe) throws QReportingException;
+   void addRecords(List<QRecord> recordList) throws QReportingException;
 
    /*******************************************************************************
     ** Called once, after all rows are available.  Meant to write a footer, or close resources, for example.
     *******************************************************************************/
    void finish() throws QReportingException;
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   default void setDisplayFormats(Map<String, String> displayFormats)
+   {
+      // noop in base class
+   }
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   default void addTotalsRow(QRecord record) throws QReportingException
+   {
+      addRecords(List.of(record));
+   }
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/FormulaInterpreter.java

@@ -0,0 +1,401 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.actions.reporting;
+
+
+import java.io.Serializable;
+import java.math.BigDecimal;
+import java.math.MathContext;
+import java.math.RoundingMode;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Objects;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.function.Supplier;
+import com.kingsrook.qqq.backend.core.exceptions.QFormulaException;
+import com.kingsrook.qqq.backend.core.exceptions.QValueException;
+import com.kingsrook.qqq.backend.core.instances.QMetaDataVariableInterpreter;
+import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
+import com.kingsrook.qqq.backend.core.utils.ValueUtils;
+
+
+/*******************************************************************************
+ ** Helper for Generating reports - to interpret formulas in report columns,
+ ** that are in "excel-style", ala: =MINUS(47,42) or
+ ** =IF(LT(ADD(${input.x},${input.y}),10,Yes,No)
+ *******************************************************************************/
+public class FormulaInterpreter
+{
+
+   /*******************************************************************************
+    ** public method to interpret a formula.  Takes a variableInterpreter, optionally
+    ** full of maps of variables, and the formula string, assumed to have its leading
+    ** '=' char already trimmed away.
+    *******************************************************************************/
+   public static Serializable interpretFormula(QMetaDataVariableInterpreter variableInterpreter, String formula) throws QFormulaException
+   {
+      try
+      {
+         List<Serializable> results = interpretFormula(variableInterpreter, formula, new AtomicInteger(0));
+         if(results.size() == 1)
+         {
+            return (results.get(0));
+         }
+         else if(results.isEmpty())
+         {
+            throw (new QFormulaException("No results from formula"));
+         }
+         else
+         {
+            throw (new QFormulaException("More than 1 result from formula"));
+         }
+      }
+      catch(Exception e)
+      {
+         throw (new QFormulaException("Error interpreting formula [" + formula + "]", e));
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** Recursive method that does the work of interpreting a formula.
+    ** Uses AtomicInteger `i` to track index through the string into and out of
+    ** recursive calls.
+    *******************************************************************************/
+   static List<Serializable> interpretFormula(QMetaDataVariableInterpreter variableInterpreter, String formula, AtomicInteger i) throws QFormulaException
+   {
+      StringBuilder      token  = new StringBuilder();
+      List<Serializable> result = new ArrayList<>();
+
+      char previousChar = 0;
+      while(i.get() < formula.length())
+      {
+         if(i.get() > 0)
+         {
+            previousChar = formula.charAt(i.get() - 1);
+         }
+         char c = formula.charAt(i.getAndIncrement());
+         if(c == '(' && i.get() < formula.length() - 1)
+         {
+            //////////////////////////////////////////////////////////////////////////////////////////
+            // open paren means:  go into a sub-parse.  Get back a list of arguments, and use those //
+            // as arguments for the current token, which must be a function name then.              //
+            //////////////////////////////////////////////////////////////////////////////////////////
+            List<Serializable> args     = interpretFormula(variableInterpreter, formula, i);
+            Serializable       evaluate = evaluate(token.toString(), args, variableInterpreter);
+            result.add(evaluate);
+         }
+         else if(c == ')')
+         {
+            //////////////////////////////////////////////////////////////////////////
+            // close paren means:  end this sub-parse.  evaluate the current token, //
+            // add it to the result list, and return the result list.               //
+            // unless we just closed a paren - then we can just return.            //
+            //////////////////////////////////////////////////////////////////////////
+            if(previousChar != ')')
+            {
+               Serializable evaluate = evaluate(token.toString(), Collections.emptyList(), variableInterpreter);
+               result.add(evaluate);
+            }
+            return (result);
+         }
+         else if(c == ',')
+         {
+            /////////////////////////////////////////////////////////////////////////
+            // comma means:  evaluate the current token; add it to the result list //
+            // unless we just closed a paren - then we can just return.            //
+            /////////////////////////////////////////////////////////////////////////
+            if(previousChar != ')')
+            {
+               Serializable evaluate = evaluate(token.toString(), Collections.emptyList(), variableInterpreter);
+               result.add(evaluate);
+            }
+            token = new StringBuilder();
+         }
+         else
+         {
+            /////////////////////////////////////////////////
+            // else, we add this char to the current token //
+            /////////////////////////////////////////////////
+            token.append(c);
+         }
+      }
+
+      ////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      // if we haven't found a result yet, assume we have just a literal, not a function call, and evaluate as such //
+      ////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      if(result.isEmpty())
+      {
+         if(!token.isEmpty())
+         {
+            Serializable evaluate = evaluate(token.toString(), Collections.emptyList(), variableInterpreter);
+            result.add(evaluate);
+         }
+      }
+
+      return (result);
+   }
+
+
+
+   /*******************************************************************************
+    ** Evaluate a token - maybe a literal, or variable, or function name -
+    ** with arguments if it's a function, and in the context of the variableInterpreter.
+    *******************************************************************************/
+   private static Serializable evaluate(String token, List<Serializable> args, QMetaDataVariableInterpreter variableInterpreter) throws QFormulaException
+   {
+      switch(token)
+      {
+         case "ADD":
+         {
+            List<BigDecimal> numbers = getNumberArgumentList(args, 2, variableInterpreter);
+            return nullIfAnyNullArgsElseBigDecimal(numbers, () -> numbers.get(0).add(numbers.get(1)));
+         }
+         case "MINUS":
+         {
+            List<BigDecimal> numbers = getNumberArgumentList(args, 2, variableInterpreter);
+            return nullIfAnyNullArgsElseBigDecimal(numbers, () -> numbers.get(0).subtract(numbers.get(1)));
+         }
+         case "MULTIPLY":
+         {
+            List<BigDecimal> numbers = getNumberArgumentList(args, 2, variableInterpreter);
+            return nullIfAnyNullArgsElseBigDecimal(numbers, () -> numbers.get(0).multiply(numbers.get(1)));
+         }
+         case "DIVIDE":
+         {
+            List<BigDecimal> numbers = getNumberArgumentList(args, 2, variableInterpreter);
+            if(numbers.get(1) == null || numbers.get(1).compareTo(BigDecimal.ZERO) == 0)
+            {
+               return null;
+            }
+            return nullIfAnyNullArgsElseBigDecimal(numbers, () -> numbers.get(0).divide(numbers.get(1), 4, RoundingMode.HALF_UP));
+         }
+         case "DIVIDE_SCALE":
+         {
+            List<BigDecimal> numbers = getNumberArgumentList(args, 3, variableInterpreter);
+            if(numbers.get(1) == null || numbers.get(1).compareTo(BigDecimal.ZERO) == 0)
+            {
+               return null;
+            }
+            return nullIfAnyNullArgsElseBigDecimal(numbers, () -> numbers.get(0).divide(numbers.get(1), numbers.get(2).intValue(), RoundingMode.HALF_UP));
+         }
+         case "ROUND":
+         {
+            List<BigDecimal> numbers = getNumberArgumentList(args, 2, variableInterpreter);
+            return nullIfAnyNullArgsElseBigDecimal(numbers, () -> numbers.get(0).round(new MathContext(numbers.get(1).intValue())));
+         }
+         case "SCALE":
+         {
+            List<BigDecimal> numbers = getNumberArgumentList(args, 2, variableInterpreter);
+            return nullIfAnyNullArgsElseBigDecimal(numbers, () -> numbers.get(0).setScale(numbers.get(1).intValue(), RoundingMode.HALF_UP));
+         }
+         case "NVL":
+         {
+            List<BigDecimal> numbers = getNumberArgumentList(args, 2, variableInterpreter);
+            return Objects.requireNonNullElse(numbers.get(0), numbers.get(1));
+         }
+         case "IF":
+         {
+            ///////////////////////////////////////////////////////////////////////////////////////
+            // IF(CONDITION,TRUE,ELSE)                                                           //
+            // behavior in a spreadsheet appears to be:                                          //
+            // booleans are evaluated naturally.                                                 //
+            // strings - if they look like 'true' or 'false, they are evaluated, else they error //
+            // numbers - 0 is false, all else are true.                                          //
+            ///////////////////////////////////////////////////////////////////////////////////////
+            List<Serializable> actualArgs = getArgumentList(args, 3, variableInterpreter);
+            Serializable       condition  = actualArgs.get(0);
+            boolean            conditionBoolean;
+            if(condition == null)
+            {
+               conditionBoolean = false;
+            }
+            else if(condition instanceof Boolean b)
+            {
+               conditionBoolean = b;
+            }
+            else if(condition instanceof BigDecimal bd)
+            {
+               conditionBoolean = (bd.compareTo(BigDecimal.ZERO) != 0);
+            }
+            else if(condition instanceof String s)
+            {
+               if("true".equalsIgnoreCase(s))
+               {
+                  conditionBoolean = true;
+               }
+               else if("false".equalsIgnoreCase(s))
+               {
+                  conditionBoolean = false;
+               }
+               else
+               {
+                  throw (new QFormulaException("Could not evaluate string '" + s + "' as a boolean."));
+               }
+            }
+            else
+            {
+               conditionBoolean = false;
+            }
+
+            return conditionBoolean ? actualArgs.get(1) : actualArgs.get(2);
+         }
+         case "LT":
+         {
+            List<BigDecimal> numbers = getNumberArgumentList(args, 2, variableInterpreter);
+            return nullIfAnyNullArgsElseBoolean(numbers, () -> numbers.get(0).compareTo(numbers.get(1)) < 0);
+         }
+         case "LTE":
+         {
+            List<BigDecimal> numbers = getNumberArgumentList(args, 2, variableInterpreter);
+            return nullIfAnyNullArgsElseBoolean(numbers, () -> numbers.get(0).compareTo(numbers.get(1)) <= 0);
+         }
+         case "GT":
+         {
+            List<BigDecimal> numbers = getNumberArgumentList(args, 2, variableInterpreter);
+            return nullIfAnyNullArgsElseBoolean(numbers, () -> numbers.get(0).compareTo(numbers.get(1)) > 0);
+         }
+         case "GTE":
+         {
+            List<BigDecimal> numbers = getNumberArgumentList(args, 2, variableInterpreter);
+            return nullIfAnyNullArgsElseBoolean(numbers, () -> numbers.get(0).compareTo(numbers.get(1)) >= 0);
+         }
+         default:
+         {
+            ////////////////////////////////////////////////////////////////////////////////////////
+            // if there aren't arguments, then we can try to evaluate the thing not as a function //
+            ////////////////////////////////////////////////////////////////////////////////////////
+            if(CollectionUtils.nullSafeIsEmpty(args))
+            {
+               try
+               {
+                  return (ValueUtils.getValueAsBigDecimal(token));
+               }
+               catch(Exception e)
+               {
+                  // continue
+               }
+
+               try
+               {
+                  return (variableInterpreter.interpret(token));
+               }
+               catch(Exception e)
+               {
+                  // continue
+               }
+            }
+         }
+      }
+
+      throw (new QFormulaException("Unable to evaluate unrecognized expression: " + token + ""));
+   }
+
+
+
+   /*******************************************************************************
+    ** if any number in the list is null, get back null - else, return the result of the supplier.
+    *******************************************************************************/
+   private static Serializable nullIfAnyNullArgsElseBigDecimal(List<BigDecimal> numbers, Supplier<BigDecimal> supplier)
+   {
+      if(numbers.stream().anyMatch(Objects::isNull))
+      {
+         return (null);
+      }
+      return supplier.get();
+   }
+
+
+
+   /*******************************************************************************
+    ** if any number in the list is null, get back null - else, return the result of the supplier.
+    *******************************************************************************/
+   private static Serializable nullIfAnyNullArgsElseBoolean(List<BigDecimal> numbers, Supplier<Boolean> supplier)
+   {
+      if(numbers.stream().anyMatch(Objects::isNull))
+      {
+         return (null);
+      }
+      return supplier.get();
+   }
+
+
+
+   /*******************************************************************************
+    ** given a list of arguments, get back a specific number of arguments, all of which we
+    ** validate to be numbers (e.g., possibly interpreted variables) - else we throw.
+    ** also throw if not the right number is present.
+    *******************************************************************************/
+   private static List<BigDecimal> getNumberArgumentList(List<Serializable> originalArgs, Integer howMany, QMetaDataVariableInterpreter variableInterpreter) throws QFormulaException
+   {
+      if(howMany != null)
+      {
+         if(!howMany.equals(originalArgs.size()))
+         {
+            throw (new QFormulaException("Wrong number of arguments (required: " + howMany + ", received: " + originalArgs.size() + ")"));
+         }
+      }
+
+      List<BigDecimal> rs = new ArrayList<>();
+      for(Serializable originalArg : originalArgs)
+      {
+         try
+         {
+            Serializable interpretedArg = variableInterpreter.interpretForObject(ValueUtils.getValueAsString(originalArg), null);
+            rs.add(ValueUtils.getValueAsBigDecimal(interpretedArg));
+         }
+         catch(QValueException e)
+         {
+            throw (new QFormulaException("Could not process [" + originalArg + "] as a number"));
+         }
+      }
+      return (rs);
+   }
+
+
+
+   /*******************************************************************************
+    ** given a list of arguments, get back a specific number of arguments, all of which we
+    ** get interpreted.  throw if not the right number of args is present.
+    *******************************************************************************/
+   private static List<Serializable> getArgumentList(List<Serializable> originalArgs, Integer howMany, QMetaDataVariableInterpreter variableInterpreter) throws QFormulaException
+   {
+      if(howMany != null)
+      {
+         if(!howMany.equals(originalArgs.size()))
+         {
+            throw (new QFormulaException("Wrong number of arguments (required: " + howMany + ", received: " + originalArgs.size() + ")"));
+         }
+      }
+
+      List<Serializable> rs = new ArrayList<>();
+      for(Serializable originalArg : originalArgs)
+      {
+         Serializable interpretedArg = variableInterpreter.interpretForObject(ValueUtils.getValueAsString(originalArg), null);
+         rs.add(interpretedArg);
+      }
+      return (rs);
+   }
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/GenerateReportAction.java

@@ -0,0 +1,885 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.actions.reporting;
+
+
+import java.io.Serializable;
+import java.math.BigDecimal;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import java.util.function.Function;
+import java.util.function.Supplier;
+import java.util.stream.Collectors;
+import com.kingsrook.qqq.backend.core.actions.async.AsyncRecordPipeLoop;
+import com.kingsrook.qqq.backend.core.actions.customizers.QCodeLoader;
+import com.kingsrook.qqq.backend.core.actions.reporting.customizers.DataSourceQueryInputCustomizer;
+import com.kingsrook.qqq.backend.core.actions.reporting.customizers.ReportViewCustomizer;
+import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
+import com.kingsrook.qqq.backend.core.actions.values.QValueFormatter;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.exceptions.QFormulaException;
+import com.kingsrook.qqq.backend.core.exceptions.QReportingException;
+import com.kingsrook.qqq.backend.core.instances.QMetaDataVariableInterpreter;
+import com.kingsrook.qqq.backend.core.logging.QLogger;
+import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
+import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
+import com.kingsrook.qqq.backend.core.model.actions.reporting.ExportInput;
+import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportFormat;
+import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.JoinsContext;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
+import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportDataSource;
+import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportField;
+import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportView;
+import com.kingsrook.qqq.backend.core.model.metadata.reporting.ReportType;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.AbstractTransformStep;
+import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.BackendStepPostRunInput;
+import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.BackendStepPostRunOutput;
+import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
+import com.kingsrook.qqq.backend.core.utils.Pair;
+import com.kingsrook.qqq.backend.core.utils.StringUtils;
+import com.kingsrook.qqq.backend.core.utils.aggregates.AggregatesInterface;
+import com.kingsrook.qqq.backend.core.utils.aggregates.BigDecimalAggregates;
+import com.kingsrook.qqq.backend.core.utils.aggregates.IntegerAggregates;
+
+
+/*******************************************************************************
+ ** Action to generate a report.
+ **
+ ** A report can contain 1 or more Data Sources - e.g., tables + filters that define
+ ** data that goes into the report, or simple data-supplier lambdas.
+ **
+ ** A report can also contain 1 or more Views - e.g., sheets in a spreadsheet workbook.
+ ** (how do those work in non-XLSX formats??). Views can either be:
+ ** - plain tables,
+ ** - summaries (like pivot tables, but called summary to avoid confusion with "native" pivot tables),
+ ** - native pivot tables (not initially supported, due to lack of support in fastexcel...).
+ *******************************************************************************/
+public class GenerateReportAction
+{
+   private static final QLogger LOG = QLogger.getLogger(GenerateReportAction.class);
+
+   /////////////////////////////////////////////////////////////////////////////////////////////////////////////
+   // summaryAggregates and varianceAggregates are multi-level maps, ala:                                     //
+   // viewName > SummaryKey > fieldName > Aggregates                                                          //
+   // e.g.:                                                                                                   //
+   // viewName: salesSummaryReport                                                                            //
+   // SummaryKey: [(state:MO),(city:St.Louis)]                                                                //
+   // fieldName: salePrice                                                                                    //
+   // Aggregates: (count:47;sum:10,000;max:2,000;min:15)                                                      //
+   // salesSummaryReport > [(state:MO),(city:St.Louis)] > salePrice  > (count:47;sum:10,000;max:2,000;min:15) //
+   /////////////////////////////////////////////////////////////////////////////////////////////////////////////
+   Map<String, Map<SummaryKey, Map<String, AggregatesInterface<?>>>> summaryAggregates  = new HashMap<>();
+   Map<String, Map<SummaryKey, Map<String, AggregatesInterface<?>>>> varianceAggregates = new HashMap<>();
+
+   Map<String, AggregatesInterface<?>> totalAggregates         = new HashMap<>();
+   Map<String, AggregatesInterface<?>> varianceTotalAggregates = new HashMap<>();
+
+   private QReportMetaData         report;
+   private ReportFormat            reportFormat;
+   private ExportStreamerInterface reportStreamer;
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public void execute(ReportInput reportInput) throws QException
+   {
+      report = reportInput.getInstance().getReport(reportInput.getReportName());
+      reportFormat = reportInput.getReportFormat();
+      if(reportFormat == null)
+      {
+         throw new QException("Report format was not specified.");
+      }
+      reportStreamer = reportFormat.newReportStreamer();
+
+      ////////////////////////////////////////////////////////////////////////////////////////////////
+      // foreach data source, do a query (possibly more than 1, if it goes to multiple table views) //
+      ////////////////////////////////////////////////////////////////////////////////////////////////
+      for(QReportDataSource dataSource : report.getDataSources())
+      {
+         //////////////////////////////////////////////////////////////////////////////
+         // make a list of the views that use this data source for various purposes. //
+         //////////////////////////////////////////////////////////////////////////////
+         List<QReportView> dataSourceTableViews = report.getViews().stream()
+            .filter(v -> v.getType().equals(ReportType.TABLE))
+            .filter(v -> v.getDataSourceName().equals(dataSource.getName()))
+            .toList();
+
+         List<QReportView> dataSourceSummaryViews = report.getViews().stream()
+            .filter(v -> v.getType().equals(ReportType.SUMMARY))
+            .filter(v -> v.getDataSourceName().equals(dataSource.getName()))
+            .toList();
+
+         List<QReportView> dataSourceVariantViews = report.getViews().stream()
+            .filter(v -> v.getType().equals(ReportType.SUMMARY))
+            .filter(v -> v.getVarianceDataSourceName() != null && v.getVarianceDataSourceName().equals(dataSource.getName()))
+            .toList();
+
+         /////////////////////////////////////////////////////////////////////////////////////////////
+         // if this data source isn't used for any table views, but it is used for one or           //
+         // more summary views (possibly as a variant), then run the query, gathering summary data. //
+         /////////////////////////////////////////////////////////////////////////////////////////////
+         if(dataSourceTableViews.isEmpty())
+         {
+            if(!dataSourceSummaryViews.isEmpty() || !dataSourceVariantViews.isEmpty())
+            {
+               gatherData(reportInput, dataSource, null, dataSourceSummaryViews, dataSourceVariantViews);
+            }
+         }
+         else
+         {
+            ////////////////////////////////////////////////////////////////////////////////////////
+            // else, foreach table view this data source is used for, run the data source's query //
+            ////////////////////////////////////////////////////////////////////////////////////////
+            for(QReportView dataSourceTableView : dataSourceTableViews)
+            {
+               /////////////////////////////////////////////////////////////////////////////////////////
+               // if there's a view customizer, run it (e.g., to customize the columns in the report) //
+               /////////////////////////////////////////////////////////////////////////////////////////
+               if(dataSourceTableView.getViewCustomizer() != null)
+               {
+                  Function<QReportView, QReportView> viewCustomizerFunction = QCodeLoader.getFunction(dataSourceTableView.getViewCustomizer());
+                  if(viewCustomizerFunction instanceof ReportViewCustomizer reportViewCustomizer)
+                  {
+                     reportViewCustomizer.setReportInput(reportInput);
+                  }
+                  dataSourceTableView = viewCustomizerFunction.apply(dataSourceTableView.clone());
+               }
+
+               ////////////////////////////////////////////////////////////////////////////////////
+               // start the table-view (e.g., open this tab in xlsx) and then run the query-loop //
+               ////////////////////////////////////////////////////////////////////////////////////
+               startTableView(reportInput, dataSource, dataSourceTableView);
+               gatherData(reportInput, dataSource, dataSourceTableView, dataSourceSummaryViews, dataSourceVariantViews);
+            }
+         }
+      }
+
+      outputSummaries(reportInput);
+
+      reportStreamer.finish();
+
+      try
+      {
+         reportInput.getReportOutputStream().close();
+      }
+      catch(Exception e)
+      {
+         throw (new QReportingException("Error completing report", e));
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private void startTableView(ReportInput reportInput, QReportDataSource dataSource, QReportView reportView) throws QException
+   {
+      QTableMetaData table = reportInput.getInstance().getTable(dataSource.getSourceTable());
+
+      QMetaDataVariableInterpreter variableInterpreter = new QMetaDataVariableInterpreter();
+      variableInterpreter.addValueMap("input", reportInput.getInputValues());
+
+      ExportInput exportInput = new ExportInput();
+      exportInput.setReportFormat(reportFormat);
+      exportInput.setFilename(reportInput.getFilename());
+      exportInput.setTitleRow(getTitle(reportView, variableInterpreter));
+      exportInput.setIncludeHeaderRow(reportView.getIncludeHeaderRow());
+      exportInput.setReportOutputStream(reportInput.getReportOutputStream());
+
+      JoinsContext joinsContext = null;
+      if(StringUtils.hasContent(dataSource.getSourceTable()))
+      {
+         joinsContext = new JoinsContext(exportInput.getInstance(), dataSource.getSourceTable(), dataSource.getQueryJoins(), dataSource.getQueryFilter());
+      }
+
+      List<QFieldMetaData> fields = new ArrayList<>();
+      for(QReportField column : reportView.getColumns())
+      {
+         if(column.getIsVirtual())
+         {
+            fields.add(column.toField());
+         }
+         else
+         {
+            String                                effectiveFieldName       = Objects.requireNonNullElse(column.getSourceFieldName(), column.getName());
+            JoinsContext.FieldAndTableNameOrAlias fieldAndTableNameOrAlias = joinsContext == null ? null : joinsContext.getFieldAndTableNameOrAlias(effectiveFieldName);
+            if(fieldAndTableNameOrAlias == null || fieldAndTableNameOrAlias.field() == null)
+            {
+               throw new QReportingException("Could not find field named [" + effectiveFieldName + "] in dataSource [" + dataSource.getName() + "]");
+            }
+
+            QFieldMetaData field = fieldAndTableNameOrAlias.field().clone();
+            field.setName(column.getName());
+            if(StringUtils.hasContent(column.getLabel()))
+            {
+               field.setLabel(column.getLabel());
+            }
+            fields.add(field);
+         }
+      }
+
+      reportStreamer.setDisplayFormats(getDisplayFormatMap(fields));
+      reportStreamer.start(exportInput, fields, reportView.getLabel());
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private void gatherData(ReportInput reportInput, QReportDataSource dataSource, QReportView tableView, List<QReportView> summaryViews, List<QReportView> variantViews) throws QException
+   {
+      ////////////////////////////////////////////////////////////////////////////////////////
+      // check if this view has a transform step - if so, set it up now and run its pre-run //
+      ////////////////////////////////////////////////////////////////////////////////////////
+      AbstractTransformStep transformStep       = null;
+      RunBackendStepInput   transformStepInput  = null;
+      RunBackendStepOutput  transformStepOutput = null;
+      if(tableView != null && tableView.getRecordTransformStep() != null)
+      {
+         transformStep = QCodeLoader.getBackendStep(AbstractTransformStep.class, tableView.getRecordTransformStep());
+
+         transformStepInput = new RunBackendStepInput();
+         transformStepInput.setValues(reportInput.getInputValues());
+
+         transformStepOutput = new RunBackendStepOutput();
+
+         transformStep.preRun(transformStepInput, transformStepOutput);
+      }
+
+      ////////////////////////////////////////////////////////////////////
+      // create effectively-final versions of these vars for the lambda //
+      ////////////////////////////////////////////////////////////////////
+      AbstractTransformStep finalTransformStep       = transformStep;
+      RunBackendStepInput   finalTransformStepInput  = transformStepInput;
+      RunBackendStepOutput  finalTransformStepOutput = transformStepOutput;
+
+      /////////////////////////////////////////////////////////////////
+      // run a record pipe loop, over the query for this data source //
+      /////////////////////////////////////////////////////////////////
+      RecordPipe recordPipe = new BufferedRecordPipe(1000);
+      new AsyncRecordPipeLoop().run("Report[" + reportInput.getReportName() + "]", null, recordPipe, (callback) ->
+      {
+         if(dataSource.getSourceTable() != null)
+         {
+            QQueryFilter queryFilter = dataSource.getQueryFilter() == null ? new QQueryFilter() : dataSource.getQueryFilter().clone();
+            setInputValuesInQueryFilter(reportInput, queryFilter);
+
+            QueryInput queryInput = new QueryInput();
+            queryInput.setRecordPipe(recordPipe);
+            queryInput.setTableName(dataSource.getSourceTable());
+            queryInput.setFilter(queryFilter);
+            queryInput.setQueryJoins(dataSource.getQueryJoins());
+
+            queryInput.setShouldTranslatePossibleValues(true);
+            queryInput.setFieldsToTranslatePossibleValues(setupFieldsToTranslatePossibleValues(reportInput, dataSource, new JoinsContext(reportInput.getInstance(), dataSource.getSourceTable(), dataSource.getQueryJoins(), queryInput.getFilter())));
+
+            if(dataSource.getQueryInputCustomizer() != null)
+            {
+               DataSourceQueryInputCustomizer queryInputCustomizer = QCodeLoader.getAdHoc(DataSourceQueryInputCustomizer.class, dataSource.getQueryInputCustomizer());
+               queryInput = queryInputCustomizer.run(reportInput, queryInput);
+            }
+
+            return (new QueryAction().execute(queryInput));
+         }
+         else if(dataSource.getStaticDataSupplier() != null)
+         {
+            @SuppressWarnings("unchecked")
+            Supplier<List<List<Serializable>>> supplier = QCodeLoader.getAdHoc(Supplier.class, dataSource.getStaticDataSupplier());
+            List<List<Serializable>> lists = supplier.get();
+            for(List<Serializable> list : lists)
+            {
+               QRecord record = new QRecord();
+               int     index  = 0;
+               for(Serializable value : list)
+               {
+                  record.setValue("column" + (index++), value);
+               }
+               recordPipe.addRecord(record);
+            }
+            return (true);
+         }
+         else
+         {
+            throw (new IllegalStateException("Misconfigured data source [" + dataSource.getName() + "]."));
+         }
+      }, () ->
+      {
+         List<QRecord> records = recordPipe.consumeAvailableRecords();
+         if(finalTransformStep != null)
+         {
+            finalTransformStepInput.setRecords(records);
+            finalTransformStep.run(finalTransformStepInput, finalTransformStepOutput);
+            records = finalTransformStepOutput.getRecords();
+         }
+
+         return (consumeRecords(reportInput, dataSource, records, tableView, summaryViews, variantViews));
+      });
+
+      ////////////////////////////////////////////////
+      // if there's a transformer, run its post-run //
+      ////////////////////////////////////////////////
+      if(transformStep != null)
+      {
+         transformStep.postRun(new BackendStepPostRunInput(transformStepInput), new BackendStepPostRunOutput(transformStepOutput));
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private Set<String> setupFieldsToTranslatePossibleValues(ReportInput reportInput, QReportDataSource dataSource, JoinsContext joinsContext)
+   {
+      Set<String> fieldsToTranslatePossibleValues = new HashSet<>();
+
+      for(QReportView view : report.getViews())
+      {
+         for(QReportField column : CollectionUtils.nonNullList(view.getColumns()))
+         {
+            ////////////////////////////////////////////////////////////////////////////////////////
+            // if this is a column marked as ShowPossibleValueLabel, then we need to translate it //
+            ////////////////////////////////////////////////////////////////////////////////////////
+            if(column.getShowPossibleValueLabel())
+            {
+               String effectiveFieldName = Objects.requireNonNullElse(column.getSourceFieldName(), column.getName());
+               fieldsToTranslatePossibleValues.add(effectiveFieldName);
+            }
+         }
+
+         for(String summaryField : CollectionUtils.nonNullList(view.getPivotFields()))
+         {
+            ///////////////////////////////////////////////////////////////////////////////
+            // all pivotFields that are possible value sources are implicitly translated //
+            ///////////////////////////////////////////////////////////////////////////////
+            QTableMetaData table = reportInput.getInstance().getTable(dataSource.getSourceTable());
+            if(table.getField(summaryField).getPossibleValueSourceName() != null)
+            {
+               fieldsToTranslatePossibleValues.add(summaryField);
+            }
+         }
+      }
+
+      return (fieldsToTranslatePossibleValues);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private void setInputValuesInQueryFilter(ReportInput reportInput, QQueryFilter queryFilter)
+   {
+      if(queryFilter == null || queryFilter.getCriteria() == null)
+      {
+         return;
+      }
+
+      queryFilter.interpretValues(reportInput.getInputValues());
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private Integer consumeRecords(ReportInput reportInput, QReportDataSource dataSource, List<QRecord> records, QReportView tableView, List<QReportView> summaryViews, List<QReportView> variantViews) throws QException
+   {
+      QTableMetaData table = reportInput.getInstance().getTable(dataSource.getSourceTable());
+
+      ////////////////////////////////////////////////////////////////////////////
+      // if this record goes on a table view, add it to the report streamer now //
+      ////////////////////////////////////////////////////////////////////////////
+      if(tableView != null)
+      {
+         ////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+         // if any fields are 'showPossibleValueLabel', then move display values for them into the record's values map //
+         ////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+         for(QReportField column : tableView.getColumns())
+         {
+            if(column.getShowPossibleValueLabel())
+            {
+               String effectiveFieldName = Objects.requireNonNullElse(column.getSourceFieldName(), column.getName());
+               for(QRecord record : records)
+               {
+                  String displayValue = record.getDisplayValue(effectiveFieldName);
+                  record.setValue(column.getName(), displayValue);
+               }
+            }
+         }
+
+         reportStreamer.addRecords(records);
+      }
+
+      /////////////////////////////////
+      // do aggregates for summaries //
+      /////////////////////////////////
+      if(summaryViews != null)
+      {
+         for(QReportView summaryView : summaryViews)
+         {
+            addRecordsToSummaryAggregates(summaryView, table, records, summaryAggregates);
+         }
+      }
+
+      if(variantViews != null)
+      {
+         for(QReportView variantView : variantViews)
+         {
+            addRecordsToSummaryAggregates(variantView, table, records, varianceAggregates);
+         }
+      }
+
+      ///////////////////////////////////////////
+      // do totals too, if any views want them //
+      ///////////////////////////////////////////
+      if(summaryViews != null && summaryViews.stream().anyMatch(QReportView::getIncludeTotalRow))
+      {
+         for(QRecord record : records)
+         {
+            addRecordToAggregatesMap(table, record, totalAggregates);
+         }
+      }
+
+      if(variantViews != null && variantViews.stream().anyMatch(QReportView::getIncludeTotalRow))
+      {
+         for(QRecord record : records)
+         {
+            addRecordToAggregatesMap(table, record, varianceTotalAggregates);
+         }
+      }
+
+      return (records.size());
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private void addRecordsToSummaryAggregates(QReportView view, QTableMetaData table, List<QRecord> records, Map<String, Map<SummaryKey, Map<String, AggregatesInterface<?>>>> aggregatesMap)
+   {
+      Map<SummaryKey, Map<String, AggregatesInterface<?>>> viewAggregates = aggregatesMap.computeIfAbsent(view.getName(), (name) -> new HashMap<>());
+
+      for(QRecord record : records)
+      {
+         SummaryKey key = new SummaryKey();
+         for(String summaryField : view.getPivotFields())
+         {
+            Serializable summaryValue = record.getValue(summaryField);
+            if(table.getField(summaryField).getPossibleValueSourceName() != null)
+            {
+               //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+               // so, this is kinda a thing - where we implicitly use possible-value labels (e.g., display values) for pivot fields... //
+               //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+               summaryValue = record.getDisplayValue(summaryField);
+            }
+            key.add(summaryField, summaryValue);
+
+            if(view.getIncludePivotSubTotals() && key.getKeys().size() < view.getPivotFields().size())
+            {
+               /////////////////////////////////////////////////////////////////////////////////////////
+               // be careful here, with these key objects, and their identity, being used as map keys //
+               /////////////////////////////////////////////////////////////////////////////////////////
+               SummaryKey subKey = key.clone();
+               addRecordToSummaryKeyAggregates(table, record, viewAggregates, subKey);
+            }
+         }
+
+         addRecordToSummaryKeyAggregates(table, record, viewAggregates, key);
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private void addRecordToSummaryKeyAggregates(QTableMetaData table, QRecord record, Map<SummaryKey, Map<String, AggregatesInterface<?>>> viewAggregates, SummaryKey key)
+   {
+      Map<String, AggregatesInterface<?>> keyAggregates = viewAggregates.computeIfAbsent(key, (name) -> new HashMap<>());
+      addRecordToAggregatesMap(table, record, keyAggregates);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private void addRecordToAggregatesMap(QTableMetaData table, QRecord record, Map<String, AggregatesInterface<?>> aggregatesMap)
+   {
+      for(QFieldMetaData field : table.getFields().values())
+      {
+         if(field.getType().equals(QFieldType.INTEGER))
+         {
+            @SuppressWarnings("unchecked")
+            AggregatesInterface<Integer> fieldAggregates = (AggregatesInterface<Integer>) aggregatesMap.computeIfAbsent(field.getName(), (name) -> new IntegerAggregates());
+            fieldAggregates.add(record.getValueInteger(field.getName()));
+         }
+         else if(field.getType().equals(QFieldType.DECIMAL))
+         {
+            @SuppressWarnings("unchecked")
+            AggregatesInterface<BigDecimal> fieldAggregates = (AggregatesInterface<BigDecimal>) aggregatesMap.computeIfAbsent(field.getName(), (name) -> new BigDecimalAggregates());
+            fieldAggregates.add(record.getValueBigDecimal(field.getName()));
+         }
+         // todo - more types (dates, at least?)
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private void outputSummaries(ReportInput reportInput) throws QReportingException, QFormulaException
+   {
+      List<QReportView> reportViews = report.getViews().stream().filter(v -> v.getType().equals(ReportType.SUMMARY)).toList();
+      for(QReportView view : reportViews)
+      {
+         QReportDataSource dataSource    = report.getDataSource(view.getDataSourceName());
+         QTableMetaData    table         = reportInput.getInstance().getTable(dataSource.getSourceTable());
+         SummaryOutput     summaryOutput = computeSummaryRowsForView(reportInput, view, table);
+
+         ExportInput exportInput = new ExportInput();
+         exportInput.setReportFormat(reportFormat);
+         exportInput.setFilename(reportInput.getFilename());
+         exportInput.setTitleRow(summaryOutput.titleRow);
+         exportInput.setIncludeHeaderRow(view.getIncludeHeaderRow());
+         exportInput.setReportOutputStream(reportInput.getReportOutputStream());
+
+         reportStreamer.setDisplayFormats(getDisplayFormatMap(view));
+         reportStreamer.start(exportInput, getFields(table, view), view.getLabel());
+
+         reportStreamer.addRecords(summaryOutput.summaryRows); // todo - what if this set is huge?
+
+         if(summaryOutput.totalRow != null)
+         {
+            reportStreamer.addTotalsRow(summaryOutput.totalRow);
+         }
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private Map<String, String> getDisplayFormatMap(QReportView view)
+   {
+      return (view.getColumns().stream()
+         .filter(c -> c.getDisplayFormat() != null)
+         .collect(Collectors.toMap(QReportField::getName, QReportField::getDisplayFormat)));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private Map<String, String> getDisplayFormatMap(List<QFieldMetaData> fields)
+   {
+      return (fields.stream()
+         .filter(f -> f.getDisplayFormat() != null)
+         .collect(Collectors.toMap(QFieldMetaData::getName, QFieldMetaData::getDisplayFormat)));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private List<QFieldMetaData> getFields(QTableMetaData table, QReportView view)
+   {
+      List<QFieldMetaData> fields = new ArrayList<>();
+      for(String pivotField : view.getPivotFields())
+      {
+         QFieldMetaData field = table.getField(pivotField);
+         fields.add(new QFieldMetaData(pivotField, field.getType()).withLabel(field.getLabel())); // todo do we need the type?  if so need table as input here
+      }
+      for(QReportField column : view.getColumns())
+      {
+         fields.add(new QFieldMetaData().withName(column.getName()).withLabel(column.getLabel())); // todo do we need the type?  if so need table as input here
+      }
+      return (fields);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private SummaryOutput computeSummaryRowsForView(ReportInput reportInput, QReportView view, QTableMetaData table) throws QReportingException, QFormulaException
+   {
+      QValueFormatter              valueFormatter      = new QValueFormatter();
+      QMetaDataVariableInterpreter variableInterpreter = new QMetaDataVariableInterpreter();
+      variableInterpreter.addValueMap("input", reportInput.getInputValues());
+      variableInterpreter.addValueMap("total", getSummaryValuesForInterpreter(totalAggregates));
+
+      ///////////
+      // title //
+      ///////////
+      String title = getTitle(view, variableInterpreter);
+
+      /////////////////////////
+      // create summary rows //
+      /////////////////////////
+      List<QRecord> summaryRows = new ArrayList<>();
+      for(Map.Entry<SummaryKey, Map<String, AggregatesInterface<?>>> entry : summaryAggregates.getOrDefault(view.getName(), Collections.emptyMap()).entrySet())
+      {
+         SummaryKey                          summaryKey      = entry.getKey();
+         Map<String, AggregatesInterface<?>> fieldAggregates = entry.getValue();
+         Map<String, Serializable>           summaryValues   = getSummaryValuesForInterpreter(fieldAggregates);
+         variableInterpreter.addValueMap("pivot", summaryValues);
+         variableInterpreter.addValueMap("summary", summaryValues);
+
+         HashMap<String, Serializable> thisRowValues = new HashMap<>();
+         variableInterpreter.addValueMap("thisRow", thisRowValues);
+
+         if(!varianceAggregates.isEmpty())
+         {
+            Map<SummaryKey, Map<String, AggregatesInterface<?>>> varianceMap    = varianceAggregates.getOrDefault(view.getName(), Collections.emptyMap());
+            Map<String, AggregatesInterface<?>>                  varianceSubMap = varianceMap.getOrDefault(summaryKey, Collections.emptyMap());
+            Map<String, Serializable>                            varianceValues = getSummaryValuesForInterpreter(varianceSubMap);
+            variableInterpreter.addValueMap("variancePivot", varianceValues);
+            variableInterpreter.addValueMap("variance", varianceValues);
+         }
+
+         QRecord summaryRow = new QRecord();
+         summaryRows.add(summaryRow);
+
+         ////////////////////////////
+         // add the summary values //
+         ////////////////////////////
+         for(Pair<String, Serializable> key : summaryKey.getKeys())
+         {
+            summaryRow.setValue(key.getA(), key.getB());
+         }
+
+         ///////////////////////////////////////////////////////////////////////////////
+         // for summary subtotals, add the text "Total" to the last field in this key //
+         ///////////////////////////////////////////////////////////////////////////////
+         if(summaryKey.getKeys().size() < view.getPivotFields().size())
+         {
+            String fieldName = summaryKey.getKeys().get(summaryKey.getKeys().size() - 1).getA();
+            summaryRow.setValue(fieldName, summaryRow.getValueString(fieldName) + " Total");
+         }
+
+         ///////////////////////////
+         // add the column values //
+         ///////////////////////////
+         for(QReportField column : view.getColumns())
+         {
+            Serializable serializable = getValueForColumn(variableInterpreter, column);
+            summaryRow.setValue(column.getName(), serializable);
+            thisRowValues.put(column.getName(), serializable);
+         }
+      }
+
+      //////////////////////////////////////////////////////////////////////////////////////
+      // sort the summary rows                                                            //
+      // Note - this will NOT work correctly if there's more than 1 pivot field, as we're //
+      // not doing anything to keep related rows them together (e.g., all MO state rows)  //
+      //////////////////////////////////////////////////////////////////////////////////////
+      if(CollectionUtils.nullSafeHasContents(view.getOrderByFields()))
+      {
+         summaryRows.sort((o1, o2) ->
+         {
+            return summaryRowComparator(view, o1, o2);
+         });
+      }
+
+      ////////////////
+      // totals row //
+      ////////////////
+      QRecord totalRow = null;
+      if(view.getIncludeTotalRow())
+      {
+         totalRow = new QRecord();
+
+         for(String pivotField : view.getPivotFields())
+         {
+            if(totalRow.getValues().isEmpty())
+            {
+               totalRow.setValue(pivotField, "Totals");
+            }
+         }
+
+         Map<String, Serializable> totalValues = getSummaryValuesForInterpreter(totalAggregates);
+         variableInterpreter.addValueMap("pivot", totalValues);
+         variableInterpreter.addValueMap("summary", totalValues);
+
+         Map<String, Serializable> varianceTotalValues = getSummaryValuesForInterpreter(varianceTotalAggregates);
+         variableInterpreter.addValueMap("variancePivot", varianceTotalValues);
+         variableInterpreter.addValueMap("variance", varianceTotalValues);
+
+         HashMap<String, Serializable> thisRowValues = new HashMap<>();
+         variableInterpreter.addValueMap("thisRow", thisRowValues);
+
+         for(QReportField column : view.getColumns())
+         {
+            Serializable serializable = getValueForColumn(variableInterpreter, column);
+            totalRow.setValue(column.getName(), serializable);
+            thisRowValues.put(column.getName(), serializable);
+
+            String formatted = valueFormatter.formatValue(column.getDisplayFormat(), serializable);
+         }
+      }
+
+      return (new SummaryOutput(summaryRows, title, totalRow));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private String getTitle(QReportView view, QMetaDataVariableInterpreter variableInterpreter)
+   {
+      String title = null;
+      if(view.getTitleFields() != null && StringUtils.hasContent(view.getTitleFormat()))
+      {
+         List<String> titleValues = new ArrayList<>();
+         for(String titleField : view.getTitleFields())
+         {
+            titleValues.add(variableInterpreter.interpret(titleField));
+         }
+
+         title = new QValueFormatter().formatStringWithValues(view.getTitleFormat(), titleValues);
+      }
+      else if(StringUtils.hasContent(view.getTitleFormat()))
+      {
+         title = view.getTitleFormat();
+      }
+
+      return title;
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private Serializable getValueForColumn(QMetaDataVariableInterpreter variableInterpreter, QReportField column) throws QFormulaException
+   {
+      String       formula = column.getFormula();
+      Serializable result;
+      if(formula.startsWith("=") && formula.length() > 1)
+      {
+         result = FormulaInterpreter.interpretFormula(variableInterpreter, formula.substring(1));
+      }
+      else
+      {
+         result = variableInterpreter.interpretForObject(formula, null);
+      }
+      return (result);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @SuppressWarnings({ "rawtypes", "unchecked" })
+   private int summaryRowComparator(QReportView view, QRecord o1, QRecord o2)
+   {
+      if(o1 == o2)
+      {
+         return (0);
+      }
+
+      for(QFilterOrderBy orderByField : view.getOrderByFields())
+      {
+         Comparable c1 = (Comparable) o1.getValue(orderByField.getFieldName());
+         Comparable c2 = (Comparable) o2.getValue(orderByField.getFieldName());
+
+         if(c1 == null && c2 == null)
+         {
+            continue;
+         }
+         if(c1 == null)
+         {
+            return (orderByField.getIsAscending() ? -1 : 1);
+         }
+         if(c2 == null)
+         {
+            return (orderByField.getIsAscending() ? 1 : -1);
+         }
+
+         int comp = orderByField.getIsAscending() ? c1.compareTo(c2) : c2.compareTo(c1);
+         if(comp != 0)
+         {
+            return (comp);
+         }
+      }
+
+      return (0);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private Map<String, Serializable> getSummaryValuesForInterpreter(Map<String, AggregatesInterface<?>> fieldAggregates)
+   {
+      Map<String, Serializable> summaryValuesForInterpreter = new HashMap<>();
+      for(Map.Entry<String, AggregatesInterface<?>> subEntry : fieldAggregates.entrySet())
+      {
+         String                 fieldName  = subEntry.getKey();
+         AggregatesInterface<?> aggregates = subEntry.getValue();
+         summaryValuesForInterpreter.put("sum." + fieldName, aggregates.getSum());
+         summaryValuesForInterpreter.put("count." + fieldName, aggregates.getCount());
+         summaryValuesForInterpreter.put("min." + fieldName, aggregates.getMin());
+         summaryValuesForInterpreter.put("max." + fieldName, aggregates.getMax());
+         summaryValuesForInterpreter.put("average." + fieldName, aggregates.getAverage());
+      }
+      return summaryValuesForInterpreter;
+   }
+
+
+
+   /*******************************************************************************
+    ** record to serve as tuple/multi-value output of computeSummaryRowsForView method.
+    *******************************************************************************/
+   private record SummaryOutput(List<QRecord> summaryRows, String titleRow, QRecord totalRow)
+   {
+   }
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/JsonExportStreamer.java

@@ -0,0 +1,177 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.actions.reporting;
+
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.io.Serializable;
+import java.nio.charset.StandardCharsets;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import com.kingsrook.qqq.backend.core.exceptions.QReportingException;
+import com.kingsrook.qqq.backend.core.logging.QLogger;
+import com.kingsrook.qqq.backend.core.model.actions.reporting.ExportInput;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+import com.kingsrook.qqq.backend.core.utils.JsonUtils;
+import com.kingsrook.qqq.backend.core.utils.StringUtils;
+
+
+/*******************************************************************************
+ ** JSON export format implementation
+ *******************************************************************************/
+public class JsonExportStreamer implements ExportStreamerInterface
+{
+   private static final QLogger LOG = QLogger.getLogger(JsonExportStreamer.class);
+
+   private ExportInput          exportInput;
+   private QTableMetaData       table;
+   private List<QFieldMetaData> fields;
+   private OutputStream         outputStream;
+
+   private boolean needComma   = false;
+   private boolean prettyPrint = true;
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public JsonExportStreamer()
+   {
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public void start(ExportInput exportInput, List<QFieldMetaData> fields, String label) throws QReportingException
+   {
+      this.exportInput = exportInput;
+      this.fields = fields;
+      table = exportInput.getTable();
+      outputStream = this.exportInput.getReportOutputStream();
+
+      try
+      {
+         outputStream.write('[');
+      }
+      catch(IOException e)
+      {
+         throw (new QReportingException("Error starting report output", e));
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public void addRecords(List<QRecord> qRecords) throws QReportingException
+   {
+      LOG.info("Consuming [" + qRecords.size() + "] records from the pipe");
+
+      for(QRecord qRecord : qRecords)
+      {
+         writeRecord(qRecord);
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private void writeRecord(QRecord qRecord) throws QReportingException
+   {
+      try
+      {
+         if(needComma)
+         {
+            outputStream.write(',');
+         }
+
+         Map<String, Serializable> mapForJson = new LinkedHashMap<>();
+         for(QFieldMetaData field : fields)
+         {
+            String labelForJson = StringUtils.lcFirst(field.getLabel().replace(" ", ""));
+            mapForJson.put(labelForJson, qRecord.getValue(field.getName()));
+         }
+
+         String json = prettyPrint ? JsonUtils.toPrettyJson(mapForJson) : JsonUtils.toJson(mapForJson);
+
+         if(prettyPrint)
+         {
+            outputStream.write('\n');
+         }
+
+         outputStream.write(json.getBytes(StandardCharsets.UTF_8));
+
+         outputStream.flush(); // todo - less often?
+         needComma = true;
+      }
+      catch(Exception e)
+      {
+         throw (new QReportingException("Error writing JSON report", e));
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public void addTotalsRow(QRecord record) throws QReportingException
+   {
+      writeRecord(record);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public void finish() throws QReportingException
+   {
+      try
+      {
+         if(prettyPrint)
+         {
+            outputStream.write('\n');
+         }
+         outputStream.write(']');
+      }
+      catch(IOException e)
+      {
+         throw (new QReportingException("Error ending report output", e));
+      }
+   }
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/ListOfMapsExportStreamer.java

@@ -0,0 +1,161 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.actions.reporting;
+
+
+import java.util.ArrayList;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import com.kingsrook.qqq.backend.core.exceptions.QReportingException;
+import com.kingsrook.qqq.backend.core.logging.QLogger;
+import com.kingsrook.qqq.backend.core.model.actions.reporting.ExportInput;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
+
+
+/*******************************************************************************
+ ** Report streamer implementation that just builds up a STATIC list of lists of strings.
+ ** Meant only for use in unit tests at this time...  would need refactored for
+ ** multi-thread/multi-use if wanted for real usage.
+ *******************************************************************************/
+public class ListOfMapsExportStreamer implements ExportStreamerInterface
+{
+   private static final QLogger LOG = QLogger.getLogger(ListOfMapsExportStreamer.class);
+
+   private ExportInput          exportInput;
+   private List<QFieldMetaData> fields;
+
+   private static Map<String, List<Map<String, String>>> rows    = new LinkedHashMap<>();
+   private static Map<String, List<String>>              headers = new LinkedHashMap<>();
+   private static String                                 currentSheetLabel;
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public ListOfMapsExportStreamer()
+   {
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public static void reset()
+   {
+      rows.clear();
+      headers.clear();
+      currentSheetLabel = null;
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for list
+    **
+    *******************************************************************************/
+   public static List<Map<String, String>> getList(String name)
+   {
+      return (rows.get(name));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public void start(ExportInput exportInput, List<QFieldMetaData> fields, String label) throws QReportingException
+   {
+      this.exportInput = exportInput;
+      this.fields = fields;
+
+      currentSheetLabel = label;
+
+      rows.put(label, new ArrayList<>());
+
+      if(exportInput.getIncludeHeaderRow())
+      {
+         headers.put(label, new ArrayList<>());
+         for(QFieldMetaData field : fields)
+         {
+            headers.get(label).add(field.getLabel());
+         }
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public void addRecords(List<QRecord> qRecords) throws QReportingException
+   {
+      LOG.info("Consuming [" + qRecords.size() + "] records from the pipe");
+      for(QRecord qRecord : qRecords)
+      {
+         addRecord(qRecord);
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private void addRecord(QRecord qRecord)
+   {
+      Map<String, String> row = new LinkedHashMap<>();
+      rows.get(currentSheetLabel).add(row);
+      for(int i = 0; i < fields.size(); i++)
+      {
+         row.put(headers.get(currentSheetLabel).get(i), qRecord.getValueString(fields.get(i).getName()));
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public void addTotalsRow(QRecord record)
+   {
+      addRecord(record);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public void finish()
+   {
+
+   }
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/RecordPipe.java

@@ -26,10 +26,10 @@ import java.util.ArrayList;
 import java.util.List;
 import java.util.concurrent.ArrayBlockingQueue;
 import java.util.concurrent.TimeUnit;
+import java.util.function.Consumer;
+import com.kingsrook.qqq.backend.core.logging.QLogger;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
 import com.kingsrook.qqq.backend.core.utils.SleepUtils;
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
 
 
 /*******************************************************************************
@@ -38,35 +38,131 @@ import org.apache.logging.log4j.Logger;
  *******************************************************************************/
 public class RecordPipe
 {
-   private static final Logger LOG = LogManager.getLogger(RecordPipe.class);
+   private static final QLogger LOG = QLogger.getLogger(RecordPipe.class);
+
+   private static final long BLOCKING_SLEEP_MILLIS = 100;
+   private static final long MAX_SLEEP_LOOP_MILLIS = 300_000; // 5 minutes
 
    private ArrayBlockingQueue<QRecord> queue = new ArrayBlockingQueue<>(1_000);
 
+   private boolean isTerminated = false;
+
+   private Consumer<List<QRecord>> postRecordActions = null;
+
+   /////////////////////////////////////
+   // See usage below for explanation //
+   /////////////////////////////////////
+   private List<QRecord> singleRecordListForPostRecordActions = new ArrayList<>();
+
+
 
    /*******************************************************************************
-    ** Add a record to the pipe
-    ** Returns true iff the record fit in the pipe; false if the pipe is currently full.
+    ** Default constructor.
+    *******************************************************************************/
+   public RecordPipe()
+   {
+
+   }
+
+
+
+   /*******************************************************************************
+    ** Construct a record pipe, with an alternative capacity for the internal queue.
+    *******************************************************************************/
+   public RecordPipe(Integer overrideCapacity)
+   {
+      queue = new ArrayBlockingQueue<>(overrideCapacity);
+   }
+
+
+
+   /*******************************************************************************
+    ** Turn off the pipe.  Stop accepting new records (just ignore them in the add
+    ** method).  Clear the existing queue.  Don't return any more records.  Note that
+    ** if consumeAvailableRecords was running in another thread, it may still return
+    ** some records that it read before this call.
+    *******************************************************************************/
+   public void terminate()
+   {
+      isTerminated = true;
+      queue.clear();
+   }
+
+
+
+   /*******************************************************************************
+    ** Add a record to the pipe.  Will block if the pipe is full.  Will noop if pipe is terminated.
     *******************************************************************************/
    public void addRecord(QRecord record)
+   {
+      if(isTerminated)
+      {
+         return;
+      }
+
+      if(postRecordActions != null)
+      {
+         ////////////////////////////////////////////////////////////////////////////////////
+         // the initial use-case of this method is to call QueryAction.postRecordActions   //
+         // that method requires that the list param be modifiable.  Originally we used    //
+         // List.of here - but that is immutable, so, instead use this single-record-list  //
+         // (which we'll create as a field in this class, to avoid always re-constructing) //
+         ////////////////////////////////////////////////////////////////////////////////////
+         singleRecordListForPostRecordActions.add(record);
+         postRecordActions.accept(singleRecordListForPostRecordActions);
+         record = singleRecordListForPostRecordActions.remove(0);
+      }
+
+      doAddRecord(record);
+   }
+
+
+
+   /*******************************************************************************
+    ** Private internal version of add record - assumes the postRecordActions have
+    ** already ran.
+    *******************************************************************************/
+   private void doAddRecord(QRecord record)
    {
       boolean offerResult = queue.offer(record);
 
-      while(!offerResult)
+      if(!offerResult && !isTerminated)
       {
-         LOG.debug("Record pipe.add failed (due to full pipe).  Blocking.");
-         SleepUtils.sleep(100, TimeUnit.MILLISECONDS);
-         offerResult = queue.offer(record);
+         LOG.debug("Pipe is full.  Waiting.");
+         long sleepLoopStartTime = System.currentTimeMillis();
+         long now                = System.currentTimeMillis();
+         while(!offerResult && !isTerminated)
+         {
+            if(now - sleepLoopStartTime > MAX_SLEEP_LOOP_MILLIS)
+            {
+               LOG.warn("Giving up adding record to pipe, due to pipe being full for more than {} millis", MAX_SLEEP_LOOP_MILLIS);
+               throw (new IllegalStateException("Giving up adding record to pipe, due to pipe staying full too long."));
+            }
+            LOG.trace("Record pipe.add failed (due to full pipe).  Blocking.");
+            SleepUtils.sleep(BLOCKING_SLEEP_MILLIS, TimeUnit.MILLISECONDS);
+            offerResult = queue.offer(record);
+            now = System.currentTimeMillis();
+         }
+         LOG.debug("Pipe has opened up.  Resuming.");
       }
    }
 
 
 
    /*******************************************************************************
-    ** Add a list of records to the pipe
+    ** Add a list of records to the pipe.  Will block if the pipe is full.  Will noop if pipe is terminated.
     *******************************************************************************/
    public void addRecords(List<QRecord> records)
    {
-      records.forEach(this::addRecord);
+      if(postRecordActions != null)
+      {
+         postRecordActions.accept(records);
+      }
+
+      //////////////////////////////////////////////////////////////////////////////////////////////////
+      // make sure to go to the private version of doAddRecord - to avoid re-running the post-actions //
+      //////////////////////////////////////////////////////////////////////////////////////////////////
+      records.forEach(this::doAddRecord);
    }
 
 
@@ -78,7 +174,7 @@ public class RecordPipe
    {
       List<QRecord> rs = new ArrayList<>();
 
-      while(true)
+      while(!isTerminated)
       {
          QRecord record = queue.poll();
          if(record == null)
@@ -98,7 +194,22 @@ public class RecordPipe
     *******************************************************************************/
    public int countAvailableRecords()
    {
+      if(isTerminated)
+      {
+         return (0);
+      }
+
       return (queue.size());
    }
 
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public void setPostRecordActions(Consumer<List<QRecord>> postRecordActions)
+   {
+      this.postRecordActions = postRecordActions;
+   }
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/SummaryKey.java

@@ -0,0 +1,131 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.actions.reporting;
+
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Objects;
+import com.kingsrook.qqq.backend.core.utils.Pair;
+
+
+/*******************************************************************************
+ ** For a summary report, a list of field/value pairs that make up a "key".
+ **
+ ** For example, in a report doing summaries by State > City > ZipCode, a SummaryKey
+ ** would look like:  [(state:MO),(city:St.Louis),(zipCode:63101)].
+ *******************************************************************************/
+public class SummaryKey implements Cloneable
+{
+   private List<Pair<String, Serializable>> keys = new ArrayList<>();
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public SummaryKey()
+   {
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public String toString()
+   {
+      return "PivotKey{keys=" + keys + '}';
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public void add(String field, Serializable value)
+   {
+      keys.add(new Pair<>(field, value));
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for keys
+    **
+    *******************************************************************************/
+   public List<Pair<String, Serializable>> getKeys()
+   {
+      return keys;
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public boolean equals(Object o)
+   {
+      if(this == o)
+      {
+         return true;
+      }
+      if(o == null || getClass() != o.getClass())
+      {
+         return false;
+      }
+      SummaryKey summaryKey = (SummaryKey) o;
+      return Objects.equals(keys, summaryKey.keys);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public int hashCode()
+   {
+      return Objects.hash(keys);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public SummaryKey clone()
+   {
+      SummaryKey clone = new SummaryKey();
+
+      for(Pair<String, Serializable> key : keys)
+      {
+         clone.add(key.getA(), key.getB());
+      }
+
+      return (clone);
+   }
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/customizers/DataSourceQueryInputCustomizer.java

@@ -0,0 +1,45 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.actions.reporting.customizers;
+
+
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
+
+
+/*******************************************************************************
+ ** Interface for customizer on a QReportDataSource's query.
+ **
+ ** Useful, for example, to look at what input field values were given, and change
+ ** the query filter (e.g., conditional criteria), or issue an error based on the
+ ** combination of input fields given.
+ *******************************************************************************/
+public interface DataSourceQueryInputCustomizer
+{
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   QueryInput run(ReportInput reportInput, QueryInput queryInput) throws QException;
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/customizers/ReportViewCustomizer.java

@@ -0,0 +1,42 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.actions.reporting.customizers;
+
+
+import java.util.function.Function;
+import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportInput;
+import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportView;
+
+
+/*******************************************************************************
+ ** Interface for customizer on a QReportView.  Extends Function by adding setter
+ ** method for reportInput.
+ *******************************************************************************/
+public interface ReportViewCustomizer extends Function<QReportView, QReportView>
+{
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   void setReportInput(ReportInput reportInput);
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/excelformatting/BoldHeaderAndFooterExcelStyler.java

@@ -0,0 +1,71 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.actions.reporting.excelformatting;
+
+
+import org.dhatim.fastexcel.BorderSide;
+import org.dhatim.fastexcel.BorderStyle;
+import org.dhatim.fastexcel.StyleSetter;
+
+
+/*******************************************************************************
+ ** Version of excel styler that does bold headers and footers, with basic borders.
+ *******************************************************************************/
+public class BoldHeaderAndFooterExcelStyler implements ExcelStylerInterface
+{
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public void styleTitleRow(StyleSetter titleRowStyle)
+   {
+      titleRowStyle
+         .bold()
+         .fontSize(14)
+         .horizontalAlignment("center");
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public void styleHeaderRow(StyleSetter headerRowStyle)
+   {
+      headerRowStyle
+         .bold()
+         .borderStyle(BorderSide.BOTTOM, BorderStyle.THIN);
+   }
+
+
+
+   @Override
+   public void styleTotalsRow(StyleSetter totalsRowStyle)
+   {
+      totalsRowStyle
+         .bold()
+         .borderStyle(BorderSide.TOP, BorderStyle.THIN)
+         .borderStyle(BorderSide.BOTTOM, BorderStyle.DOUBLE);
+   }
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/excelformatting/ExcelStylerInterface.java

@@ -0,0 +1,59 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.actions.reporting.excelformatting;
+
+
+import org.dhatim.fastexcel.StyleSetter;
+
+
+/*******************************************************************************
+ ** Interface for classes that know how to apply styles to an Excel stream being
+ ** built by fastexcel.
+ *******************************************************************************/
+public interface ExcelStylerInterface
+{
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   default void styleTitleRow(StyleSetter titleRowStyle)
+   {
+
+   }
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   default void styleHeaderRow(StyleSetter headerRowStyle)
+   {
+
+   }
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   default void styleTotalsRow(StyleSetter totalsRowStyle)
+   {
+
+   }
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/excelformatting/PlainExcelStyler.java

@@ -0,0 +1,31 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.actions.reporting.excelformatting;
+
+
+/*******************************************************************************
+ ** Excel styler that does nothing - just takes defaults (which are all no-op) from the interface.
+ *******************************************************************************/
+public class PlainExcelStyler implements ExcelStylerInterface
+{
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/scripts/ExecuteCodeAction.java

@@ -0,0 +1,119 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.actions.scripts;
+
+
+import java.io.Serializable;
+import java.util.HashMap;
+import java.util.Map;
+import com.kingsrook.qqq.backend.core.actions.scripts.logging.Log4jCodeExecutionLogger;
+import com.kingsrook.qqq.backend.core.actions.scripts.logging.QCodeExecutionLoggerInterface;
+import com.kingsrook.qqq.backend.core.exceptions.QCodeException;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.actions.scripts.ExecuteCodeInput;
+import com.kingsrook.qqq.backend.core.model.actions.scripts.ExecuteCodeOutput;
+import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
+
+
+/*******************************************************************************
+ ** Action to execute user/runtime defined code.
+ **
+ ** This action is designed to support code in multiple languages, by using
+ ** executors, e.g., provided by additional runtime qqq dependencies.  Initially
+ ** we are building qqq-language-support-javascript.
+ **
+ ** We also have a Java executor, to provide at least a little bit of testability
+ ** within qqq-backend-core.  This executor is a candidate to be replaced in the
+ ** future with something that would do actual dynamic java (whether that's compiled
+ ** at runtime, or loaded from a plugin jar at runtime).  In other words, the java
+ ** executor in place today is just meant to be a placeholder.
+ *******************************************************************************/
+public class ExecuteCodeAction
+{
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @SuppressWarnings("checkstyle:indentation")
+   public void run(ExecuteCodeInput input, ExecuteCodeOutput output) throws QException, QCodeException
+   {
+      QCodeReference codeReference = input.getCodeReference();
+
+      QCodeExecutionLoggerInterface executionLogger = input.getExecutionLogger();
+      if(executionLogger == null)
+      {
+         executionLogger = getDefaultExecutionLogger();
+      }
+      executionLogger.acceptExecutionStart(input);
+
+      try
+      {
+         String languageExecutor = switch(codeReference.getCodeType())
+            {
+               case JAVA -> "com.kingsrook.qqq.backend.core.actions.scripts.QJavaExecutor";
+               case JAVA_SCRIPT -> "com.kingsrook.qqq.languages.javascript.QJavaScriptExecutor";
+            };
+
+         @SuppressWarnings("unchecked")
+         Class<? extends QCodeExecutor> executorClass = (Class<? extends QCodeExecutor>) Class.forName(languageExecutor);
+         QCodeExecutor qCodeExecutor = executorClass.getConstructor().newInstance();
+
+         ////////////////////////////////////////////////////////////////////////////////////////////////////
+         // merge all of the input context, plus the input... input - into a context for the code executor //
+         ////////////////////////////////////////////////////////////////////////////////////////////////////
+         Map<String, Serializable> context = new HashMap<>();
+         if(input.getContext() != null)
+         {
+            context.putAll(input.getContext());
+         }
+         if(input.getInput() != null)
+         {
+            context.putAll(input.getInput());
+         }
+
+         Serializable codeOutput = qCodeExecutor.execute(codeReference, context, executionLogger);
+         output.setOutput(codeOutput);
+         executionLogger.acceptExecutionEnd(codeOutput);
+      }
+      catch(QCodeException qCodeException)
+      {
+         executionLogger.acceptException(qCodeException);
+         throw (qCodeException);
+      }
+      catch(Exception e)
+      {
+         executionLogger.acceptException(e);
+         throw (new QException("Error executing code [" + codeReference + "]", e));
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private QCodeExecutionLoggerInterface getDefaultExecutionLogger()
+   {
+      return (new Log4jCodeExecutionLogger());
+   }
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/scripts/QCodeExecutor.java

@@ -0,0 +1,44 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.actions.scripts;
+
+
+import java.io.Serializable;
+import java.util.Map;
+import com.kingsrook.qqq.backend.core.actions.scripts.logging.QCodeExecutionLoggerInterface;
+import com.kingsrook.qqq.backend.core.exceptions.QCodeException;
+import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
+
+
+/*******************************************************************************
+ ** Interface to be implemented by language-specific code executors, e.g., in
+ ** qqq-language-support-${languageName} maven modules.
+ *******************************************************************************/
+public interface QCodeExecutor
+{
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   Serializable execute(QCodeReference codeReference, Map<String, Serializable> inputContext, QCodeExecutionLoggerInterface executionLogger) throws QCodeException;
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/scripts/QJavaExecutor.java

@@ -0,0 +1,68 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.actions.scripts;
+
+
+import java.io.Serializable;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.function.Function;
+import com.kingsrook.qqq.backend.core.actions.customizers.QCodeLoader;
+import com.kingsrook.qqq.backend.core.actions.scripts.logging.QCodeExecutionLoggerInterface;
+import com.kingsrook.qqq.backend.core.exceptions.QCodeException;
+import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
+
+
+/*******************************************************************************
+ ** Java
+ *******************************************************************************/
+public class QJavaExecutor implements QCodeExecutor
+{
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public Serializable execute(QCodeReference codeReference, Map<String, Serializable> inputContext, QCodeExecutionLoggerInterface executionLogger) throws QCodeException
+   {
+      Map<String, Object> context = new HashMap<>(inputContext);
+      if(!context.containsKey("logger"))
+      {
+         context.put("logger", executionLogger);
+      }
+
+      Serializable output;
+      try
+      {
+         Function<Map<String, Object>, Serializable> function = QCodeLoader.getFunction(codeReference);
+         output = function.apply(context);
+      }
+      catch(Exception e)
+      {
+         QCodeException qCodeException = new QCodeException("Error executing script", e);
+         throw (qCodeException);
+      }
+
+      return (output);
+   }
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/scripts/RunAdHocRecordScriptAction.java

@@ -0,0 +1,219 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.actions.scripts;
+
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import com.kingsrook.qqq.backend.core.actions.ActionHelper;
+import com.kingsrook.qqq.backend.core.actions.scripts.logging.QCodeExecutionLoggerInterface;
+import com.kingsrook.qqq.backend.core.actions.scripts.logging.ScriptExecutionLoggerInterface;
+import com.kingsrook.qqq.backend.core.actions.scripts.logging.StoreScriptLogAndScriptLogLineExecutionLogger;
+import com.kingsrook.qqq.backend.core.actions.tables.GetAction;
+import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
+import com.kingsrook.qqq.backend.core.context.QContext;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.logging.QLogger;
+import com.kingsrook.qqq.backend.core.model.actions.scripts.ExecuteCodeInput;
+import com.kingsrook.qqq.backend.core.model.actions.scripts.ExecuteCodeOutput;
+import com.kingsrook.qqq.backend.core.model.actions.scripts.RunAdHocRecordScriptInput;
+import com.kingsrook.qqq.backend.core.model.actions.scripts.RunAdHocRecordScriptOutput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetOutput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryJoin;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput;
+import com.kingsrook.qqq.backend.core.model.metadata.code.AdHocScriptCodeReference;
+import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
+import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeType;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+import com.kingsrook.qqq.backend.core.model.scripts.Script;
+import com.kingsrook.qqq.backend.core.model.scripts.ScriptRevision;
+import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
+
+
+/*******************************************************************************
+ **
+ *******************************************************************************/
+public class RunAdHocRecordScriptAction
+{
+   private static final QLogger LOG = QLogger.getLogger(RunAdHocRecordScriptAction.class);
+
+   private Map<Integer, ScriptRevision> scriptRevisionCacheByScriptRevisionId = new HashMap<>();
+   private Map<Integer, ScriptRevision> scriptRevisionCacheByScriptId         = new HashMap<>();
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public void run(RunAdHocRecordScriptInput input, RunAdHocRecordScriptOutput output) throws QException
+   {
+      try
+      {
+         ActionHelper.validateSession(input);
+
+         /////////////////////////
+         // figure out the code //
+         /////////////////////////
+         ScriptRevision scriptRevision = getScriptRevision(input);
+         if(scriptRevision == null)
+         {
+            throw (new QException("Script revision was not found."));
+         }
+
+         ////////////////////////////
+         // figure out the records //
+         ////////////////////////////
+         QTableMetaData table = QContext.getQInstance().getTable(input.getTableName());
+         if(CollectionUtils.nullSafeIsEmpty(input.getRecordList()))
+         {
+            QueryInput queryInput = new QueryInput();
+            queryInput.setTableName(input.getTableName());
+            queryInput.setFilter(new QQueryFilter(new QFilterCriteria(table.getPrimaryKeyField(), QCriteriaOperator.IN, input.getRecordPrimaryKeyList())));
+            QueryOutput queryOutput = new QueryAction().execute(queryInput);
+            input.setRecordList(queryOutput.getRecords());
+         }
+
+         if(CollectionUtils.nullSafeIsEmpty(input.getRecordList()))
+         {
+            ////////////////////////////////////////
+            // just return if nothing found?  idk //
+            ////////////////////////////////////////
+            LOG.info("No records supplied as input (or found via primary keys); exiting with noop");
+            return;
+         }
+
+         /////////////
+         // run it! //
+         /////////////
+         ExecuteCodeInput executeCodeInput = new ExecuteCodeInput();
+         executeCodeInput.setInput(new HashMap<>(Objects.requireNonNullElseGet(input.getInputValues(), HashMap::new)));
+         executeCodeInput.getInput().put("records", new ArrayList<>(input.getRecordList()));
+         executeCodeInput.setContext(new HashMap<>());
+         if(input.getOutputObject() != null)
+         {
+            executeCodeInput.getContext().put("output", input.getOutputObject());
+         }
+
+         if(input.getScriptUtils() != null)
+         {
+            executeCodeInput.getContext().put("scriptUtils", input.getScriptUtils());
+         }
+
+         executeCodeInput.getContext().put("api", new ScriptApi());
+
+         executeCodeInput.setCodeReference(new QCodeReference().withInlineCode(scriptRevision.getContents()).withCodeType(QCodeType.JAVA_SCRIPT)); // todo - code type as attribute of script!!
+
+         /////////////////////////////////////////////////////////////////////////////////////////////////
+         // let caller supply a logger, or by default use StoreScriptLogAndScriptLogLineExecutionLogger //
+         /////////////////////////////////////////////////////////////////////////////////////////////////
+         QCodeExecutionLoggerInterface executionLogger = Objects.requireNonNullElseGet(input.getLogger(), () -> new StoreScriptLogAndScriptLogLineExecutionLogger(scriptRevision.getScriptId(), scriptRevision.getId()));
+         executeCodeInput.setExecutionLogger(executionLogger);
+         if(executionLogger instanceof ScriptExecutionLoggerInterface scriptExecutionLoggerInterface)
+         {
+            ////////////////////////////////////////////////////////////////////////////////////////////////////
+            // if logger is aware of scripts (as opposed to a generic CodeExecution logger), give it the ids. //
+            ////////////////////////////////////////////////////////////////////////////////////////////////////
+            scriptExecutionLoggerInterface.setScriptId(scriptRevision.getScriptId());
+            scriptExecutionLoggerInterface.setScriptRevisionId(scriptRevision.getId());
+         }
+
+         ExecuteCodeOutput executeCodeOutput = new ExecuteCodeOutput();
+         new ExecuteCodeAction().run(executeCodeInput, executeCodeOutput);
+
+         output.setOutput(executeCodeOutput.getOutput());
+         output.setLogger(executionLogger);
+      }
+      catch(Exception e)
+      {
+         output.setException(Optional.of(e));
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private ScriptRevision getScriptRevision(RunAdHocRecordScriptInput input) throws QException
+   {
+      AdHocScriptCodeReference codeReference = input.getCodeReference();
+      if(codeReference.getScriptRevisionRecord() != null)
+      {
+         return (new ScriptRevision(codeReference.getScriptRevisionRecord()));
+      }
+
+      if(codeReference.getScriptRevisionId() != null)
+      {
+         if(!scriptRevisionCacheByScriptRevisionId.containsKey(codeReference.getScriptRevisionId()))
+         {
+            GetInput getInput = new GetInput();
+            getInput.setTableName(ScriptRevision.TABLE_NAME);
+            getInput.setPrimaryKey(codeReference.getScriptRevisionId());
+            GetOutput getOutput = new GetAction().execute(getInput);
+            if(getOutput.getRecord() != null)
+            {
+               scriptRevisionCacheByScriptRevisionId.put(codeReference.getScriptRevisionId(), new ScriptRevision(getOutput.getRecord()));
+            }
+            else
+            {
+               scriptRevisionCacheByScriptRevisionId.put(codeReference.getScriptRevisionId(), null);
+            }
+         }
+
+         return (scriptRevisionCacheByScriptRevisionId.get(codeReference.getScriptRevisionId()));
+      }
+
+      if(codeReference.getScriptId() != null)
+      {
+         if(!scriptRevisionCacheByScriptId.containsKey(codeReference.getScriptId()))
+         {
+            QueryInput queryInput = new QueryInput();
+            queryInput.setTableName(ScriptRevision.TABLE_NAME);
+            queryInput.setFilter(new QQueryFilter(new QFilterCriteria("script.id", QCriteriaOperator.EQUALS, codeReference.getScriptId())));
+            queryInput.withQueryJoin(new QueryJoin(Script.TABLE_NAME).withBaseTableOrAlias(ScriptRevision.TABLE_NAME).withJoinMetaData(QContext.getQInstance().getJoin("currentScriptRevision")));
+            QueryOutput queryOutput = new QueryAction().execute(queryInput);
+
+            if(CollectionUtils.nullSafeHasContents(queryOutput.getRecords()))
+            {
+               scriptRevisionCacheByScriptId.put(codeReference.getScriptId(), new ScriptRevision(queryOutput.getRecords().get(0)));
+            }
+            else
+            {
+               scriptRevisionCacheByScriptId.put(codeReference.getScriptId(), null);
+            }
+         }
+
+         return (scriptRevisionCacheByScriptId.get(codeReference.getScriptId()));
+      }
+
+      throw (new QException("Code reference did not contain a scriptRevision, scriptRevisionId, or scriptId"));
+   }
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/scripts/RunAssociatedScriptAction.java

@@ -0,0 +1,191 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.actions.scripts;
+
+
+import java.io.Serializable;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Objects;
+import com.kingsrook.qqq.backend.core.actions.ActionHelper;
+import com.kingsrook.qqq.backend.core.actions.scripts.logging.QCodeExecutionLoggerInterface;
+import com.kingsrook.qqq.backend.core.actions.scripts.logging.ScriptExecutionLoggerInterface;
+import com.kingsrook.qqq.backend.core.actions.scripts.logging.StoreScriptLogAndScriptLogLineExecutionLogger;
+import com.kingsrook.qqq.backend.core.actions.tables.GetAction;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.exceptions.QNotFoundException;
+import com.kingsrook.qqq.backend.core.model.actions.scripts.ExecuteCodeInput;
+import com.kingsrook.qqq.backend.core.model.actions.scripts.ExecuteCodeOutput;
+import com.kingsrook.qqq.backend.core.model.actions.scripts.RunAssociatedScriptInput;
+import com.kingsrook.qqq.backend.core.model.actions.scripts.RunAssociatedScriptOutput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetOutput;
+import com.kingsrook.qqq.backend.core.model.metadata.code.AssociatedScriptCodeReference;
+import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
+import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeType;
+import com.kingsrook.qqq.backend.core.model.scripts.Script;
+import com.kingsrook.qqq.backend.core.model.scripts.ScriptRevision;
+
+
+/*******************************************************************************
+ **
+ *******************************************************************************/
+public class RunAssociatedScriptAction
+{
+   private Map<AssociatedScriptCodeReference, ScriptRevision> scriptRevisionCache = new HashMap<>();
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public void run(RunAssociatedScriptInput input, RunAssociatedScriptOutput output) throws QException
+   {
+      ActionHelper.validateSession(input);
+
+      ScriptRevision scriptRevision = getScriptRevision(input);
+
+      ExecuteCodeInput executeCodeInput = new ExecuteCodeInput();
+      executeCodeInput.setInput(new HashMap<>(input.getInputValues()));
+      executeCodeInput.setContext(new HashMap<>());
+      if(input.getOutputObject() != null)
+      {
+         executeCodeInput.getContext().put("output", input.getOutputObject());
+      }
+
+      if(input.getScriptUtils() != null)
+      {
+         executeCodeInput.getContext().put("scriptUtils", input.getScriptUtils());
+      }
+
+      executeCodeInput.setCodeReference(new QCodeReference().withInlineCode(scriptRevision.getContents()).withCodeType(QCodeType.JAVA_SCRIPT)); // todo - code type as attribute of script!!
+
+      /////////////////////////////////////////////////////////////////////////////////////////////////
+      // let caller supply a logger, or by default use StoreScriptLogAndScriptLogLineExecutionLogger //
+      /////////////////////////////////////////////////////////////////////////////////////////////////
+      QCodeExecutionLoggerInterface executionLogger = Objects.requireNonNullElseGet(input.getLogger(), () -> new StoreScriptLogAndScriptLogLineExecutionLogger(scriptRevision.getScriptId(), scriptRevision.getId()));
+      executeCodeInput.setExecutionLogger(executionLogger);
+      if(executionLogger instanceof ScriptExecutionLoggerInterface scriptExecutionLoggerInterface)
+      {
+         ////////////////////////////////////////////////////////////////////////////////////////////////////
+         // if logger is aware of scripts (as opposed to a generic CodeExecution logger), give it the ids. //
+         ////////////////////////////////////////////////////////////////////////////////////////////////////
+         scriptExecutionLoggerInterface.setScriptId(scriptRevision.getScriptId());
+         scriptExecutionLoggerInterface.setScriptRevisionId(scriptRevision.getId());
+      }
+
+      ExecuteCodeOutput executeCodeOutput = new ExecuteCodeOutput();
+      new ExecuteCodeAction().run(executeCodeInput, executeCodeOutput);
+
+      output.setOutput(executeCodeOutput.getOutput());
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private ScriptRevision getScriptRevision(RunAssociatedScriptInput input) throws QException
+   {
+      if(!scriptRevisionCache.containsKey(input.getCodeReference()))
+      {
+         Serializable scriptId = getScriptId(input);
+         if(scriptId == null)
+         {
+            throw (new QNotFoundException("The input record [" + input.getCodeReference().getRecordTable() + "][" + input.getCodeReference().getRecordPrimaryKey()
+               + "] does not have a script specified for [" + input.getCodeReference().getFieldName() + "]"));
+         }
+
+         Script script = getScript(input, scriptId);
+         if(script.getCurrentScriptRevisionId() == null)
+         {
+            throw (new QNotFoundException("The script for record [" + input.getCodeReference().getRecordTable() + "][" + input.getCodeReference().getRecordPrimaryKey()
+               + "] (scriptId=" + scriptId + ") does not have a current version."));
+         }
+
+         ScriptRevision scriptRevision = getCurrentScriptRevision(input, script.getCurrentScriptRevisionId());
+         scriptRevisionCache.put(input.getCodeReference(), scriptRevision);
+      }
+
+      return scriptRevisionCache.get(input.getCodeReference());
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private ScriptRevision getCurrentScriptRevision(RunAssociatedScriptInput input, Serializable scriptRevisionId) throws QException
+   {
+      GetInput getInput = new GetInput();
+      getInput.setTableName("scriptRevision");
+      getInput.setPrimaryKey(scriptRevisionId);
+      GetOutput getOutput = new GetAction().execute(getInput);
+      if(getOutput.getRecord() == null)
+      {
+         throw (new QNotFoundException("The current revision of the script for record [" + input.getCodeReference().getRecordTable() + "][" + input.getCodeReference().getRecordPrimaryKey() + "]["
+            + input.getCodeReference().getFieldName() + "] (scriptRevisionId=" + scriptRevisionId + ") was not found."));
+      }
+
+      return (new ScriptRevision(getOutput.getRecord()));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private Script getScript(RunAssociatedScriptInput input, Serializable scriptId) throws QException
+   {
+      GetInput getInput = new GetInput();
+      getInput.setTableName("script");
+      getInput.setPrimaryKey(scriptId);
+      GetOutput getOutput = new GetAction().execute(getInput);
+
+      if(getOutput.getRecord() == null)
+      {
+         throw (new QNotFoundException("The script for record [" + input.getCodeReference().getRecordTable() + "][" + input.getCodeReference().getRecordPrimaryKey() + "]["
+            + input.getCodeReference().getFieldName() + "] (script id=" + scriptId + ") was not found."));
+      }
+
+      return (new Script(getOutput.getRecord()));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private Serializable getScriptId(RunAssociatedScriptInput input) throws QException
+   {
+      GetInput getInput = new GetInput();
+      getInput.setTableName(input.getCodeReference().getRecordTable());
+      getInput.setPrimaryKey(input.getCodeReference().getRecordPrimaryKey());
+      GetOutput getOutput = new GetAction().execute(getInput);
+      if(getOutput.getRecord() == null)
+      {
+         throw (new QNotFoundException("The requested record [" + input.getCodeReference().getRecordTable() + "][" + input.getCodeReference().getRecordPrimaryKey() + "] was not found."));
+      }
+
+      return (getOutput.getRecord().getValue(input.getCodeReference().getFieldName()));
+   }
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/scripts/ScriptApi.java

@@ -0,0 +1,236 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2023.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.actions.scripts;
+
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.List;
+import com.kingsrook.qqq.backend.core.actions.permissions.PermissionsHelper;
+import com.kingsrook.qqq.backend.core.actions.permissions.TablePermissionSubType;
+import com.kingsrook.qqq.backend.core.actions.tables.DeleteAction;
+import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
+import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
+import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction;
+import com.kingsrook.qqq.backend.core.context.QContext;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+
+
+/*******************************************************************************
+ ** Object made available to scripts for access to qqq api (e.g., query, insert,
+ ** etc, plus object constructors).
+ *******************************************************************************/
+public class ScriptApi implements Serializable
+{
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public QueryInput newQueryInput()
+   {
+      return (new QueryInput());
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public QQueryFilter newQueryFilter()
+   {
+      return (new QQueryFilter());
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public QFilterCriteria newFilterCriteria()
+   {
+      return (new QFilterCriteria());
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public QFilterOrderBy newFilterOrderBy()
+   {
+      return (new QFilterOrderBy());
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public QRecord newRecord()
+   {
+      return (new QRecord());
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public List<QRecord> query(String tableName, QQueryFilter filter) throws QException
+   {
+      QueryInput queryInput = new QueryInput();
+      queryInput.setTableName(tableName);
+      queryInput.setFilter(filter);
+      PermissionsHelper.checkTablePermissionThrowing(queryInput, TablePermissionSubType.READ);
+      return (new QueryAction().execute(queryInput).getRecords());
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public List<QRecord> query(QueryInput queryInput) throws QException
+   {
+      PermissionsHelper.checkTablePermissionThrowing(queryInput, TablePermissionSubType.READ);
+      return (new QueryAction().execute(queryInput).getRecords());
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public void insert(String tableName, QRecord record) throws QException
+   {
+      insert(tableName, List.of(record));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public void insert(String tableName, List<QRecord> recordList) throws QException
+   {
+      InsertInput insertInput = new InsertInput();
+      insertInput.setTableName(tableName);
+      insertInput.setRecords(recordList);
+      PermissionsHelper.checkTablePermissionThrowing(insertInput, TablePermissionSubType.INSERT);
+      new InsertAction().execute(insertInput);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public void update(String tableName, QRecord record) throws QException
+   {
+      update(tableName, List.of(record));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public void update(String tableName, List<QRecord> recordList) throws QException
+   {
+      UpdateInput updateInput = new UpdateInput();
+      updateInput.setTableName(tableName);
+      updateInput.setRecords(recordList);
+      PermissionsHelper.checkTablePermissionThrowing(updateInput, TablePermissionSubType.EDIT);
+      new UpdateAction().execute(updateInput);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public void delete(String tableName, Serializable primaryKey) throws QException
+   {
+      delete(tableName, List.of(primaryKey));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public void delete(String tableName, QRecord record) throws QException
+   {
+      delete(tableName, List.of(record));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public void delete(String tableName, List<?> recordOrPrimaryKeyList) throws QException
+   {
+      QTableMetaData table       = QContext.getQInstance().getTable(tableName);
+      DeleteInput    deleteInput = new DeleteInput();
+      deleteInput.setTableName(tableName);
+      List<Serializable> primaryKeyList = new ArrayList<>();
+      for(Object o : recordOrPrimaryKeyList)
+      {
+         if(o instanceof QRecord qRecord)
+         {
+            primaryKeyList.add(qRecord.getValue(table.getPrimaryKeyField()));
+         }
+         else
+         {
+            primaryKeyList.add((Serializable) o);
+         }
+      }
+      deleteInput.setPrimaryKeys(primaryKeyList);
+      PermissionsHelper.checkTablePermissionThrowing(deleteInput, TablePermissionSubType.DELETE);
+      new DeleteAction().execute(deleteInput);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public void delete(String tableName, QQueryFilter filter) throws QException
+   {
+      DeleteInput deleteInput = new DeleteInput();
+      deleteInput.setTableName(tableName);
+      deleteInput.setQueryFilter(filter);
+      PermissionsHelper.checkTablePermissionThrowing(deleteInput, TablePermissionSubType.DELETE);
+      new DeleteAction().execute(deleteInput);
+   }
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/scripts/StoreAssociatedScriptAction.java

@@ -0,0 +1,219 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.actions.scripts;
+
+
+import java.io.Serializable;
+import java.util.List;
+import java.util.Optional;
+import com.kingsrook.qqq.backend.core.actions.ActionHelper;
+import com.kingsrook.qqq.backend.core.actions.tables.GetAction;
+import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
+import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
+import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.actions.scripts.StoreAssociatedScriptInput;
+import com.kingsrook.qqq.backend.core.model.actions.scripts.StoreAssociatedScriptOutput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetOutput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertOutput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.AssociatedScript;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+import com.kingsrook.qqq.backend.core.utils.StringUtils;
+
+
+/*******************************************************************************
+ ** Action to store a new version of a script, associated with a record.
+ **
+ ** If there's never been a script assigned to the record (for the specified field),
+ ** then a new Script record is first inserted.
+ **
+ ** The script referenced by the record is always updated to point at the new
+ ** scriptRevision record that is inserted.
+ **
+ *******************************************************************************/
+public class StoreAssociatedScriptAction
+{
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public void run(StoreAssociatedScriptInput input, StoreAssociatedScriptOutput output) throws QException
+   {
+      ActionHelper.validateSession(input);
+
+      QTableMetaData             table               = input.getTable();
+      Optional<AssociatedScript> optAssociatedScript = table.getAssociatedScripts().stream().filter(as -> as.getFieldName().equals(input.getFieldName())).findFirst();
+      if(optAssociatedScript.isEmpty())
+      {
+         throw (new QException("Field to update associated script for is not an associated script field."));
+      }
+      AssociatedScript associatedScript = optAssociatedScript.get();
+
+      /////////////////////////////////////////////////////////////
+      // get the record that the script is to be associated with //
+      /////////////////////////////////////////////////////////////
+      QRecord associatedRecord;
+      {
+         GetInput getInput = new GetInput();
+         getInput.setTableName(input.getTableName());
+         getInput.setPrimaryKey(input.getRecordPrimaryKey());
+         getInput.setShouldGenerateDisplayValues(true);
+         GetOutput getOutput = new GetAction().execute(getInput);
+         associatedRecord = getOutput.getRecord();
+      }
+      if(associatedRecord == null)
+      {
+         throw (new QException("Record to associated with script was not found."));
+      }
+
+      //////////////////////////////////////////////////////////////////
+      // check if there's currently a script referenced by the record //
+      //////////////////////////////////////////////////////////////////
+      Serializable existingScriptId = associatedRecord.getValueString(input.getFieldName());
+      QRecord      script;
+      Integer      nextSequenceNo   = 1;
+      if(existingScriptId == null)
+      {
+         ////////////////////////////////////////////////////////////////////
+         // get the script type - that'll be part of the new script's name //
+         ////////////////////////////////////////////////////////////////////
+         GetInput getInput = new GetInput();
+         getInput.setTableName("scriptType");
+         getInput.setPrimaryKey(associatedScript.getScriptTypeId());
+         getInput.setShouldGenerateDisplayValues(true);
+         GetOutput getOutput  = new GetAction().execute(getInput);
+         QRecord   scriptType = getOutput.getRecord();
+         if(scriptType == null)
+         {
+            throw (new QException("Script type [" + associatedScript.getScriptTypeId() + "] was not found."));
+         }
+
+         /////////////////////////
+         // insert a new script //
+         /////////////////////////
+         script = new QRecord();
+         script.setValue("scriptTypeId", associatedScript.getScriptTypeId());
+         script.setValue("name", associatedRecord.getRecordLabel() + " - " + scriptType.getRecordLabel());
+         InsertInput insertInput = new InsertInput();
+         insertInput.setTableName("script");
+         insertInput.setRecords(List.of(script));
+         InsertOutput insertOutput = new InsertAction().execute(insertInput);
+         script = insertOutput.getRecords().get(0);
+
+         /////////////////////////////////////////////////////////////
+         // update the associated record to point at the new script //
+         /////////////////////////////////////////////////////////////
+         UpdateInput updateInput = new UpdateInput();
+         updateInput.setTableName(input.getTableName());
+         updateInput.setRecords(List.of(new QRecord()
+            .withValue(table.getPrimaryKeyField(), associatedRecord.getValue(table.getPrimaryKeyField()))
+            .withValue(input.getFieldName(), script.getValue("id"))
+         ));
+         new UpdateAction().execute(updateInput);
+      }
+      else
+      {
+         ////////////////////////////////////////
+         // get the existing script, to update //
+         ////////////////////////////////////////
+         GetInput getInput = new GetInput();
+         getInput.setTableName("script");
+         getInput.setPrimaryKey(existingScriptId);
+         GetOutput getOutput = new GetAction().execute(getInput);
+         script = getOutput.getRecord();
+
+         QueryInput queryInput = new QueryInput();
+         queryInput.setTableName("scriptRevision");
+         queryInput.setFilter(new QQueryFilter()
+            .withCriteria(new QFilterCriteria("scriptId", QCriteriaOperator.EQUALS, List.of(script.getValue("id"))))
+            .withOrderBy(new QFilterOrderBy("sequenceNo", false))
+         );
+         queryInput.setLimit(1);
+         QueryOutput queryOutput = new QueryAction().execute(queryInput);
+         if(!queryOutput.getRecords().isEmpty())
+         {
+            nextSequenceNo = queryOutput.getRecords().get(0).getValueInteger("sequenceNo") + 1;
+         }
+      }
+
+      //////////////////////////////////
+      // insert a new script revision //
+      //////////////////////////////////
+      String commitMessage = input.getCommitMessage();
+      if(!StringUtils.hasContent(commitMessage))
+      {
+         if(nextSequenceNo == 1)
+         {
+            commitMessage = "Initial version";
+         }
+         else
+         {
+            commitMessage = "No commit message given";
+         }
+      }
+
+      QRecord scriptRevision = new QRecord()
+         .withValue("scriptId", script.getValue("id"))
+         .withValue("contents", input.getCode())
+         .withValue("commitMessage", commitMessage)
+         .withValue("sequenceNo", nextSequenceNo);
+
+      try
+      {
+         scriptRevision.setValue("author", input.getSession().getUser().getFullName());
+      }
+      catch(Exception e)
+      {
+         scriptRevision.setValue("author", "Unknown");
+      }
+
+      InsertInput insertInput = new InsertInput();
+      insertInput.setTableName("scriptRevision");
+      insertInput.setRecords(List.of(scriptRevision));
+      InsertOutput insertOutput = new InsertAction().execute(insertInput);
+      scriptRevision = insertOutput.getRecords().get(0);
+
+      ////////////////////////////////////////////////////
+      // update the script to point at the new revision //
+      ////////////////////////////////////////////////////
+      script.setValue("currentScriptRevisionId", scriptRevision.getValue("id"));
+      UpdateInput updateInput = new UpdateInput();
+      updateInput.setTableName("script");
+      updateInput.setRecords(List.of(script));
+      new UpdateAction().execute(updateInput);
+
+      output.setScriptId(script.getValueInteger("id"));
+      output.setScriptName(script.getValueString("name"));
+      output.setScriptRevisionId(scriptRevision.getValueInteger("id"));
+      output.setScriptRevisionSequenceNo(scriptRevision.getValueInteger("sequenceNo"));
+   }
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/scripts/TestScriptActionInterface.java

@@ -0,0 +1,108 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.actions.scripts;
+
+
+import java.io.Serializable;
+import java.util.HashMap;
+import java.util.List;
+import com.kingsrook.qqq.backend.core.actions.scripts.logging.BuildScriptLogAndScriptLogLineExecutionLogger;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.actions.scripts.ExecuteCodeInput;
+import com.kingsrook.qqq.backend.core.model.actions.scripts.ExecuteCodeOutput;
+import com.kingsrook.qqq.backend.core.model.actions.scripts.TestScriptInput;
+import com.kingsrook.qqq.backend.core.model.actions.scripts.TestScriptOutput;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
+
+
+/*******************************************************************************
+ ** Interface to be implemented by script-running actions, if they want to allow
+ ** themselves to be used for user-testing of their script.
+ *******************************************************************************/
+public interface TestScriptActionInterface
+{
+   /*******************************************************************************
+    ** Called to adapt or translate data from the TestScriptInput (which would just
+    ** have a map of name-value pairs) to the actual input object(s) used by the script.
+    **
+    ** Note - such a method may want or need to put an "output" object into the
+    ** executeCodeInput's context map.
+    *******************************************************************************/
+   void setupTestScriptInput(TestScriptInput testScriptInput, ExecuteCodeInput executeCodeInput);
+
+
+   /*******************************************************************************
+    ** Called to adapt or translate the output object of the script execution to
+    ** something suitable for returning to the caller.
+    **
+    ** Default implementation may always be suitable?
+    *******************************************************************************/
+   default Serializable processTestScriptOutput(ExecuteCodeOutput executeCodeOutput)
+   {
+      return (executeCodeOutput.getOutput());
+   }
+
+
+   /*******************************************************************************
+    ** Define the list of input fields for testing the script.  The names of these
+    ** fields will end up as keys in the setupTestScriptInput method's testScriptInput object.
+    *******************************************************************************/
+   List<QFieldMetaData> getTestInputFields();
+
+
+   /*******************************************************************************
+    ** Define the list of output fields when testing the script.  The output object
+    ** returned from processTestScriptOutput should have keys that match these field names.
+    *******************************************************************************/
+   List<QFieldMetaData> getTestOutputFields();
+
+   /*******************************************************************************
+    ** Execute a test script.
+    *******************************************************************************/
+   default void execute(TestScriptInput input, TestScriptOutput output) throws QException
+   {
+      ExecuteCodeInput executeCodeInput = new ExecuteCodeInput();
+      executeCodeInput.setContext(new HashMap<>());
+
+      executeCodeInput.setCodeReference(input.getCodeReference());
+      BuildScriptLogAndScriptLogLineExecutionLogger executionLogger = new BuildScriptLogAndScriptLogLineExecutionLogger(null, null);
+      executeCodeInput.setExecutionLogger(executionLogger);
+
+      setupTestScriptInput(input, executeCodeInput);
+
+      ExecuteCodeOutput executeCodeOutput = new ExecuteCodeOutput();
+
+      try
+      {
+         new ExecuteCodeAction().run(executeCodeInput, executeCodeOutput);
+         output.setOutputObject(processTestScriptOutput(executeCodeOutput));
+      }
+      catch(Exception e)
+      {
+         output.setException(e);
+      }
+
+      output.setScriptLog(executionLogger.getScriptLog());
+      output.setScriptLogLines(executionLogger.getScriptLogLines());
+   }
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/scripts/logging/AccumulatingBuildScriptLogAndScriptLogLineExecutionLogger.java

@@ -0,0 +1,160 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2023.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.actions.scripts.logging;
+
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.List;
+import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
+import com.kingsrook.qqq.backend.core.logging.QLogger;
+import com.kingsrook.qqq.backend.core.model.actions.scripts.ExecuteCodeInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertOutput;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+
+
+/*******************************************************************************
+ **
+ *******************************************************************************/
+public class AccumulatingBuildScriptLogAndScriptLogLineExecutionLogger extends BuildScriptLogAndScriptLogLineExecutionLogger implements ScriptExecutionLoggerInterface
+{
+   private static final QLogger LOG = QLogger.getLogger(AccumulatingBuildScriptLogAndScriptLogLineExecutionLogger.class);
+
+   private List<QRecord>       scriptLogs     = new ArrayList<>();
+   private List<List<QRecord>> scriptLogLines = new ArrayList<>();
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public AccumulatingBuildScriptLogAndScriptLogLineExecutionLogger()
+   {
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public void acceptExecutionStart(ExecuteCodeInput executeCodeInput)
+   {
+      super.acceptExecutionStart(executeCodeInput);
+      super.setScriptLogLines(new ArrayList<>());
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public void acceptException(Exception exception)
+   {
+      accumulate(null, exception);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public void acceptExecutionEnd(Serializable output)
+   {
+      accumulate(output, null);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private void accumulate(Serializable output, Exception exception)
+   {
+      super.updateHeaderAtEnd(output, exception);
+      scriptLogs.add(super.getScriptLog());
+      scriptLogLines.add(new ArrayList<>(super.getScriptLogLines()));
+      super.getScriptLogLines().clear();
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public void storeAndClear()
+   {
+      try
+      {
+         InsertInput insertInput = new InsertInput();
+         insertInput.setTableName("scriptLog");
+         insertInput.setRecords(scriptLogs);
+         InsertOutput insertOutput = new InsertAction().execute(insertInput);
+
+         List<QRecord> flatScriptLogLines = new ArrayList<>();
+         for(int i = 0; i < insertOutput.getRecords().size(); i++)
+         {
+            QRecord       insertedScriptLog = insertOutput.getRecords().get(i);
+            List<QRecord> subScriptLogLines = scriptLogLines.get(i);
+            subScriptLogLines.forEach(r -> r.setValue("scriptLogId", insertedScriptLog.getValueInteger("id")));
+            flatScriptLogLines.addAll(subScriptLogLines);
+         }
+
+         insertInput = new InsertInput();
+         insertInput.setTableName("scriptLogLine");
+         insertInput.setRecords(flatScriptLogLines);
+         new InsertAction().execute(insertInput);
+      }
+      catch(Exception e)
+      {
+         LOG.warn("Error storing script logs", e);
+      }
+
+      scriptLogs.clear();
+      scriptLogLines.clear();
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public void setScriptId(Integer scriptId)
+   {
+      super.setScriptId(scriptId);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public void setScriptRevisionId(Integer scriptRevisionId)
+   {
+      super.setScriptRevisionId(scriptRevisionId);
+   }
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/scripts/logging/BuildScriptLogAndScriptLogLineExecutionLogger.java

@@ -0,0 +1,253 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.actions.scripts.logging;
+
+
+import java.io.Serializable;
+import java.time.Instant;
+import java.time.temporal.ChronoUnit;
+import java.util.ArrayList;
+import java.util.List;
+import com.kingsrook.qqq.backend.core.logging.QLogger;
+import com.kingsrook.qqq.backend.core.model.actions.scripts.ExecuteCodeInput;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.utils.StringUtils;
+import com.kingsrook.qqq.backend.core.utils.ValueUtils;
+
+
+/*******************************************************************************
+ ** Implementation of a code execution logger that builds a scriptLog and 0 or more
+ ** scriptLogLine records - but doesn't insert them.  e.g., useful for testing
+ ** (both in junit, and for users in-app).
+ *******************************************************************************/
+public class BuildScriptLogAndScriptLogLineExecutionLogger implements QCodeExecutionLoggerInterface, ScriptExecutionLoggerInterface
+{
+   private static final QLogger LOG = QLogger.getLogger(BuildScriptLogAndScriptLogLineExecutionLogger.class);
+
+   private   QRecord          scriptLog;
+   private   List<QRecord>    scriptLogLines = new ArrayList<>();
+   protected ExecuteCodeInput executeCodeInput;
+
+   private Serializable scriptId;
+   private Serializable scriptRevisionId;
+
+
+
+   /*******************************************************************************
+    ** Constructor
+    **
+    *******************************************************************************/
+   public BuildScriptLogAndScriptLogLineExecutionLogger()
+   {
+   }
+
+
+
+   /*******************************************************************************
+    ** Constructor
+    **
+    *******************************************************************************/
+   public BuildScriptLogAndScriptLogLineExecutionLogger(Serializable scriptId, Serializable scriptRevisionId)
+   {
+      this.scriptId = scriptId;
+      this.scriptRevisionId = scriptRevisionId;
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private QRecord buildHeaderRecord(ExecuteCodeInput executeCodeInput)
+   {
+      return (new QRecord()
+         .withValue("scriptId", scriptId)
+         .withValue("scriptRevisionId", scriptRevisionId)
+         .withValue("startTimestamp", Instant.now())
+         .withValue("input", truncate(executeCodeInput.getInput())));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   protected QRecord buildDetailLogRecord(String logLine)
+   {
+      return (new QRecord()
+         .withValue("scriptLogId", scriptLog.getValue("id"))
+         .withValue("timestamp", Instant.now())
+         .withValue("text", truncate(logLine)));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private String truncate(Object o)
+   {
+      return StringUtils.safeTruncate(ValueUtils.getValueAsString(o), 1000, "...");
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   protected void updateHeaderAtEnd(Serializable output, Exception exception)
+   {
+      Instant startTimestamp = (Instant) scriptLog.getValue("startTimestamp");
+      Instant endTimestamp   = Instant.now();
+      scriptLog.setValue("endTimestamp", endTimestamp);
+      scriptLog.setValue("runTimeMillis", startTimestamp.until(endTimestamp, ChronoUnit.MILLIS));
+
+      if(exception != null)
+      {
+         scriptLog.setValue("hadError", true);
+         scriptLog.setValue("error", exception.getMessage());
+      }
+      else
+      {
+         scriptLog.setValue("hadError", false);
+         scriptLog.setValue("output", truncate(output));
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public void acceptExecutionStart(ExecuteCodeInput executeCodeInput)
+   {
+      try
+      {
+         this.executeCodeInput = executeCodeInput;
+         this.scriptLog = buildHeaderRecord(executeCodeInput);
+      }
+      catch(Exception e)
+      {
+         LOG.warn("Error starting storage of script log", e);
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public void acceptLogLine(String logLine)
+   {
+      scriptLogLines.add(buildDetailLogRecord(logLine));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public void acceptException(Exception exception)
+   {
+      updateHeaderAtEnd(null, exception);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public void acceptExecutionEnd(Serializable output)
+   {
+      updateHeaderAtEnd(output, null);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for scriptLog
+    **
+    *******************************************************************************/
+   public QRecord getScriptLog()
+   {
+      return scriptLog;
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for scriptLogLines
+    **
+    *******************************************************************************/
+   public List<QRecord> getScriptLogLines()
+   {
+      return scriptLogLines;
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for scriptLog
+    **
+    *******************************************************************************/
+   protected void setScriptLog(QRecord scriptLog)
+   {
+      this.scriptLog = scriptLog;
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for scriptLogLines
+    **
+    *******************************************************************************/
+   protected void setScriptLogLines(List<QRecord> scriptLogLines)
+   {
+      this.scriptLogLines = scriptLogLines;
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public void setScriptId(Integer scriptId)
+   {
+      this.scriptId = scriptId;
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public void setScriptRevisionId(Integer scriptRevisionId)
+   {
+      this.scriptRevisionId = scriptRevisionId;
+   }
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/scripts/logging/Log4jCodeExecutionLogger.java

@@ -0,0 +1,92 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.actions.scripts.logging;
+
+
+import java.io.Serializable;
+import java.util.UUID;
+import com.kingsrook.qqq.backend.core.logging.QLogger;
+import com.kingsrook.qqq.backend.core.model.actions.scripts.ExecuteCodeInput;
+import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
+import com.kingsrook.qqq.backend.core.utils.StringUtils;
+import com.kingsrook.qqq.backend.core.utils.ValueUtils;
+
+
+/*******************************************************************************
+ ** Implementation of a code execution logger that logs to LOG 4j
+ *******************************************************************************/
+public class Log4jCodeExecutionLogger implements QCodeExecutionLoggerInterface
+{
+   private static final QLogger LOG = QLogger.getLogger(Log4jCodeExecutionLogger.class);
+
+   private QCodeReference qCodeReference;
+   private String         uuid = UUID.randomUUID().toString();
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public void acceptExecutionStart(ExecuteCodeInput executeCodeInput)
+   {
+      this.qCodeReference = executeCodeInput.getCodeReference();
+
+      String inputString = StringUtils.safeTruncate(ValueUtils.getValueAsString(executeCodeInput.getInput()), 250, "...");
+      LOG.info("Starting script execution: " + qCodeReference.getName() + ", uuid: " + uuid + ", with input: " + inputString);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public void acceptLogLine(String logLine)
+   {
+      LOG.info("Script log: " + uuid + ": " + logLine);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public void acceptException(Exception exception)
+   {
+      LOG.info("Script Exception: " + uuid, exception);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public void acceptExecutionEnd(Serializable output)
+   {
+      String outputString = StringUtils.safeTruncate(ValueUtils.getValueAsString(output), 250, "...");
+      LOG.info("Finished script execution: " + qCodeReference.getName() + ", uuid: " + uuid + ", with output: " + outputString);
+   }
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/scripts/logging/NoopCodeExecutionLogger.java

@@ -19,61 +19,56 @@
  * along with this program.  If not, see <https://www.gnu.org/licenses/>.
  */
 
-package com.kingsrook.qqq.backend.core.modules.authentication.metadata;
+package com.kingsrook.qqq.backend.core.actions.scripts.logging;
 
 
-import com.kingsrook.qqq.backend.core.model.metadata.QAuthenticationType;
+import java.io.Serializable;
+import com.kingsrook.qqq.backend.core.model.actions.scripts.ExecuteCodeInput;
 
 
 /*******************************************************************************
- ** Meta-data to provide details of an Auth0 Authentication module
+ ** Implementation of a code execution logger that just noop's every action.
  *******************************************************************************/
-public class Auth0AuthenticationMetaData extends QAuthenticationMetaData
+public class NoopCodeExecutionLogger implements QCodeExecutionLoggerInterface
 {
-   private String baseUrl;
-
 
 
    /*******************************************************************************
-    ** Default Constructor.
-    *******************************************************************************/
-   public Auth0AuthenticationMetaData()
-   {
-      super();
-      setType(QAuthenticationType.AUTH_0);
-   }
-
-
-
-   /*******************************************************************************
-    ** Fluent setter, override to help fluent flows
-    *******************************************************************************/
-   public Auth0AuthenticationMetaData withBaseUrl(String baseUrl)
-   {
-      setBaseUrl(baseUrl);
-      return this;
-   }
-
-
-
-   /*******************************************************************************
-    ** Getter for baseUrl
     **
     *******************************************************************************/
-   public String getBaseUrl()
+   @Override
+   public void acceptExecutionStart(ExecuteCodeInput executeCodeInput)
    {
-      return baseUrl;
    }
 
 
 
    /*******************************************************************************
-    ** Setter for baseUrl
     **
     *******************************************************************************/
-   public void setBaseUrl(String baseUrl)
+   @Override
+   public void acceptLogLine(String logLine)
+   {
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public void acceptException(Exception exception)
+   {
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public void acceptExecutionEnd(Serializable output)
    {
-      this.baseUrl = baseUrl;
    }
 
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/scripts/logging/QCodeExecutionLoggerInterface.java

@@ -0,0 +1,64 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.actions.scripts.logging;
+
+
+import java.io.Serializable;
+import com.kingsrook.qqq.backend.core.model.actions.scripts.ExecuteCodeInput;
+
+
+/*******************************************************************************
+ ** Interface to provide logging functionality to QCodeExecution (e.g., scripts)
+ *******************************************************************************/
+public interface QCodeExecutionLoggerInterface
+{
+
+   /*******************************************************************************
+    ** Called when the execution starts - takes the execution's input object.
+    *******************************************************************************/
+   void acceptExecutionStart(ExecuteCodeInput executeCodeInput);
+
+   /*******************************************************************************
+    ** Called to log a line, a message.
+    *******************************************************************************/
+   void acceptLogLine(String logLine);
+
+   /*******************************************************************************
+    ** In case the loggerInterface object is provided to the script as context,
+    ** this method gives a clean interface for the script to log a line.
+    *******************************************************************************/
+   default void log(String message)
+   {
+      acceptLogLine(message);
+   }
+
+   /*******************************************************************************
+    ** Called if the script fails with an exception.
+    *******************************************************************************/
+   void acceptException(Exception exception);
+
+   /*******************************************************************************
+    ** Called if the script completes without exception.
+    *******************************************************************************/
+   void acceptExecutionEnd(Serializable output);
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/scripts/logging/ScriptExecutionLoggerInterface.java

@@ -0,0 +1,39 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2023.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.actions.scripts.logging;
+
+
+/*******************************************************************************
+ **
+ *******************************************************************************/
+public interface ScriptExecutionLoggerInterface
+{
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   void setScriptId(Integer scriptId);
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   void setScriptRevisionId(Integer scriptRevisionId);
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/scripts/logging/StoreScriptLogAndScriptLogLineExecutionLogger.java

@@ -0,0 +1,132 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.actions.scripts.logging;
+
+
+import java.io.Serializable;
+import java.util.List;
+import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
+import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction;
+import com.kingsrook.qqq.backend.core.logging.QLogger;
+import com.kingsrook.qqq.backend.core.model.actions.scripts.ExecuteCodeInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertOutput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
+import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
+
+
+/*******************************************************************************
+ ** Implementation of a code execution logger that logs into scriptLog and scriptLogLine
+ ** tables - e.g., as defined in ScriptMetaDataProvider.
+ *******************************************************************************/
+public class StoreScriptLogAndScriptLogLineExecutionLogger extends BuildScriptLogAndScriptLogLineExecutionLogger
+{
+   private static final QLogger LOG = QLogger.getLogger(StoreScriptLogAndScriptLogLineExecutionLogger.class);
+
+
+
+   /*******************************************************************************
+    ** Constructor
+    **
+    *******************************************************************************/
+   public StoreScriptLogAndScriptLogLineExecutionLogger(Serializable scriptId, Serializable scriptRevisionId)
+   {
+      super(scriptId, scriptRevisionId);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public void acceptExecutionStart(ExecuteCodeInput executeCodeInput)
+   {
+      try
+      {
+         super.acceptExecutionStart(executeCodeInput);
+
+         InsertInput insertInput = new InsertInput();
+         insertInput.setTableName("scriptLog");
+         insertInput.setRecords(List.of(getScriptLog()));
+         InsertOutput insertOutput = new InsertAction().execute(insertInput);
+
+         setScriptLog(insertOutput.getRecords().get(0));
+      }
+      catch(Exception e)
+      {
+         LOG.warn("Error starting storage of script log", e);
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public void acceptException(Exception exception)
+   {
+      store(null, exception);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public void acceptExecutionEnd(Serializable output)
+   {
+      store(output, null);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private void store(Serializable output, Exception exception)
+   {
+      try
+      {
+         updateHeaderAtEnd(output, exception);
+         UpdateInput updateInput = new UpdateInput();
+         updateInput.setTableName("scriptLog");
+         updateInput.setRecords(List.of(getScriptLog()));
+         new UpdateAction().execute(updateInput);
+
+         if(CollectionUtils.nullSafeHasContents(getScriptLogLines()))
+         {
+            InsertInput insertInput = new InsertInput();
+            insertInput.setTableName("scriptLogLine");
+            insertInput.setRecords(getScriptLogLines());
+            new InsertAction().execute(insertInput);
+         }
+      }
+      catch(Exception e)
+      {
+         LOG.warn("Error storing script log", e);
+      }
+   }
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/AggregateAction.java

@@ -0,0 +1,53 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.actions.tables;
+
+
+import com.kingsrook.qqq.backend.core.actions.ActionHelper;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateOutput;
+import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleDispatcher;
+import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleInterface;
+
+
+/*******************************************************************************
+ ** Action to run an aggregate against a table.
+ **
+ *******************************************************************************/
+public class AggregateAction
+{
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public AggregateOutput execute(AggregateInput aggregateInput) throws QException
+   {
+      ActionHelper.validateSession(aggregateInput);
+
+      QBackendModuleDispatcher qBackendModuleDispatcher = new QBackendModuleDispatcher();
+      QBackendModuleInterface  qModule                  = qBackendModuleDispatcher.getQBackendModule(aggregateInput.getBackend());
+      // todo pre-customization - just get to modify the request?
+      AggregateOutput aggregateOutput = qModule.getAggregateInterface().execute(aggregateInput);
+      // todo post-customization - can do whatever w/ the result if you want
+      return aggregateOutput;
+   }
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/DeleteAction.java

@@ -24,19 +24,35 @@ package com.kingsrook.qqq.backend.core.actions.tables;
 
 import java.io.Serializable;
 import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
 import java.util.List;
+import java.util.Map;
+import java.util.Set;
 import com.kingsrook.qqq.backend.core.actions.ActionHelper;
+import com.kingsrook.qqq.backend.core.actions.audits.DMLAuditAction;
 import com.kingsrook.qqq.backend.core.actions.interfaces.DeleteInterface;
+import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.logging.QLogger;
+import com.kingsrook.qqq.backend.core.model.actions.audits.DMLAuditInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteOutput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.metadata.audits.AuditLevel;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.joins.QJoinMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.Association;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleDispatcher;
 import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleInterface;
 import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
+import com.kingsrook.qqq.backend.core.utils.ValueUtils;
 
 
 /*******************************************************************************
@@ -45,7 +61,9 @@ import org.apache.logging.log4j.Logger;
  *******************************************************************************/
 public class DeleteAction
 {
-   private static final Logger LOG = LogManager.getLogger(DeleteAction.class);
+   private static final QLogger LOG = QLogger.getLogger(DeleteAction.class);
+
+   public static final String NOT_FOUND_ERROR_PREFIX = "No record was found to delete";
 
 
 
@@ -58,7 +76,6 @@ public class DeleteAction
 
       QBackendModuleDispatcher qBackendModuleDispatcher = new QBackendModuleDispatcher();
       QBackendModuleInterface  qModule                  = qBackendModuleDispatcher.getQBackendModule(deleteInput.getBackend());
-      // todo pre-customization - just get to modify the request?
 
       if(CollectionUtils.nullSafeHasContents(deleteInput.getPrimaryKeys()) && deleteInput.getQueryFilter() != null)
       {
@@ -82,10 +99,186 @@ public class DeleteAction
          }
       }
 
-      DeleteOutput deleteResult = deleteInterface.execute(deleteInput);
-      // todo post-customization - can do whatever w/ the result if you want
+      List<QRecord> recordListForAudit          = getRecordListForAuditIfNeeded(deleteInput);
+      List<QRecord> recordsWithValidationErrors = validateRecordsExistAndCanBeAccessed(deleteInput, recordListForAudit);
 
-      return deleteResult;
+      DeleteOutput deleteOutput = deleteInterface.execute(deleteInput);
+
+      /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      // merge the backend's output with any validation errors we found (whose ids wouldn't have gotten into the backend delete) //
+      /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      List<QRecord> outputRecordsWithErrors = deleteOutput.getRecordsWithErrors();
+      if(outputRecordsWithErrors == null)
+      {
+         deleteOutput.setRecordsWithErrors(new ArrayList<>());
+         outputRecordsWithErrors = deleteOutput.getRecordsWithErrors();
+      }
+
+      outputRecordsWithErrors.addAll(recordsWithValidationErrors);
+
+      manageAssociations(deleteInput);
+
+      new DMLAuditAction().execute(new DMLAuditInput().withTableActionInput(deleteInput).withRecordList(recordListForAudit));
+
+      return deleteOutput;
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private void manageAssociations(DeleteInput deleteInput) throws QException
+   {
+      QTableMetaData table = deleteInput.getTable();
+      for(Association association : CollectionUtils.nonNullList(table.getAssociations()))
+      {
+         // e.g., order -> orderLine
+         QJoinMetaData join = QContext.getQInstance().getJoin(association.getJoinName()); // todo ... ever need to flip?
+         // just assume this, at least for now... if(BooleanUtils.isTrue(association.getDoInserts()))
+
+         QQueryFilter filter = new QQueryFilter();
+
+         if(join.getJoinOns().size() == 1 && join.getJoinOns().get(0).getLeftField().equals(table.getPrimaryKeyField()))
+         {
+            filter.addCriteria(new QFilterCriteria(join.getJoinOns().get(0).getRightField(), QCriteriaOperator.IN, deleteInput.getPrimaryKeys()));
+         }
+         else
+         {
+            throw (new QException("Join of this type is not supported for an associated delete at this time..."));
+         }
+
+         QTableMetaData associatedTable = QContext.getQInstance().getTable(association.getAssociatedTableName());
+
+         QueryInput queryInput = new QueryInput();
+         queryInput.setTransaction(deleteInput.getTransaction());
+         queryInput.setTableName(association.getAssociatedTableName());
+         queryInput.setFilter(filter);
+         QueryOutput        queryOutput    = new QueryAction().execute(queryInput);
+         List<Serializable> associatedKeys = queryOutput.getRecords().stream().map(r -> r.getValue(associatedTable.getPrimaryKeyField())).toList();
+
+         DeleteInput nextLevelDeleteInput = new DeleteInput();
+         nextLevelDeleteInput.setTransaction(deleteInput.getTransaction());
+         nextLevelDeleteInput.setTableName(association.getAssociatedTableName());
+         nextLevelDeleteInput.setPrimaryKeys(associatedKeys);
+         DeleteOutput nextLevelDeleteOutput = new DeleteAction().execute(nextLevelDeleteInput);
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private static List<QRecord> getRecordListForAuditIfNeeded(DeleteInput deleteInput) throws QException
+   {
+      List<QRecord> recordListForAudit = null;
+
+      AuditLevel auditLevel = DMLAuditAction.getAuditLevel(deleteInput);
+      if(AuditLevel.RECORD.equals(auditLevel) || AuditLevel.FIELD.equals(auditLevel))
+      {
+         List<Serializable> primaryKeyList = deleteInput.getPrimaryKeys();
+         if(CollectionUtils.nullSafeIsEmpty(deleteInput.getPrimaryKeys()) && deleteInput.getQueryFilter() != null)
+         {
+            primaryKeyList = getPrimaryKeysFromQueryFilter(deleteInput);
+         }
+
+         if(CollectionUtils.nullSafeHasContents(primaryKeyList))
+         {
+            ////////////////////////////////////////////////////////////////////////////////////
+            // always fetch the records - we'll use them anyway for checking not-exist  below //
+            ////////////////////////////////////////////////////////////////////////////////////
+            QueryInput queryInput = new QueryInput();
+            queryInput.setTransaction(deleteInput.getTransaction());
+            queryInput.setTableName(deleteInput.getTableName());
+            queryInput.setFilter(new QQueryFilter(new QFilterCriteria(deleteInput.getTable().getPrimaryKeyField(), QCriteriaOperator.IN, primaryKeyList)));
+            QueryOutput queryOutput = new QueryAction().execute(queryInput);
+            recordListForAudit = queryOutput.getRecords();
+         }
+      }
+
+      return (recordListForAudit);
+   }
+
+
+
+   /*******************************************************************************
+    ** Note - the "can be accessed" part of this method name - it implies that
+    ** records that you can't see because of security - that they won't be found
+    ** by the query here, so it's the same to you as if they don't exist at all!
+    **
+    ** This method, if it finds any missing records, will:
+    ** - remove those ids from the deleteInput
+    ** - create a QRecord with that id and a not-found error message.
+    *******************************************************************************/
+   private List<QRecord> validateRecordsExistAndCanBeAccessed(DeleteInput deleteInput, List<QRecord> oldRecordList) throws QException
+   {
+      List<QRecord> recordsWithErrors = new ArrayList<>();
+
+      QTableMetaData table           = deleteInput.getTable();
+      QFieldMetaData primaryKeyField = table.getField(table.getPrimaryKeyField());
+
+      Set<Serializable> primaryKeysToRemoveFromInput = new HashSet<>();
+
+      List<List<Serializable>> pages = CollectionUtils.getPages(deleteInput.getPrimaryKeys(), 1000);
+      for(List<Serializable> page : pages)
+      {
+         List<Serializable> primaryKeysToLookup = new ArrayList<>();
+         for(Serializable primaryKeyValue : page)
+         {
+            if(primaryKeyValue != null)
+            {
+               primaryKeysToLookup.add(primaryKeyValue);
+            }
+         }
+
+         Map<Serializable, QRecord> lookedUpRecords = new HashMap<>();
+         if(CollectionUtils.nullSafeHasContents(oldRecordList))
+         {
+            for(QRecord record : oldRecordList)
+            {
+               Serializable primaryKeyValue = record.getValue(table.getPrimaryKeyField());
+               primaryKeyValue = ValueUtils.getValueAsFieldType(primaryKeyField.getType(), primaryKeyValue);
+               lookedUpRecords.put(primaryKeyValue, record);
+            }
+         }
+         else if(!primaryKeysToLookup.isEmpty())
+         {
+            QueryInput queryInput = new QueryInput();
+            queryInput.setTransaction(deleteInput.getTransaction());
+            queryInput.setTableName(table.getName());
+            queryInput.setFilter(new QQueryFilter(new QFilterCriteria(table.getPrimaryKeyField(), QCriteriaOperator.IN, primaryKeysToLookup)));
+            QueryOutput queryOutput = new QueryAction().execute(queryInput);
+            for(QRecord record : queryOutput.getRecords())
+            {
+               lookedUpRecords.put(record.getValue(table.getPrimaryKeyField()), record);
+            }
+         }
+
+         for(Serializable primaryKeyValue : page)
+         {
+            primaryKeyValue = ValueUtils.getValueAsFieldType(primaryKeyField.getType(), primaryKeyValue);
+            if(!lookedUpRecords.containsKey(primaryKeyValue))
+            {
+               QRecord recordWithError = new QRecord();
+               recordsWithErrors.add(recordWithError);
+               recordWithError.setValue(primaryKeyField.getName(), primaryKeyValue);
+               recordWithError.addError(NOT_FOUND_ERROR_PREFIX + " for " + primaryKeyField.getLabel() + " = " + primaryKeyValue);
+               primaryKeysToRemoveFromInput.add(primaryKeyValue);
+            }
+         }
+
+         /////////////////////////////////////////////////////////////////
+         // do one mass removal of any bad keys from the input key list //
+         /////////////////////////////////////////////////////////////////
+         if(!primaryKeysToRemoveFromInput.isEmpty())
+         {
+            deleteInput.getPrimaryKeys().removeAll(primaryKeysToRemoveFromInput);
+            primaryKeysToRemoveFromInput.clear();
+         }
+      }
+
+      return (recordsWithErrors);
    }
 
 
@@ -102,7 +295,8 @@ public class DeleteAction
          QBackendModuleDispatcher qBackendModuleDispatcher = new QBackendModuleDispatcher();
          QBackendModuleInterface  qModule                  = qBackendModuleDispatcher.getQBackendModule(deleteInput.getBackend());
 
-         QueryInput queryInput = new QueryInput(deleteInput.getInstance(), deleteInput.getSession());
+         QueryInput queryInput = new QueryInput();
+         queryInput.setTransaction(deleteInput.getTransaction());
          queryInput.setTableName(deleteInput.getTableName());
          queryInput.setFilter(deleteInput.getQueryFilter());
          QueryOutput queryOutput = qModule.getQueryInterface().execute(queryInput);

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/GetAction.java

@@ -0,0 +1,381 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.actions.tables;
+
+
+import java.io.Serializable;
+import java.time.Instant;
+import java.time.temporal.ChronoUnit;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import com.kingsrook.qqq.backend.core.actions.ActionHelper;
+import com.kingsrook.qqq.backend.core.actions.customizers.AbstractPostQueryCustomizer;
+import com.kingsrook.qqq.backend.core.actions.customizers.QCodeLoader;
+import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizers;
+import com.kingsrook.qqq.backend.core.actions.interfaces.GetInterface;
+import com.kingsrook.qqq.backend.core.actions.values.QPossibleValueTranslator;
+import com.kingsrook.qqq.backend.core.actions.values.QValueFormatter;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.logging.LogPair;
+import com.kingsrook.qqq.backend.core.logging.QLogger;
+import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetOutput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertOutput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateOutput;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.cache.CacheUseCase;
+import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleDispatcher;
+import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleInterface;
+import com.kingsrook.qqq.backend.core.modules.backend.implementations.utils.BackendQueryFilterUtils;
+import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
+import com.kingsrook.qqq.backend.core.utils.StringUtils;
+import org.apache.commons.lang.NotImplementedException;
+
+
+/*******************************************************************************
+ ** Action to run a get against a table.
+ **
+ *******************************************************************************/
+public class GetAction
+{
+   private static final QLogger LOG = QLogger.getLogger(InsertAction.class);
+
+   private Optional<AbstractPostQueryCustomizer> postGetRecordCustomizer;
+
+   private GetInput                 getInput;
+   private QPossibleValueTranslator qPossibleValueTranslator;
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public GetOutput execute(GetInput getInput) throws QException
+   {
+      ActionHelper.validateSession(getInput);
+
+      QTableMetaData table = getInput.getTable();
+      if(table == null)
+      {
+         throw (new QException("Requested to Get a record from an unrecognized table: " + getInput.getTableName()));
+      }
+
+      postGetRecordCustomizer = QCodeLoader.getTableCustomizer(AbstractPostQueryCustomizer.class, table, TableCustomizers.POST_QUERY_RECORD.getRole());
+      this.getInput = getInput;
+
+      QBackendModuleDispatcher qBackendModuleDispatcher = new QBackendModuleDispatcher();
+      QBackendModuleInterface  qModule                  = qBackendModuleDispatcher.getQBackendModule(getInput.getBackend());
+      // todo pre-customization - just get to modify the request?
+
+      GetInterface getInterface = null;
+      try
+      {
+         getInterface = qModule.getGetInterface();
+      }
+      catch(IllegalStateException ise)
+      {
+         //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+         // if a module doesn't implement Get directly - try to do a Get by a Query in the DefaultGetInterface (inner class) //
+         //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      }
+
+      GetOutput getOutput;
+      if(getInterface == null)
+      {
+         getInterface = new DefaultGetInterface();
+      }
+
+      getInterface.validateInput(getInput);
+      getOutput = getInterface.execute(getInput);
+
+      ////////////////////////////
+      // handle cache use-cases //
+      ////////////////////////////
+      if(table.getCacheOf() != null)
+      {
+         if(getOutput.getRecord() == null)
+         {
+            ///////////////////////////////////////////////////////////////////////
+            // if the record wasn't found, see if we should look in cache-source //
+            ///////////////////////////////////////////////////////////////////////
+            QRecord recordFromSource = tryToGetFromCacheSource(getInput, getOutput);
+            if(recordFromSource != null)
+            {
+               QRecord recordToCache     = mapSourceRecordToCacheRecord(table, recordFromSource);
+               boolean shouldCacheRecord = true;
+
+               ////////////////////////////////////////////////////////////////////////////////
+               // see if there are any exclustions that need to be considered for this table //
+               ////////////////////////////////////////////////////////////////////////////////
+               recordMatchExclusionLoop:
+               for(CacheUseCase useCase : CollectionUtils.nonNullList(table.getCacheOf().getUseCases()))
+               {
+                  for(QQueryFilter filter : CollectionUtils.nonNullList(useCase.getExcludeRecordsMatching()))
+                  {
+                     if(BackendQueryFilterUtils.doesRecordMatch(filter, recordToCache))
+                     {
+                        LOG.info("Not caching record because it matches a use case's filter exclusion", new LogPair("record", recordToCache), new LogPair("filter", filter));
+                        shouldCacheRecord = false;
+                        break recordMatchExclusionLoop;
+                     }
+                  }
+               }
+
+               if(shouldCacheRecord)
+               {
+                  InsertInput insertInput = new InsertInput();
+                  insertInput.setTableName(getInput.getTableName());
+                  insertInput.setRecords(List.of(recordToCache));
+                  InsertOutput insertOutput = new InsertAction().execute(insertInput);
+                  getOutput.setRecord(insertOutput.getRecords().get(0));
+               }
+            }
+         }
+         else
+         {
+            /////////////////////////////////////////////////////////////////////////////////
+            // if the record was found, but it's too old, maybe re-fetch from cache source //
+            /////////////////////////////////////////////////////////////////////////////////
+            refreshCacheIfExpired(getInput, getOutput);
+         }
+      }
+
+      ////////////////////////////////////////////////////////
+      // if the record is found, perform post-actions on it //
+      ////////////////////////////////////////////////////////
+      if(getOutput.getRecord() != null)
+      {
+         getOutput.setRecord(postRecordActions(getOutput.getRecord()));
+      }
+
+      return getOutput;
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private static QRecord mapSourceRecordToCacheRecord(QTableMetaData table, QRecord recordFromSource)
+   {
+      QRecord cacheRecord = new QRecord(recordFromSource);
+      if(StringUtils.hasContent(table.getCacheOf().getCachedDateFieldName()))
+      {
+         cacheRecord.setValue(table.getCacheOf().getCachedDateFieldName(), Instant.now());
+      }
+      return (cacheRecord);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private void refreshCacheIfExpired(GetInput getInput, GetOutput getOutput) throws QException
+   {
+      QTableMetaData table             = getInput.getTable();
+      Integer        expirationSeconds = table.getCacheOf().getExpirationSeconds();
+      if(expirationSeconds != null)
+      {
+         QRecord cachedRecord = getOutput.getRecord();
+         Instant cachedDate   = cachedRecord.getValueInstant(table.getCacheOf().getCachedDateFieldName());
+         if(cachedDate == null || cachedDate.isBefore(Instant.now().minus(expirationSeconds, ChronoUnit.SECONDS)))
+         {
+            QRecord recordFromSource = tryToGetFromCacheSource(getInput, getOutput);
+            if(recordFromSource != null)
+            {
+               ///////////////////////////////////////////////////////////////////
+               // if the record was found in the source, update it in the cache //
+               ///////////////////////////////////////////////////////////////////
+               QRecord recordToCache = mapSourceRecordToCacheRecord(table, recordFromSource);
+               recordToCache.setValue(table.getPrimaryKeyField(), cachedRecord.getValue(table.getPrimaryKeyField()));
+
+               UpdateInput updateInput = new UpdateInput();
+               updateInput.setTableName(getInput.getTableName());
+               updateInput.setRecords(List.of(recordToCache));
+               UpdateOutput updateOutput = new UpdateAction().execute(updateInput);
+
+               getOutput.setRecord(updateOutput.getRecords().get(0));
+            }
+            else
+            {
+               /////////////////////////////////////////////////////////////////////////////
+               // if the record is no longer in the source, then remove it from the cache //
+               /////////////////////////////////////////////////////////////////////////////
+               DeleteInput deleteInput = new DeleteInput();
+               deleteInput.setTableName(getInput.getTableName());
+               deleteInput.setPrimaryKeys(List.of(getOutput.getRecord().getValue(table.getPrimaryKeyField())));
+               new DeleteAction().execute(deleteInput);
+
+               getOutput.setRecord(null);
+            }
+         }
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private QRecord tryToGetFromCacheSource(GetInput getInput, GetOutput getOutput) throws QException
+   {
+      QRecord        recordFromSource = null;
+      QTableMetaData table            = getInput.getTable();
+
+      for(CacheUseCase cacheUseCase : CollectionUtils.nonNullList(table.getCacheOf().getUseCases()))
+      {
+         if(CacheUseCase.Type.UNIQUE_KEY_TO_UNIQUE_KEY.equals(cacheUseCase.getType()) && getInput.getUniqueKey() != null)
+         {
+            recordFromSource = getFromCachedSourceForUniqueKeyToUniqueKey(getInput, table.getCacheOf().getSourceTable());
+            break;
+         }
+         else
+         {
+            // todo!!
+            throw new NotImplementedException("Not-yet-implemented cache use case type: " + cacheUseCase.getType());
+         }
+      }
+
+      return (recordFromSource);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private QRecord getFromCachedSourceForUniqueKeyToUniqueKey(GetInput getInput, String sourceTableName) throws QException
+   {
+      /////////////////////////////////////////////////////
+      // do a Get on the source table, by the unique key //
+      /////////////////////////////////////////////////////
+      GetInput sourceGetInput = new GetInput();
+      sourceGetInput.setTableName(sourceTableName);
+      sourceGetInput.setUniqueKey(getInput.getUniqueKey());
+      GetOutput sourceGetOutput = new GetAction().execute(sourceGetInput);
+      QRecord   outputRecord    = sourceGetOutput.getRecord();
+
+      return (outputRecord);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private static class DefaultGetInterface implements GetInterface
+   {
+      @Override
+      public GetOutput execute(GetInput getInput) throws QException
+      {
+         QueryInput queryInput = new QueryInput();
+         queryInput.setTableName(getInput.getTableName());
+
+         //////////////////////////////////////////////////
+         // build filter using either pkey or unique key //
+         //////////////////////////////////////////////////
+         QQueryFilter filter = new QQueryFilter();
+         if(getInput.getPrimaryKey() != null)
+         {
+            filter.addCriteria(new QFilterCriteria(getInput.getTable().getPrimaryKeyField(), QCriteriaOperator.EQUALS, getInput.getPrimaryKey()));
+         }
+         else if(getInput.getUniqueKey() != null)
+         {
+            for(Map.Entry<String, Serializable> entry : getInput.getUniqueKey().entrySet())
+            {
+               if(entry.getValue() == null)
+               {
+                  filter.addCriteria(new QFilterCriteria(entry.getKey(), QCriteriaOperator.IS_BLANK));
+               }
+               else
+               {
+                  filter.addCriteria(new QFilterCriteria(entry.getKey(), QCriteriaOperator.EQUALS, entry.getValue()));
+               }
+            }
+         }
+         else
+         {
+            throw (new QException("No primaryKey or uniqueKey was passed to Get"));
+         }
+
+         queryInput.setFilter(filter);
+         queryInput.setIncludeAssociations(getInput.getIncludeAssociations());
+         queryInput.setAssociationNamesToInclude(getInput.getAssociationNamesToInclude());
+         queryInput.setShouldFetchHeavyFields(getInput.getShouldFetchHeavyFields());
+
+         QueryOutput queryOutput = new QueryAction().execute(queryInput);
+
+         GetOutput getOutput = new GetOutput();
+         if(!queryOutput.getRecords().isEmpty())
+         {
+            getOutput.setRecord(queryOutput.getRecords().get(0));
+         }
+         return (getOutput);
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** Run the necessary actions on a record.  This may include setting display values,
+    ** translating possible values, and running post-record customizations.
+    *******************************************************************************/
+   public QRecord postRecordActions(QRecord record)
+   {
+      QRecord returnRecord = record;
+      if(this.postGetRecordCustomizer.isPresent())
+      {
+         returnRecord = postGetRecordCustomizer.get().apply(List.of(record)).get(0);
+      }
+
+      if(getInput.getShouldTranslatePossibleValues())
+      {
+         if(qPossibleValueTranslator == null)
+         {
+            qPossibleValueTranslator = new QPossibleValueTranslator(getInput.getInstance(), getInput.getSession());
+         }
+         qPossibleValueTranslator.translatePossibleValuesInRecords(getInput.getTable(), List.of(returnRecord));
+      }
+
+      if(getInput.getShouldGenerateDisplayValues())
+      {
+         QValueFormatter.setDisplayValuesInRecords(getInput.getTable(), List.of(returnRecord));
+      }
+
+      //////////////////////////////////////////////////////////////////////////////
+      // note - shouldFetchHeavyFields should be handled by the underlying action //
+      //////////////////////////////////////////////////////////////////////////////
+
+      return (returnRecord);
+   }
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/InsertAction.java

@@ -22,41 +22,249 @@
 package com.kingsrook.qqq.backend.core.actions.tables;
 
 
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.stream.Collectors;
+import com.kingsrook.qqq.backend.core.actions.AbstractQActionFunction;
 import com.kingsrook.qqq.backend.core.actions.ActionHelper;
 import com.kingsrook.qqq.backend.core.actions.QBackendTransaction;
+import com.kingsrook.qqq.backend.core.actions.audits.DMLAuditAction;
+import com.kingsrook.qqq.backend.core.actions.automation.AutomationStatus;
+import com.kingsrook.qqq.backend.core.actions.automation.RecordAutomationStatusUpdater;
+import com.kingsrook.qqq.backend.core.actions.customizers.AbstractPostInsertCustomizer;
+import com.kingsrook.qqq.backend.core.actions.customizers.QCodeLoader;
+import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizers;
+import com.kingsrook.qqq.backend.core.actions.tables.helpers.UniqueKeyHelper;
+import com.kingsrook.qqq.backend.core.actions.tables.helpers.ValidateRecordSecurityLockHelper;
+import com.kingsrook.qqq.backend.core.actions.values.ValueBehaviorApplier;
+import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.logging.QLogger;
+import com.kingsrook.qqq.backend.core.model.actions.audits.DMLAuditInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertOutput;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinOn;
+import com.kingsrook.qqq.backend.core.model.metadata.joins.QJoinMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.Association;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.UniqueKey;
 import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleDispatcher;
 import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleInterface;
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
+import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
+import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
 
 
 /*******************************************************************************
  ** Action to insert one or more records.
  **
  *******************************************************************************/
-public class InsertAction
+public class InsertAction extends AbstractQActionFunction<InsertInput, InsertOutput>
 {
-   private static final Logger LOG = LogManager.getLogger(InsertAction.class);
+   private static final QLogger LOG = QLogger.getLogger(InsertAction.class);
 
 
 
    /*******************************************************************************
     **
     *******************************************************************************/
+   @Override
    public InsertOutput execute(InsertInput insertInput) throws QException
    {
+      ActionHelper.validateSession(insertInput);
+      QTableMetaData table = insertInput.getTable();
+
+      if(table == null)
+      {
+         throw (new QException("Error:  Undefined table: " + insertInput.getTableName()));
+      }
+
+      Optional<AbstractPostInsertCustomizer> postInsertCustomizer = QCodeLoader.getTableCustomizer(AbstractPostInsertCustomizer.class, table, TableCustomizers.POST_INSERT_RECORD.getRole());
+      setAutomationStatusField(insertInput);
+
       QBackendModuleInterface qModule = getBackendModuleInterface(insertInput);
       // todo pre-customization - just get to modify the request?
+
+      ValueBehaviorApplier.applyFieldBehaviors(insertInput.getInstance(), table, insertInput.getRecords());
+      setErrorsIfUniqueKeyErrors(insertInput, table);
+      validateRequiredFields(insertInput);
+      ValidateRecordSecurityLockHelper.validateSecurityFields(insertInput.getTable(), insertInput.getRecords(), ValidateRecordSecurityLockHelper.Action.INSERT);
+
       InsertOutput insertOutput = qModule.getInsertInterface().execute(insertInput);
+      List<String> errors       = insertOutput.getRecords().stream().flatMap(r -> r.getErrors().stream()).toList();
+      if(CollectionUtils.nullSafeHasContents(errors))
+      {
+         LOG.warn("Errors in insertAction", logPair("tableName", table.getName()), logPair("errorCount", errors.size()), errors.size() < 10 ? logPair("errors", errors) : logPair("first10Errors", errors.subList(0, 10)));
+      }
+
+      manageAssociations(table, insertOutput.getRecords(), insertInput.getTransaction());
+
       // todo post-customization - can do whatever w/ the result if you want
+
+      if(insertInput.getOmitDmlAudit())
+      {
+         LOG.debug("Requested to omit DML audit");
+      }
+      else
+      {
+         new DMLAuditAction().execute(new DMLAuditInput().withTableActionInput(insertInput).withRecordList(insertOutput.getRecords()));
+      }
+
+      if(postInsertCustomizer.isPresent())
+      {
+         postInsertCustomizer.get().setInsertInput(insertInput);
+         insertOutput.setRecords(postInsertCustomizer.get().apply(insertOutput.getRecords()));
+      }
+
       return insertOutput;
    }
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private void validateRequiredFields(InsertInput insertInput)
+   {
+      QTableMetaData table = insertInput.getTable();
+      Set<QFieldMetaData> requiredFields = table.getFields().values().stream()
+         .filter(f -> f.getIsRequired())
+         .collect(Collectors.toSet());
+
+      if(!requiredFields.isEmpty())
+      {
+         for(QRecord record : insertInput.getRecords())
+         {
+            for(QFieldMetaData requiredField : requiredFields)
+            {
+               if(record.getValue(requiredField.getName()) == null || (requiredField.getType().isStringLike() && record.getValueString(requiredField.getName()).trim().equals("")))
+               {
+                  record.addError("Missing value in required field: " + requiredField.getLabel());
+               }
+            }
+         }
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private void manageAssociations(QTableMetaData table, List<QRecord> insertedRecords, QBackendTransaction transaction) throws QException
+   {
+      for(Association association : CollectionUtils.nonNullList(table.getAssociations()))
+      {
+         // e.g., order -> orderLine
+         QJoinMetaData join = QContext.getQInstance().getJoin(association.getJoinName()); // todo ... ever need to flip?
+         // just assume this, at least for now... if(BooleanUtils.isTrue(association.getDoInserts()))
+
+         List<QRecord> nextLevelInserts = new ArrayList<>();
+         for(QRecord record : insertedRecords)
+         {
+            if(record.getAssociatedRecords() != null && record.getAssociatedRecords().containsKey(association.getName()))
+            {
+               for(QRecord associatedRecord : CollectionUtils.nonNullList(record.getAssociatedRecords().get(association.getName())))
+               {
+                  for(JoinOn joinOn : join.getJoinOns())
+                  {
+                     associatedRecord.setValue(joinOn.getRightField(), record.getValue(joinOn.getLeftField()));
+                  }
+                  nextLevelInserts.add(associatedRecord);
+               }
+            }
+         }
+
+         InsertInput nextLevelInsertInput = new InsertInput();
+         nextLevelInsertInput.setTransaction(transaction);
+         nextLevelInsertInput.setTableName(association.getAssociatedTableName());
+         nextLevelInsertInput.setRecords(nextLevelInserts);
+         InsertOutput nextLevelInsertOutput = new InsertAction().execute(nextLevelInsertInput);
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private void setErrorsIfUniqueKeyErrors(InsertInput insertInput, QTableMetaData table) throws QException
+   {
+      if(CollectionUtils.nullSafeHasContents(table.getUniqueKeys()))
+      {
+         Map<UniqueKey, Set<List<Serializable>>> keysInThisList = new HashMap<>();
+         if(insertInput.getSkipUniqueKeyCheck())
+         {
+            LOG.debug("Skipping unique key check in " + insertInput.getTableName() + " insert.");
+            return;
+         }
+
+         ////////////////////////////////////////////
+         // check for any pre-existing unique keys //
+         ////////////////////////////////////////////
+         Map<UniqueKey, Set<List<Serializable>>> existingKeys = new HashMap<>();
+         List<UniqueKey>                         uniqueKeys   = CollectionUtils.nonNullList(table.getUniqueKeys());
+         for(UniqueKey uniqueKey : uniqueKeys)
+         {
+            existingKeys.put(uniqueKey, UniqueKeyHelper.getExistingKeys(insertInput.getTransaction(), table, insertInput.getRecords(), uniqueKey).keySet());
+         }
+
+         /////////////////////////////////////
+         // make sure this map is populated //
+         /////////////////////////////////////
+         uniqueKeys.forEach(uk -> keysInThisList.computeIfAbsent(uk, x -> new HashSet<>()));
+
+         for(QRecord record : insertInput.getRecords())
+         {
+            //////////////////////////////////////////////////////////
+            // check if this record violates any of the unique keys //
+            //////////////////////////////////////////////////////////
+            boolean foundDupe = false;
+            for(UniqueKey uniqueKey : uniqueKeys)
+            {
+               Optional<List<Serializable>> keyValues = UniqueKeyHelper.getKeyValues(table, uniqueKey, record);
+               if(keyValues.isPresent() && (existingKeys.get(uniqueKey).contains(keyValues.get()) || keysInThisList.get(uniqueKey).contains(keyValues.get())))
+               {
+                  record.addError("Another record already exists with this " + uniqueKey.getDescription(table));
+                  foundDupe = true;
+                  break;
+               }
+            }
+
+            ///////////////////////////////////////////////////////////////////////////////
+            // if this record doesn't violate any uk's, then we can add it to the output //
+            ///////////////////////////////////////////////////////////////////////////////
+            if(!foundDupe)
+            {
+               for(UniqueKey uniqueKey : uniqueKeys)
+               {
+                  Optional<List<Serializable>> keyValues = UniqueKeyHelper.getKeyValues(table, uniqueKey, record);
+                  keyValues.ifPresent(kv -> keysInThisList.get(uniqueKey).add(kv));
+               }
+            }
+         }
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** If the table being inserted into uses an automation-status field, populate it now.
+    *******************************************************************************/
+   private void setAutomationStatusField(InsertInput insertInput)
+   {
+      RecordAutomationStatusUpdater.setAutomationStatusInRecords(insertInput.getSession(), insertInput.getTable(), insertInput.getRecords(), AutomationStatus.PENDING_INSERT_AUTOMATIONS);
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/QueryAction.java

@@ -22,13 +22,37 @@
 package com.kingsrook.qqq.backend.core.actions.tables;
 
 
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Optional;
+import java.util.Set;
 import com.kingsrook.qqq.backend.core.actions.ActionHelper;
+import com.kingsrook.qqq.backend.core.actions.customizers.AbstractPostQueryCustomizer;
+import com.kingsrook.qqq.backend.core.actions.customizers.QCodeLoader;
+import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizers;
+import com.kingsrook.qqq.backend.core.actions.reporting.BufferedRecordPipe;
+import com.kingsrook.qqq.backend.core.actions.values.QPossibleValueTranslator;
 import com.kingsrook.qqq.backend.core.actions.values.QValueFormatter;
+import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.logging.QLogger;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinOn;
+import com.kingsrook.qqq.backend.core.model.metadata.joins.QJoinMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.Association;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleDispatcher;
 import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleInterface;
+import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
+import com.kingsrook.qqq.backend.core.utils.ListingHash;
 
 
 /*******************************************************************************
@@ -37,6 +61,15 @@ import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleInterface;
  *******************************************************************************/
 public class QueryAction
 {
+   private static final QLogger LOG = QLogger.getLogger(QueryAction.class);
+
+   private Optional<AbstractPostQueryCustomizer> postQueryRecordCustomizer;
+
+   private QueryInput               queryInput;
+   private QPossibleValueTranslator qPossibleValueTranslator;
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/
@@ -44,18 +77,175 @@ public class QueryAction
    {
       ActionHelper.validateSession(queryInput);
 
+      postQueryRecordCustomizer = QCodeLoader.getTableCustomizer(AbstractPostQueryCustomizer.class, queryInput.getTable(), TableCustomizers.POST_QUERY_RECORD.getRole());
+      this.queryInput = queryInput;
+
+      if(queryInput.getRecordPipe() != null)
+      {
+         queryInput.getRecordPipe().setPostRecordActions(this::postRecordActions);
+      }
+
+      if(queryInput.getIncludeAssociations() && queryInput.getRecordPipe() != null)
+      {
+         //////////////////////////////////////////////
+         // todo - support this in the future maybe? //
+         //////////////////////////////////////////////
+         throw (new QException("Associations may not be fetched into a RecordPipe."));
+      }
+
       QBackendModuleDispatcher qBackendModuleDispatcher = new QBackendModuleDispatcher();
-      QBackendModuleInterface qModule = qBackendModuleDispatcher.getQBackendModule(queryInput.getBackend());
+      QBackendModuleInterface  qModule                  = qBackendModuleDispatcher.getQBackendModule(queryInput.getBackend());
       // todo pre-customization - just get to modify the request?
       QueryOutput queryOutput = qModule.getQueryInterface().execute(queryInput);
       // todo post-customization - can do whatever w/ the result if you want
 
-      if (queryInput.getRecordPipe() == null)
+      if(queryInput.getRecordPipe() instanceof BufferedRecordPipe bufferedRecordPipe)
       {
-         QValueFormatter.setDisplayValuesInRecords(queryInput.getTable(), queryOutput.getRecords());
+         bufferedRecordPipe.finalFlush();
+      }
+
+      if(queryInput.getRecordPipe() == null)
+      {
+         postRecordActions(queryOutput.getRecords());
+      }
+
+      if(queryInput.getIncludeAssociations())
+      {
+         manageAssociations(queryInput, queryOutput);
       }
 
       return queryOutput;
    }
 
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private void manageAssociations(QueryInput queryInput, QueryOutput queryOutput) throws QException
+   {
+      QTableMetaData table = queryInput.getTable();
+      for(Association association : CollectionUtils.nonNullList(table.getAssociations()))
+      {
+         if(queryInput.getAssociationNamesToInclude() == null || queryInput.getAssociationNamesToInclude().contains(association.getName()))
+         {
+            // e.g., order -> orderLine
+            QJoinMetaData join = QContext.getQInstance().getJoin(association.getJoinName()); // todo ... ever need to flip?
+            // just assume this, at least for now... if(BooleanUtils.isTrue(association.getDoInserts()))
+
+            QueryInput nextLevelQueryInput = new QueryInput();
+            nextLevelQueryInput.setTableName(association.getAssociatedTableName());
+            nextLevelQueryInput.setIncludeAssociations(true);
+            nextLevelQueryInput.setAssociationNamesToInclude(buildNextLevelAssociationNamesToInclude(association.getName(), queryInput.getAssociationNamesToInclude()));
+
+            QQueryFilter filter = new QQueryFilter();
+            nextLevelQueryInput.setFilter(filter);
+
+            ListingHash<List<Serializable>, QRecord> outerResultMap = new ListingHash<>();
+
+            if(join.getJoinOns().size() == 1)
+            {
+               JoinOn            joinOn = join.getJoinOns().get(0);
+               Set<Serializable> values = new HashSet<>();
+               for(QRecord record : queryOutput.getRecords())
+               {
+                  Serializable value = record.getValue(joinOn.getLeftField());
+                  values.add(value);
+                  outerResultMap.add(List.of(value), record);
+               }
+               filter.addCriteria(new QFilterCriteria(joinOn.getRightField(), QCriteriaOperator.IN, new ArrayList<>(values)));
+            }
+            else
+            {
+               filter.setBooleanOperator(QQueryFilter.BooleanOperator.OR);
+
+               for(QRecord record : queryOutput.getRecords())
+               {
+                  QQueryFilter subFilter = new QQueryFilter();
+                  filter.addSubFilter(subFilter);
+                  List<Serializable> values = new ArrayList<>();
+                  for(JoinOn joinOn : join.getJoinOns())
+                  {
+                     Serializable value = record.getValue(joinOn.getLeftField());
+                     values.add(value);
+                     subFilter.addCriteria(new QFilterCriteria(joinOn.getRightField(), QCriteriaOperator.EQUALS, value));
+                  }
+                  outerResultMap.add(values, record);
+               }
+            }
+
+            QueryOutput nextLevelQueryOutput = new QueryAction().execute(nextLevelQueryInput);
+            for(QRecord record : nextLevelQueryOutput.getRecords())
+            {
+               List<Serializable> values = new ArrayList<>();
+               for(JoinOn joinOn : join.getJoinOns())
+               {
+                  Serializable value = record.getValue(joinOn.getRightField());
+                  values.add(value);
+               }
+
+               if(outerResultMap.containsKey(values))
+               {
+                  for(QRecord outerRecord : outerResultMap.get(values))
+                  {
+                     outerRecord.withAssociatedRecord(association.getName(), record);
+                  }
+               }
+            }
+         }
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private Collection<String> buildNextLevelAssociationNamesToInclude(String name, Collection<String> associationNamesToInclude)
+   {
+      if(associationNamesToInclude == null)
+      {
+         return (associationNamesToInclude);
+      }
+
+      Set<String> rs = new HashSet<>();
+      for(String nextLevelCandidateName : associationNamesToInclude)
+      {
+         if(nextLevelCandidateName.startsWith(name + "."))
+         {
+            rs.add(nextLevelCandidateName.replaceFirst(name + ".", ""));
+         }
+      }
+
+      return (rs);
+   }
+
+
+
+   /*******************************************************************************
+    ** Run the necessary actions on a list of records (which must be a mutable list - e.g.,
+    ** not one created via List.of()).  This may include setting display values,
+    ** translating possible values, and running post-record customizations.
+    *******************************************************************************/
+   public void postRecordActions(List<QRecord> records)
+   {
+      if(this.postQueryRecordCustomizer.isPresent())
+      {
+         records = postQueryRecordCustomizer.get().apply(records);
+      }
+
+      if(queryInput.getShouldTranslatePossibleValues())
+      {
+         if(qPossibleValueTranslator == null)
+         {
+            qPossibleValueTranslator = new QPossibleValueTranslator(queryInput.getInstance(), queryInput.getSession());
+         }
+         qPossibleValueTranslator.translatePossibleValuesInRecords(queryInput.getTable(), records, queryInput.getQueryJoins(), queryInput.getFieldsToTranslatePossibleValues());
+      }
+
+      if(queryInput.getShouldGenerateDisplayValues())
+      {
+         QValueFormatter.setDisplayValuesInRecords(queryInput.getTable(), records);
+      }
+   }
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/UpdateAction.java

@@ -22,12 +22,46 @@
 package com.kingsrook.qqq.backend.core.actions.tables;
 
 
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
 import com.kingsrook.qqq.backend.core.actions.ActionHelper;
+import com.kingsrook.qqq.backend.core.actions.audits.DMLAuditAction;
+import com.kingsrook.qqq.backend.core.actions.automation.AutomationStatus;
+import com.kingsrook.qqq.backend.core.actions.automation.RecordAutomationStatusUpdater;
+import com.kingsrook.qqq.backend.core.actions.tables.helpers.ValidateRecordSecurityLockHelper;
+import com.kingsrook.qqq.backend.core.actions.values.ValueBehaviorApplier;
+import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.logging.QLogger;
+import com.kingsrook.qqq.backend.core.model.actions.audits.DMLAuditInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteOutput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertOutput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateOutput;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.metadata.audits.AuditLevel;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinOn;
+import com.kingsrook.qqq.backend.core.model.metadata.joins.QJoinMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.Association;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleDispatcher;
 import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleInterface;
+import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
+import com.kingsrook.qqq.backend.core.utils.ValueUtils;
+import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
 
 
 /*******************************************************************************
@@ -36,18 +70,337 @@ import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleInterface;
  *******************************************************************************/
 public class UpdateAction
 {
+   private static final QLogger LOG = QLogger.getLogger(UpdateAction.class);
+
+   public static final String NOT_FOUND_ERROR_PREFIX = "No record was found to update";
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/
    public UpdateOutput execute(UpdateInput updateInput) throws QException
    {
       ActionHelper.validateSession(updateInput);
+      setAutomationStatusField(updateInput);
+
+      ValueBehaviorApplier.applyFieldBehaviors(updateInput.getInstance(), updateInput.getTable(), updateInput.getRecords());
+      // todo - need to handle records with errors coming out of here...
+
+      List<QRecord> oldRecordList = getOldRecordListForAuditIfNeeded(updateInput);
 
       QBackendModuleDispatcher qBackendModuleDispatcher = new QBackendModuleDispatcher();
-      QBackendModuleInterface qModule = qBackendModuleDispatcher.getQBackendModule(updateInput.getBackend());
+      QBackendModuleInterface  qModule                  = qBackendModuleDispatcher.getQBackendModule(updateInput.getBackend());
+
+      validatePrimaryKeysAreGiven(updateInput);
+      validateRecordsExistAndCanBeAccessed(updateInput, oldRecordList);
+      validateRequiredFields(updateInput);
+      ValidateRecordSecurityLockHelper.validateSecurityFields(updateInput.getTable(), updateInput.getRecords(), ValidateRecordSecurityLockHelper.Action.UPDATE);
+
       // todo pre-customization - just get to modify the request?
-      UpdateOutput updateResult = qModule.getUpdateInterface().execute(updateInput);
+      UpdateOutput updateOutput = qModule.getUpdateInterface().execute(updateInput);
       // todo post-customization - can do whatever w/ the result if you want
-      return updateResult;
+
+      List<String> errors = updateOutput.getRecords().stream().flatMap(r -> r.getErrors().stream()).toList();
+      if(CollectionUtils.nullSafeHasContents(errors))
+      {
+         LOG.warn("Errors in updateAction", logPair("tableName", updateInput.getTableName()), logPair("errorCount", errors.size()), errors.size() < 10 ? logPair("errors", errors) : logPair("first10Errors", errors.subList(0, 10)));
+      }
+
+      manageAssociations(updateInput);
+
+      if(updateInput.getOmitDmlAudit())
+      {
+         LOG.debug("Requested to omit DML audit");
+      }
+      else
+      {
+         new DMLAuditAction().execute(new DMLAuditInput().withTableActionInput(updateInput).withRecordList(updateOutput.getRecords()).withOldRecordList(oldRecordList));
+      }
+
+      return updateOutput;
    }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private void validatePrimaryKeysAreGiven(UpdateInput updateInput)
+   {
+      QTableMetaData table = updateInput.getTable();
+      for(QRecord record : CollectionUtils.nonNullList(updateInput.getRecords()))
+      {
+         ////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+         // to update a record, we must have its primary key value - so - check - if it's missing, mark it as an error //
+         ////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+         if(record.getValue(table.getPrimaryKeyField()) == null)
+         {
+            record.addError("Missing value in primary key field");
+         }
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** Note - the "can be accessed" part of this method name - it implies that
+    ** records that you can't see because of security - that they won't be found
+    ** by the query here, so it's the same to you as if they don't exist at all!
+    *******************************************************************************/
+   private void validateRecordsExistAndCanBeAccessed(UpdateInput updateInput, List<QRecord> oldRecordList) throws QException
+   {
+      QTableMetaData table           = updateInput.getTable();
+      QFieldMetaData primaryKeyField = table.getField(table.getPrimaryKeyField());
+
+      for(List<QRecord> page : CollectionUtils.getPages(updateInput.getRecords(), 1000))
+      {
+         List<Serializable> primaryKeysToLookup = new ArrayList<>();
+         for(QRecord record : page)
+         {
+            Serializable primaryKeyValue = record.getValue(table.getPrimaryKeyField());
+            if(primaryKeyValue != null)
+            {
+               primaryKeysToLookup.add(primaryKeyValue);
+            }
+         }
+
+         Map<Serializable, QRecord> lookedUpRecords = new HashMap<>();
+         if(CollectionUtils.nullSafeHasContents(oldRecordList))
+         {
+            for(QRecord record : oldRecordList)
+            {
+               lookedUpRecords.put(record.getValue(table.getPrimaryKeyField()), record);
+            }
+         }
+         else if(!primaryKeysToLookup.isEmpty())
+         {
+            QueryInput queryInput = new QueryInput();
+            queryInput.setTransaction(updateInput.getTransaction());
+            queryInput.setTableName(table.getName());
+            queryInput.setFilter(new QQueryFilter(new QFilterCriteria(table.getPrimaryKeyField(), QCriteriaOperator.IN, primaryKeysToLookup)));
+            QueryOutput queryOutput = new QueryAction().execute(queryInput);
+            for(QRecord record : queryOutput.getRecords())
+            {
+               lookedUpRecords.put(record.getValue(table.getPrimaryKeyField()), record);
+            }
+         }
+
+         for(QRecord record : page)
+         {
+            Serializable value = ValueUtils.getValueAsFieldType(primaryKeyField.getType(), record.getValue(table.getPrimaryKeyField()));
+            if(value == null)
+            {
+               continue;
+            }
+
+            if(!lookedUpRecords.containsKey(value))
+            {
+               record.addError(NOT_FOUND_ERROR_PREFIX + " for " + primaryKeyField.getLabel() + " = " + value);
+            }
+         }
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private void validateRequiredFields(UpdateInput updateInput)
+   {
+      QTableMetaData table = updateInput.getTable();
+      Set<QFieldMetaData> requiredFields = table.getFields().values().stream()
+         .filter(f -> f.getIsRequired())
+         .collect(Collectors.toSet());
+
+      if(!requiredFields.isEmpty())
+      {
+         for(QRecord record : CollectionUtils.nonNullList(updateInput.getRecords()))
+         {
+            for(QFieldMetaData requiredField : requiredFields)
+            {
+               /////////////////////////////////////////////////////////////////////////////////////////////
+               // only consider fields that were set in the record to be updated (e.g., "patch" semantic) //
+               /////////////////////////////////////////////////////////////////////////////////////////////
+               if(record.getValues().containsKey(requiredField.getName()))
+               {
+                  if(record.getValue(requiredField.getName()) == null || (requiredField.getType().isStringLike() && record.getValueString(requiredField.getName()).trim().equals("")))
+                  {
+                     record.addError("Missing value in required field: " + requiredField.getLabel());
+                  }
+               }
+            }
+         }
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private void manageAssociations(UpdateInput updateInput) throws QException
+   {
+      QTableMetaData table = updateInput.getTable();
+      for(Association association : CollectionUtils.nonNullList(table.getAssociations()))
+      {
+         // e.g., order -> orderLine
+         QTableMetaData associatedTable = QContext.getQInstance().getTable(association.getAssociatedTableName());
+         QJoinMetaData  join            = QContext.getQInstance().getJoin(association.getJoinName()); // todo ... ever need to flip?
+         // just assume this, at least for now... if(BooleanUtils.isTrue(association.getDoInserts()))
+
+         for(List<QRecord> page : CollectionUtils.getPages(updateInput.getRecords(), 500))
+         {
+            List<QRecord> nextLevelUpdates  = new ArrayList<>();
+            List<QRecord> nextLevelInserts  = new ArrayList<>();
+            QQueryFilter  findDeletesFilter = new QQueryFilter().withBooleanOperator(QQueryFilter.BooleanOperator.OR);
+            boolean       lookForDeletes    = false;
+
+            //////////////////////////////////////////////////////
+            // for each updated record, look at as associations //
+            //////////////////////////////////////////////////////
+            for(QRecord record : page)
+            {
+               if(record.getAssociatedRecords() != null && record.getAssociatedRecords().containsKey(association.getName()))
+               {
+                  ///////////////////////////////////////////////////////////////////////////////////////////////////////////////
+                  // build a sub-query to find the children of this record - and we'll exclude (below) any whose ids are given //
+                  ///////////////////////////////////////////////////////////////////////////////////////////////////////////////
+                  QQueryFilter subFilter = new QQueryFilter();
+                  findDeletesFilter.addSubFilter(subFilter);
+                  lookForDeletes = true;
+                  List<Serializable> idsBeingUpdated = new ArrayList<>();
+                  for(JoinOn joinOn : join.getJoinOns())
+                  {
+                     subFilter.addCriteria(new QFilterCriteria(joinOn.getRightField(), QCriteriaOperator.EQUALS, record.getValue(joinOn.getLeftField())));
+                  }
+
+                  //////////////////////////////////////////////////////////////////////////////////////////////////////////////
+                  // for any associated records present here, figure out if they're being inserted (no primaryKey) or updated //
+                  //////////////////////////////////////////////////////////////////////////////////////////////////////////////
+                  for(QRecord associatedRecord : CollectionUtils.nonNullList(record.getAssociatedRecords().get(association.getName())))
+                  {
+                     Serializable associatedId = associatedRecord.getValue(associatedTable.getPrimaryKeyField());
+                     if(associatedId == null)
+                     {
+                        //////////////////////////////////////////////////////////////////////////////////////////////////////////
+                        // if inserting, add to the inserts list, and propagate values from the header record down to the child //
+                        //////////////////////////////////////////////////////////////////////////////////////////////////////////
+                        for(JoinOn joinOn : join.getJoinOns())
+                        {
+                           associatedRecord.setValue(joinOn.getRightField(), record.getValue(joinOn.getLeftField()));
+                        }
+                        nextLevelInserts.add(associatedRecord);
+                     }
+                     else
+                     {
+                        ///////////////////////////////////////////////////////////////////////////////
+                        // if updating, add to the updates list, and add the id as one to not delete //
+                        ///////////////////////////////////////////////////////////////////////////////
+                        idsBeingUpdated.add(associatedId);
+                        nextLevelUpdates.add(associatedRecord);
+                     }
+                  }
+
+                  if(!idsBeingUpdated.isEmpty())
+                  {
+                     ///////////////////////////////////////////////////////////////////////////////
+                     // if any records are being updated, add them to the query to NOT be deleted //
+                     ///////////////////////////////////////////////////////////////////////////////
+                     subFilter.addCriteria(new QFilterCriteria(associatedTable.getPrimaryKeyField(), QCriteriaOperator.NOT_IN, idsBeingUpdated));
+                  }
+               }
+            }
+
+            if(lookForDeletes)
+            {
+               QueryInput queryInput = new QueryInput();
+               queryInput.setTransaction(updateInput.getTransaction());
+               queryInput.setTableName(associatedTable.getName());
+               queryInput.setFilter(findDeletesFilter);
+               QueryOutput queryOutput = new QueryAction().execute(queryInput);
+               if(!queryOutput.getRecords().isEmpty())
+               {
+                  LOG.debug("Deleting associatedRecords", logPair("associatedTable", associatedTable.getName()), logPair("noOfRecords", queryOutput.getRecords().size()));
+                  DeleteInput deleteInput = new DeleteInput();
+                  deleteInput.setTransaction(updateInput.getTransaction());
+                  deleteInput.setTableName(association.getAssociatedTableName());
+                  deleteInput.setPrimaryKeys(queryOutput.getRecords().stream().map(r -> r.getValue(associatedTable.getPrimaryKeyField())).collect(Collectors.toList()));
+                  DeleteOutput deleteOutput = new DeleteAction().execute(deleteInput);
+               }
+            }
+
+            if(CollectionUtils.nullSafeHasContents(nextLevelUpdates))
+            {
+               LOG.debug("Updating associatedRecords", logPair("associatedTable", associatedTable.getName()), logPair("noOfRecords", nextLevelUpdates.size()));
+               UpdateInput nextLevelUpdateInput = new UpdateInput();
+               nextLevelUpdateInput.setTransaction(updateInput.getTransaction());
+               nextLevelUpdateInput.setTableName(association.getAssociatedTableName());
+               nextLevelUpdateInput.setRecords(nextLevelUpdates);
+               UpdateOutput nextLevelUpdateOutput = new UpdateAction().execute(nextLevelUpdateInput);
+            }
+
+            if(CollectionUtils.nullSafeHasContents(nextLevelInserts))
+            {
+               LOG.debug("Inserting associatedRecords", logPair("associatedTable", associatedTable.getName()), logPair("noOfRecords", nextLevelUpdates.size()));
+               InsertInput nextLevelInsertInput = new InsertInput();
+               nextLevelInsertInput.setTransaction(updateInput.getTransaction());
+               nextLevelInsertInput.setTableName(association.getAssociatedTableName());
+               nextLevelInsertInput.setRecords(nextLevelInserts);
+               InsertOutput nextLevelInsertOutput = new InsertAction().execute(nextLevelInsertInput);
+            }
+         }
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private static List<QRecord> getOldRecordListForAuditIfNeeded(UpdateInput updateInput)
+   {
+      if(updateInput.getOmitDmlAudit())
+      {
+         return (null);
+      }
+
+      try
+      {
+         AuditLevel    auditLevel    = DMLAuditAction.getAuditLevel(updateInput);
+         List<QRecord> oldRecordList = null;
+         if(AuditLevel.FIELD.equals(auditLevel))
+         {
+            String             primaryKeyField   = updateInput.getTable().getPrimaryKeyField();
+            List<Serializable> pkeysBeingUpdated = CollectionUtils.nonNullList(updateInput.getRecords()).stream().map(r -> r.getValue(primaryKeyField)).toList();
+
+            QueryInput queryInput = new QueryInput();
+            queryInput.setTransaction(updateInput.getTransaction());
+            queryInput.setTableName(updateInput.getTableName());
+            queryInput.setFilter(new QQueryFilter(new QFilterCriteria(primaryKeyField, QCriteriaOperator.IN, pkeysBeingUpdated)));
+            // todo - need a limit?  what if too many??
+            QueryOutput queryOutput = new QueryAction().execute(queryInput);
+
+            oldRecordList = queryOutput.getRecords();
+         }
+         return oldRecordList;
+      }
+      catch(Exception e)
+      {
+         LOG.warn("Error getting old record list for audit", e, logPair("table", updateInput.getTableName()));
+         return (null);
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** If the table being updated uses an automation-status field, populate it now.
+    *******************************************************************************/
+   private void setAutomationStatusField(UpdateInput updateInput)
+   {
+      RecordAutomationStatusUpdater.setAutomationStatusInRecords(updateInput.getSession(), updateInput.getTable(), updateInput.getRecords(), AutomationStatus.PENDING_UPDATE_AUTOMATIONS);
+   }
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/helpers/UniqueKeyHelper.java

@@ -0,0 +1,171 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.actions.tables.helpers;
+
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.stream.Collectors;
+import com.kingsrook.qqq.backend.core.actions.QBackendTransaction;
+import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.UniqueKey;
+import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
+import com.kingsrook.qqq.backend.core.utils.ValueUtils;
+
+
+/*******************************************************************************
+ ** Methods to help with unique key checks.
+ *******************************************************************************/
+public class UniqueKeyHelper
+{
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public static Map<List<Serializable>, Serializable> getExistingKeys(QBackendTransaction transaction, QTableMetaData table, List<QRecord> recordList, UniqueKey uniqueKey) throws QException
+   {
+      List<String>                          ukFieldNames    = uniqueKey.getFieldNames();
+      Map<List<Serializable>, Serializable> existingRecords = new HashMap<>();
+      if(ukFieldNames != null)
+      {
+         QueryInput queryInput = new QueryInput();
+         queryInput.setTableName(table.getName());
+         queryInput.setTransaction(transaction);
+
+         QQueryFilter filter = new QQueryFilter();
+         if(ukFieldNames.size() == 1)
+         {
+            List<Serializable> values = recordList.stream()
+               .filter(r -> CollectionUtils.nullSafeIsEmpty(r.getErrors()))
+               .map(r -> r.getValue(ukFieldNames.get(0)))
+               .collect(Collectors.toList());
+            filter.addCriteria(new QFilterCriteria(ukFieldNames.get(0), QCriteriaOperator.IN, values));
+         }
+         else
+         {
+            filter.setBooleanOperator(QQueryFilter.BooleanOperator.OR);
+            for(QRecord record : recordList)
+            {
+               if(CollectionUtils.nullSafeHasContents(record.getErrors()))
+               {
+                  continue;
+               }
+
+               QQueryFilter subFilter = new QQueryFilter();
+               filter.addSubFilter(subFilter);
+               for(String fieldName : ukFieldNames)
+               {
+                  Serializable value = record.getValue(fieldName);
+                  if(value == null)
+                  {
+                     subFilter.addCriteria(new QFilterCriteria(fieldName, QCriteriaOperator.IS_BLANK));
+                  }
+                  else
+                  {
+                     subFilter.addCriteria(new QFilterCriteria(fieldName, QCriteriaOperator.EQUALS, value));
+                  }
+               }
+            }
+
+            if(CollectionUtils.nullSafeIsEmpty(filter.getSubFilters()))
+            {
+               ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+               // if we didn't build any sub-filters (because all records have errors in them), don't run a query w/ no clauses - rather - return early. //
+               ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+               return (existingRecords);
+            }
+         }
+
+         queryInput.setFilter(filter);
+         QueryOutput queryOutput = new QueryAction().execute(queryInput);
+         for(QRecord record : queryOutput.getRecords())
+         {
+            Optional<List<Serializable>> keyValues = getKeyValues(table, uniqueKey, record);
+            if(keyValues.isPresent())
+            {
+               existingRecords.put(keyValues.get(), record.getValue(table.getPrimaryKeyField()));
+            }
+         }
+      }
+
+      return (existingRecords);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public static Optional<List<Serializable>> getKeyValues(QTableMetaData table, UniqueKey uniqueKey, QRecord record)
+   {
+      try
+      {
+         List<Serializable> keyValues = new ArrayList<>();
+         for(String fieldName : uniqueKey.getFieldNames())
+         {
+            QFieldMetaData field      = table.getField(fieldName);
+            Serializable   value      = record.getValue(fieldName);
+            Serializable   typedValue = ValueUtils.getValueAsFieldType(field.getType(), value);
+            keyValues.add(typedValue == null ? new NullUniqueKeyValue() : typedValue);
+         }
+         return (Optional.of(keyValues));
+      }
+      catch(Exception e)
+      {
+         return (Optional.empty());
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** To make a list of unique key values here behave like they do in an RDBMS
+    ** (which is what we're trying to mimic - which is - 2 null values in a field
+    ** aren't considered the same, so they don't violate a unique key) (at least, that's
+    ** how some RDBMS's work, right??) - use this value instead of nulls in the
+    ** output of getKeyValues - where interestingly, this class always returns
+    ** false in it equals method... Unclear how bad this is, e.g., if it's violating
+    ** the contract for equals and hashCode...
+    *******************************************************************************/
+   public static class NullUniqueKeyValue implements Serializable
+   {
+      @Override
+      public boolean equals(Object obj)
+      {
+         return (false);
+      }
+   }
+
+}