CE-847 Add omitTriggeringAutomations to updateInput to ... let you say that you want to ... omit trigger automations

Fix how column-stats backend handles date-times, grouping by hour.  u…

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/audits/DMLAuditAction.java

@@ -78,6 +78,7 @@ public class DMLAuditAction extends AbstractQActionFunction<DMLAuditInput, DMLAu
    private static Set<String> loggedUnauditableTableNames = new HashSet<>();
 
 
+
    /*******************************************************************************
     **
     *******************************************************************************/
@@ -210,6 +211,19 @@ public class DMLAuditAction extends AbstractQActionFunction<DMLAuditInput, DMLAu
          contextSuffix.append(" ").append(input.getAuditContext());
       }
 
+      //////////////////////////////////////////////////////////////
+      // look for a context value place directly into the session //
+      //////////////////////////////////////////////////////////////
+      QSession qSession = QContext.getQSession();
+      if(qSession != null)
+      {
+         String sessionContext = qSession.getValue(AUDIT_CONTEXT_FIELD_NAME);
+         if(StringUtils.hasContent(sessionContext))
+         {
+            contextSuffix.append(" ").append(sessionContext);
+         }
+      }
+
       /////////////////////////////////////////////////////////////////////////////////////
       // note process label (and a possible context from the process's state) if present //
       /////////////////////////////////////////////////////////////////////////////////////
@@ -233,17 +247,20 @@ public class DMLAuditAction extends AbstractQActionFunction<DMLAuditInput, DMLAu
       ///////////////////////////////////////////////////
       // use api label & version if present in session //
       ///////////////////////////////////////////////////
-      QSession qSession   = QContext.getQSession();
-      String   apiVersion = qSession.getValue("apiVersion");
-      if(apiVersion != null)
+      if(qSession != null)
       {
-         String apiLabel = qSession.getValue("apiLabel");
-         if(!StringUtils.hasContent(apiLabel))
+         String apiVersion = qSession.getValue("apiVersion");
+         if(apiVersion != null)
          {
-            apiLabel = "API";
+            String apiLabel = qSession.getValue("apiLabel");
+            if(!StringUtils.hasContent(apiLabel))
+            {
+               apiLabel = "API";
+            }
+            contextSuffix.append(" via ").append(apiLabel).append(" Version: ").append(apiVersion);
          }
-         contextSuffix.append(" via ").append(apiLabel).append(" Version: ").append(apiVersion);
       }
+
       return (contextSuffix.toString());
    }
 

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/automation/AutomationStatus.java

@@ -22,6 +22,7 @@
 package com.kingsrook.qqq.backend.core.actions.automation;
 
 
+import java.util.Objects;
 import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.PossibleValueEnum;
 
 
@@ -55,6 +56,30 @@ public enum AutomationStatus implements PossibleValueEnum<Integer>
 
 
 
+   /*******************************************************************************
+    ** Get instance by id
+    **
+    *******************************************************************************/
+   public static AutomationStatus getById(Integer id)
+   {
+      if(id == null)
+      {
+         return (null);
+      }
+
+      for(AutomationStatus value : AutomationStatus.values())
+      {
+         if(Objects.equals(value.id, id))
+         {
+            return (value);
+         }
+      }
+
+      return (null);
+   }
+
+
+
    /*******************************************************************************
     ** Getter for id
     **
@@ -106,10 +131,10 @@ public enum AutomationStatus implements PossibleValueEnum<Integer>
    public String getInsertOrUpdate()
    {
       return switch(this)
-         {
-            case PENDING_INSERT_AUTOMATIONS, RUNNING_INSERT_AUTOMATIONS, FAILED_INSERT_AUTOMATIONS -> "Insert";
-            case PENDING_UPDATE_AUTOMATIONS, RUNNING_UPDATE_AUTOMATIONS, FAILED_UPDATE_AUTOMATIONS -> "Update";
-            case OK -> "";
-         };
+      {
+         case PENDING_INSERT_AUTOMATIONS, RUNNING_INSERT_AUTOMATIONS, FAILED_INSERT_AUTOMATIONS -> "Insert";
+         case PENDING_UPDATE_AUTOMATIONS, RUNNING_UPDATE_AUTOMATIONS, FAILED_UPDATE_AUTOMATIONS -> "Update";
+         case OK -> "";
+      };
    }
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/automation/RecordAutomationStatusUpdater.java

@@ -22,30 +22,40 @@
 package com.kingsrook.qqq.backend.core.actions.automation;
 
 
+import java.io.Serializable;
+import java.time.Duration;
+import java.time.temporal.ChronoUnit;
 import java.util.Collections;
+import java.util.HashSet;
 import java.util.List;
+import java.util.Optional;
+import java.util.Set;
+import com.kingsrook.qqq.backend.core.actions.QBackendTransaction;
 import com.kingsrook.qqq.backend.core.actions.tables.CountAction;
+import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
 import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction;
 import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.instances.QMetaDataVariableInterpreter;
 import com.kingsrook.qqq.backend.core.logging.QLogger;
 import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountOutput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
 import com.kingsrook.qqq.backend.core.model.automation.TableTrigger;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
-import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.AutomationStatusTrackingType;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.QTableAutomationDetails;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.TableAutomationAction;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.TriggerEvent;
-import com.kingsrook.qqq.backend.core.model.session.QSession;
 import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
+import com.kingsrook.qqq.backend.core.utils.memoization.Memoization;
 import org.apache.commons.lang.NotImplementedException;
+import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
 
 
 /*******************************************************************************
@@ -55,19 +65,37 @@ public class RecordAutomationStatusUpdater
 {
    private static final QLogger LOG = QLogger.getLogger(RecordAutomationStatusUpdater.class);
 
+   ///////////////////////////////////////////////////////////////////////////////////////////////////////
+   // feature flag - by default, will be true - before setting records to PENDING_UPDATE_AUTOMATIONS,   //
+   // we will fetch them (if we didn't take them in from the caller, which, UpdateAction does if its    //
+   // backend supports it), to check their current automationStatus - and if they are currently PENDING //
+   // or RUNNING inserts or updates, we won't update them.                                              //
+   ///////////////////////////////////////////////////////////////////////////////////////////////////////
+   private static boolean allowPreUpdateFetch = new QMetaDataVariableInterpreter().getBooleanFromPropertyOrEnvironment("qqq.recordAutomationStatusUpdater.allowPreUpdateFetch", "QQQ_RECORD_AUTOMATION_STATUS_UPDATER_ALLOW_PRE_UPDATE_FETCH", true);
+
+   ///////////////////////////////////////////////////////////////////////////////////////////////
+   // feature flag - by default, we'll memoize the check for triggers - but we can turn it off. //
+   ///////////////////////////////////////////////////////////////////////////////////////////////
+   private static boolean memoizeCheckForTriggers = new QMetaDataVariableInterpreter().getBooleanFromPropertyOrEnvironment("qqq.recordAutomationStatusUpdater.memoizeCheckForTriggers", "QQQ_RECORD_AUTOMATION_STATUS_UPDATER_MEMOIZE_CHECK_FOR_TRIGGERS", true);
+
+   private static Memoization<Key, Boolean> areThereTableTriggersForTableMemoization = new Memoization<Key, Boolean>().withTimeout(Duration.of(60, ChronoUnit.SECONDS));
+
 
 
    /*******************************************************************************
     ** for a list of records from a table, set their automation status - based on
     ** how the table is configured.
     *******************************************************************************/
-   public static boolean setAutomationStatusInRecords(QSession session, QTableMetaData table, List<QRecord> records, AutomationStatus automationStatus)
+   public static boolean setAutomationStatusInRecords(QTableMetaData table, List<QRecord> records, AutomationStatus automationStatus, QBackendTransaction transaction, List<QRecord> oldRecordList)
    {
       if(table == null || table.getAutomationDetails() == null || CollectionUtils.nullSafeIsEmpty(records))
       {
          return (false);
       }
 
+      QTableAutomationDetails automationDetails   = table.getAutomationDetails();
+      Set<Serializable>       pkeysWeMayNotUpdate = new HashSet<>();
+
       ///////////////////////////////////////////////////////////////////////////////////////////////////
       // In case an automation is running, and it updates records - don't let those records be marked  //
       // as PENDING_UPDATE_AUTOMATIONS... this is meant to avoid having a record's automation update   //
@@ -81,12 +109,60 @@ public class RecordAutomationStatusUpdater
          for(StackTraceElement stackTraceElement : e.getStackTrace())
          {
             String className = stackTraceElement.getClassName();
-            if(className.contains("com.kingsrook.qqq.backend.core.actions.automation") && !className.equals(RecordAutomationStatusUpdater.class.getName()) && !className.endsWith("Test"))
+            if(className.contains(RecordAutomationStatusUpdater.class.getPackageName()) && !className.equals(RecordAutomationStatusUpdater.class.getName()) && !className.endsWith("Test") && !className.contains("Test$"))
             {
                LOG.debug("Avoiding re-setting automation status to PENDING_UPDATE while running an automation");
                return (false);
             }
          }
+
+         ///////////////////////////////////////////////////////////////////////////////
+         // if table uses field-in-table status tracking, then check the old records, //
+         // before we set them to pending-updates, to avoid losing other pending or   //
+         // running status information.  We will allow moving  from OK or the 2       //
+         // failed statuses into pending-updates - which seems right.                 //
+         // This is added to fix cases where an update that comes in before insert    //
+         // -automations have run, will cause the pending-insert status to be missed. //
+         ///////////////////////////////////////////////////////////////////////////////
+         if(automationDetails.getStatusTracking() != null && AutomationStatusTrackingType.FIELD_IN_TABLE.equals(automationDetails.getStatusTracking().getType()))
+         {
+            try
+            {
+               if(CollectionUtils.nullSafeIsEmpty(oldRecordList))
+               {
+                  ///////////////////////////////////////////////////////////////////////////////////////////////
+                  // if we didn't get the oldRecordList as input (though UpdateAction should usually pass it?) //
+                  // then check feature-flag if we're allowed to do a lookup here & now.  If so, then do.      //
+                  ///////////////////////////////////////////////////////////////////////////////////////////////
+                  if(allowPreUpdateFetch)
+                  {
+                     List<Serializable> pkeysToLookup = records.stream().map(r -> r.getValue(table.getPrimaryKeyField())).toList();
+                     oldRecordList = new QueryAction().execute(new QueryInput(table.getName())
+                        .withFilter(new QQueryFilter(new QFilterCriteria(table.getPrimaryKeyField(), QCriteriaOperator.IN, pkeysToLookup)))
+                        .withTransaction(transaction)
+                     ).getRecords();
+                  }
+               }
+
+               for(QRecord freshRecord : CollectionUtils.nonNullList(oldRecordList))
+               {
+                  Serializable recordStatus = freshRecord.getValue(automationDetails.getStatusTracking().getFieldName());
+                  if(AutomationStatus.PENDING_INSERT_AUTOMATIONS.getId().equals(recordStatus)
+                     || AutomationStatus.PENDING_UPDATE_AUTOMATIONS.getId().equals(recordStatus)
+                     || AutomationStatus.RUNNING_INSERT_AUTOMATIONS.getId().equals(recordStatus)
+                     || AutomationStatus.RUNNING_UPDATE_AUTOMATIONS.getId().equals(recordStatus))
+                  {
+                     Serializable primaryKey = freshRecord.getValue(table.getPrimaryKeyField());
+                     LOG.debug("May not update automation status", logPair("table", table.getName()), logPair("id", primaryKey), logPair("currentStatus", recordStatus), logPair("requestedStatus", automationStatus.getId()));
+                     pkeysWeMayNotUpdate.add(primaryKey);
+                  }
+               }
+            }
+            catch(QException qe)
+            {
+               LOG.error("Error checking existing automation status before setting new automation status - more records will be updated than maybe should be...", qe);
+            }
+         }
       }
 
       ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
@@ -98,19 +174,15 @@ public class RecordAutomationStatusUpdater
          automationStatus = AutomationStatus.OK;
       }
 
-      QTableAutomationDetails automationDetails = table.getAutomationDetails();
       if(automationDetails.getStatusTracking() != null && AutomationStatusTrackingType.FIELD_IN_TABLE.equals(automationDetails.getStatusTracking().getType()))
       {
          for(QRecord record : records)
          {
-            ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-            // todo - seems like there's some case here, where if an order was in PENDING_INSERT, but then some other job updated the record, that we'd  //
-            //  lose that pending status, which would be a Bad Thing™...                                                                                 //
-            //  problem is - we may not have the full record in here, so we can't necessarily check the record to see what status it's currently in...   //
-            ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-
-            record.setValue(automationDetails.getStatusTracking().getFieldName(), automationStatus.getId());
-            // todo - another field - for the automation timestamp??
+            if(!pkeysWeMayNotUpdate.contains(record.getValue(table.getPrimaryKeyField())))
+            {
+               record.setValue(automationDetails.getStatusTracking().getFieldName(), automationStatus.getId());
+               // todo - another field - for the automation timestamp??
+            }
          }
       }
 
@@ -188,11 +260,29 @@ public class RecordAutomationStatusUpdater
          return (false);
       }
 
+      if(memoizeCheckForTriggers)
+      {
+         ///////////////////////////////////////////////////////////////////////////////////////
+         // as within the lookup method, error on the side of "yes, maybe there are triggers" //
+         ///////////////////////////////////////////////////////////////////////////////////////
+         Optional<Boolean> result = areThereTableTriggersForTableMemoization.getResult(new Key(table, triggerEvent), key -> lookupIfThereAreTriggersForTable(table, triggerEvent));
+         return result.orElse(true);
+      }
+      else
+      {
+         return lookupIfThereAreTriggersForTable(table, triggerEvent);
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private static Boolean lookupIfThereAreTriggersForTable(QTableMetaData table, TriggerEvent triggerEvent)
+   {
       try
       {
-         ///////////////////
-         // todo - cache? //
-         ///////////////////
          CountInput countInput = new CountInput();
          countInput.setTableName(TableTrigger.TABLE_NAME);
          countInput.setFilter(new QQueryFilter(
@@ -207,6 +297,7 @@ public class RecordAutomationStatusUpdater
          ///////////////////////////////////////////////////////////////////////////////////////////////////////////
          // if the count query failed, we're a bit safer to err on the side of "yeah, there might be automations" //
          ///////////////////////////////////////////////////////////////////////////////////////////////////////////
+         LOG.warn("Error looking if there are triggers for table", e, logPair("tableName", table.getName()));
          return (true);
       }
    }
@@ -217,12 +308,12 @@ public class RecordAutomationStatusUpdater
     ** for a list of records, update their automation status and actually Update the
     ** backend as well.
     *******************************************************************************/
-   public static void setAutomationStatusInRecordsAndUpdate(QInstance instance, QSession session, QTableMetaData table, List<QRecord> records, AutomationStatus automationStatus) throws QException
+   public static void setAutomationStatusInRecordsAndUpdate(QTableMetaData table, List<QRecord> records, AutomationStatus automationStatus, QBackendTransaction transaction) throws QException
    {
       QTableAutomationDetails automationDetails = table.getAutomationDetails();
       if(automationDetails != null && AutomationStatusTrackingType.FIELD_IN_TABLE.equals(automationDetails.getStatusTracking().getType()))
       {
-         boolean didSetStatusField = setAutomationStatusInRecords(session, table, records, automationStatus);
+         boolean didSetStatusField = setAutomationStatusInRecords(table, records, automationStatus, transaction, null);
          if(didSetStatusField)
          {
             UpdateInput updateInput = new UpdateInput();
@@ -237,6 +328,7 @@ public class RecordAutomationStatusUpdater
                .withValue(table.getPrimaryKeyField(), r.getValue(table.getPrimaryKeyField()))
                .withValue(automationDetails.getStatusTracking().getFieldName(), r.getValue(automationDetails.getStatusTracking().getFieldName()))).toList());
             updateInput.setAreAllValuesBeingUpdatedTheSame(true);
+            updateInput.setTransaction(transaction);
             updateInput.setOmitDmlAudit(true);
 
             new UpdateAction().execute(updateInput);
@@ -250,4 +342,8 @@ public class RecordAutomationStatusUpdater
       }
    }
 
+
+
+   private record Key(QTableMetaData table, TriggerEvent triggerEvent) {}
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/automation/polling/PollingAutomationPerTableRunner.java

@@ -252,8 +252,7 @@ public class PollingAutomationPerTableRunner implements Runnable
 
       try
       {
-         QSession session = sessionSupplier != null ? sessionSupplier.get() : new QSession();
-         processTableInsertOrUpdate(instance.getTable(tableActions.tableName()), session, tableActions.status());
+         processTableInsertOrUpdate(instance.getTable(tableActions.tableName()), tableActions.status());
       }
       catch(Exception e)
       {
@@ -271,7 +270,7 @@ public class PollingAutomationPerTableRunner implements Runnable
    /*******************************************************************************
     ** Query for and process records that have a PENDING_INSERT or PENDING_UPDATE status on a given table.
     *******************************************************************************/
-   public void processTableInsertOrUpdate(QTableMetaData table, QSession session, AutomationStatus automationStatus) throws QException
+   public void processTableInsertOrUpdate(QTableMetaData table, AutomationStatus automationStatus) throws QException
    {
       /////////////////////////////////////////////////////////////////////////
       // get the actions to run against this table in this automation status //
@@ -322,7 +321,7 @@ public class PollingAutomationPerTableRunner implements Runnable
          }, () ->
          {
             List<QRecord> records = recordPipe.consumeAvailableRecords();
-            applyActionsToRecords(session, table, records, actions, automationStatus);
+            applyActionsToRecords(table, records, actions, automationStatus);
             return (records.size());
          }
       );
@@ -430,7 +429,7 @@ public class PollingAutomationPerTableRunner implements Runnable
     ** table's actions against them - IF they are found to match the action's filter
     ** (assuming it has one - if it doesn't, then all records match).
     *******************************************************************************/
-   private void applyActionsToRecords(QSession session, QTableMetaData table, List<QRecord> records, List<TableAutomationAction> actions, AutomationStatus automationStatus) throws QException
+   private void applyActionsToRecords(QTableMetaData table, List<QRecord> records, List<TableAutomationAction> actions, AutomationStatus automationStatus) throws QException
    {
       if(CollectionUtils.nullSafeIsEmpty(records))
       {
@@ -440,7 +439,7 @@ public class PollingAutomationPerTableRunner implements Runnable
       ///////////////////////////////////////////////////
       // mark the records as RUNNING their automations //
       ///////////////////////////////////////////////////
-      RecordAutomationStatusUpdater.setAutomationStatusInRecordsAndUpdate(instance, session, table, records, pendingToRunningStatusMap.get(automationStatus));
+      RecordAutomationStatusUpdater.setAutomationStatusInRecordsAndUpdate(table, records, pendingToRunningStatusMap.get(automationStatus), null);
 
       ///////////////////////////////////////////////////////////////////////////////////////////////////////////////
       // foreach action - run it against the records (but only if they match the action's filter, if there is one) //
@@ -460,11 +459,11 @@ public class PollingAutomationPerTableRunner implements Runnable
       ////////////////////////////////////////
       if(anyActionsFailed)
       {
-         RecordAutomationStatusUpdater.setAutomationStatusInRecordsAndUpdate(instance, session, table, records, pendingToFailedStatusMap.get(automationStatus));
+         RecordAutomationStatusUpdater.setAutomationStatusInRecordsAndUpdate(table, records, pendingToFailedStatusMap.get(automationStatus), null);
       }
       else
       {
-         RecordAutomationStatusUpdater.setAutomationStatusInRecordsAndUpdate(instance, session, table, records, AutomationStatus.OK);
+         RecordAutomationStatusUpdater.setAutomationStatusInRecordsAndUpdate(table, records, AutomationStatus.OK, null);
       }
    }
 

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/dashboard/widgets/DateTimeGroupBy.java

@@ -297,4 +297,21 @@ public enum DateTimeGroupBy
       ZonedDateTime zoned = instant.atZone(zoneId);
       return (zoned.plus(noOfChronoUnitsToAdd, chronoUnitToAdd).toInstant());
    }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public static DateTimeFormatter sqlDateFormatToSelectedDateTimeFormatter(String sqlDateFormat)
+   {
+      for(DateTimeGroupBy value : values())
+      {
+         if(value.sqlDateFormat.equals(sqlDateFormat))
+         {
+            return (value.selectedStringFormatter);
+         }
+      }
+      return null;
+   }
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/scripts/RunAdHocRecordScriptAction.java

@@ -30,6 +30,7 @@ import java.util.List;
 import java.util.Map;
 import java.util.Optional;
 import com.kingsrook.qqq.backend.core.actions.ActionHelper;
+import com.kingsrook.qqq.backend.core.actions.audits.DMLAuditAction;
 import com.kingsrook.qqq.backend.core.actions.tables.GetAction;
 import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
 import com.kingsrook.qqq.backend.core.context.QContext;
@@ -47,12 +48,14 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryJoin;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
 import com.kingsrook.qqq.backend.core.model.metadata.code.AdHocScriptCodeReference;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 import com.kingsrook.qqq.backend.core.model.scripts.Script;
 import com.kingsrook.qqq.backend.core.model.scripts.ScriptRevision;
 import com.kingsrook.qqq.backend.core.model.scripts.ScriptsMetaDataProvider;
 import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
+import com.kingsrook.qqq.backend.core.utils.memoization.Memoization;
 
 
 /*******************************************************************************
@@ -65,6 +68,8 @@ public class RunAdHocRecordScriptAction
    private Map<Integer, ScriptRevision> scriptRevisionCacheByScriptRevisionId = new HashMap<>();
    private Map<Integer, ScriptRevision> scriptRevisionCacheByScriptId         = new HashMap<>();
 
+   private static Memoization<Integer, Script> scriptMemoizationById = new Memoization<>();
+
 
 
    /*******************************************************************************
@@ -85,6 +90,12 @@ public class RunAdHocRecordScriptAction
             throw (new QException("Script revision was not found."));
          }
 
+         Optional<Script> script = getScript(scriptRevision);
+
+         QContext.getQSession().setValue(DMLAuditAction.AUDIT_CONTEXT_FIELD_NAME, script.isPresent()
+            ? "via Script \"%s\"".formatted(script.get().getName())
+            : "via Script id " + scriptRevision.getScriptId());
+
          ////////////////////////////
          // figure out the records //
          ////////////////////////////
@@ -124,6 +135,10 @@ public class RunAdHocRecordScriptAction
       {
          output.setException(Optional.of(e));
       }
+      finally
+      {
+         QContext.getQSession().removeValue(DMLAuditAction.AUDIT_CONTEXT_FIELD_NAME);
+      }
    }
 
 
@@ -217,4 +232,44 @@ public class RunAdHocRecordScriptAction
       throw (new QException("Code reference did not contain a scriptRevision, scriptRevisionId, or scriptId"));
    }
 
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private Optional<Script> getScript(ScriptRevision scriptRevision)
+   {
+      if(scriptRevision == null || scriptRevision.getScriptId() == null)
+      {
+         return (Optional.empty());
+      }
+
+      try
+      {
+         return scriptMemoizationById.getResult(scriptRevision.getScriptId(), scriptId ->
+         {
+            try
+            {
+               QRecord scriptRecord = new GetAction().executeForRecord(new GetInput(Script.TABLE_NAME).withPrimaryKey(scriptRevision.getScriptId()));
+               if(scriptRecord != null)
+               {
+                  Script script = new Script(scriptRecord);
+                  scriptMemoizationById.storeResult(scriptRevision.getScriptId(), script);
+                  return (script);
+               }
+            }
+            catch(Exception e)
+            {
+               LOG.info("");
+            }
+
+            return (null);
+         });
+      }
+      catch(Exception e)
+      {
+         return (Optional.empty());
+      }
+   }
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/InsertAction.java

@@ -444,7 +444,7 @@ public class InsertAction extends AbstractQActionFunction<InsertInput, InsertOut
     *******************************************************************************/
    private void setAutomationStatusField(InsertInput insertInput)
    {
-      RecordAutomationStatusUpdater.setAutomationStatusInRecords(insertInput.getSession(), insertInput.getTable(), insertInput.getRecords(), AutomationStatus.PENDING_INSERT_AUTOMATIONS);
+      RecordAutomationStatusUpdater.setAutomationStatusInRecords(insertInput.getTable(), insertInput.getRecords(), AutomationStatus.PENDING_INSERT_AUTOMATIONS, insertInput.getTransaction(), null);
    }
 
 

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/UpdateAction.java

@@ -113,7 +113,6 @@ public class UpdateAction
    public UpdateOutput execute(UpdateInput updateInput) throws QException
    {
       ActionHelper.validateSession(updateInput);
-      setAutomationStatusField(updateInput);
 
       QTableMetaData table = updateInput.getTable();
 
@@ -130,6 +129,16 @@ public class UpdateAction
       ////////////////////////////////////////////////////////////////////////////////
       Optional<List<QRecord>> oldRecordList = fetchOldRecords(updateInput, updateInterface);
 
+      ///////////////////////////////////////////////////////////////////////////////////////
+      // allow caller to specify that we don't want to trigger automations. this isn't     //
+      // isn't expected to be used much - by design, only for the process that is meant to //
+      // heal automation status, so that it can force us into status=Pending-inserts       //
+      ///////////////////////////////////////////////////////////////////////////////////////
+      if(!updateInput.getOmitTriggeringAutomations())
+      {
+         setAutomationStatusField(updateInput, oldRecordList);
+      }
+
       performValidations(updateInput, oldRecordList, false);
 
       ////////////////////////////////////
@@ -561,9 +570,9 @@ public class UpdateAction
    /*******************************************************************************
     ** If the table being updated uses an automation-status field, populate it now.
     *******************************************************************************/
-   private void setAutomationStatusField(UpdateInput updateInput)
+   private void setAutomationStatusField(UpdateInput updateInput, Optional<List<QRecord>> oldRecordList)
    {
-      RecordAutomationStatusUpdater.setAutomationStatusInRecords(updateInput.getSession(), updateInput.getTable(), updateInput.getRecords(), AutomationStatus.PENDING_UPDATE_AUTOMATIONS);
+      RecordAutomationStatusUpdater.setAutomationStatusInRecords(updateInput.getTable(), updateInput.getRecords(), AutomationStatus.PENDING_UPDATE_AUTOMATIONS, updateInput.getTransaction(), oldRecordList.orElse(null));
    }
 
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QQueryFilter.java

@@ -323,6 +323,18 @@ public class QQueryFilter implements Serializable, Cloneable
 
 
 
+   /*******************************************************************************
+    ** Fluent setter for adding a single subFilter
+    **
+    *******************************************************************************/
+   public QQueryFilter withSubFilter(QQueryFilter subFilter)
+   {
+      addSubFilter(subFilter);
+      return (this);
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/update/UpdateInput.java

@@ -51,6 +51,7 @@ public class UpdateInput extends AbstractTableActionInput
    ////////////////////////////////////////////////////////////////////////////////////////////
    private Boolean areAllValuesBeingUpdatedTheSame = null;
 
+   private boolean omitTriggeringAutomations = false;
    private boolean omitDmlAudit = false;
    private String  auditContext = null;
 
@@ -321,4 +322,35 @@ public class UpdateInput extends AbstractTableActionInput
       return (this);
    }
 
+
+
+   /*******************************************************************************
+    ** Getter for omitTriggeringAutomations
+    *******************************************************************************/
+   public boolean getOmitTriggeringAutomations()
+   {
+      return (this.omitTriggeringAutomations);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for omitTriggeringAutomations
+    *******************************************************************************/
+   public void setOmitTriggeringAutomations(boolean omitTriggeringAutomations)
+   {
+      this.omitTriggeringAutomations = omitTriggeringAutomations;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for omitTriggeringAutomations
+    *******************************************************************************/
+   public UpdateInput withOmitTriggeringAutomations(boolean omitTriggeringAutomations)
+   {
+      this.omitTriggeringAutomations = omitTriggeringAutomations;
+      return (this);
+   }
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/data/QRecord.java

@@ -32,6 +32,7 @@ import java.util.ArrayList;
 import java.util.Date;
 import java.util.HashMap;
 import java.util.LinkedHashMap;
+import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
@@ -167,6 +168,11 @@ public class QRecord implements Serializable
             ArrayList<?> cloneList = new ArrayList<>(arrayList);
             clone.put(entry.getKey(), (V) cloneList);
          }
+         else if(entry.getValue() instanceof LinkedList<?> linkedList)
+         {
+            LinkedList<?> cloneList = new LinkedList<>(linkedList);
+            clone.put(entry.getKey(), (V) cloneList);
+         }
          else if(entry.getValue() instanceof LinkedHashMap<?, ?> linkedHashMap)
          {
             LinkedHashMap<?, ?> cloneMap = new LinkedHashMap<>(linkedHashMap);

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/dashboard/nocode/HtmlWrapper.java

@@ -51,6 +51,7 @@ public class HtmlWrapper implements Serializable
    public static final String STYLE_INDENT_2        = "padding-left: 2rem; ";
    public static final String STYLE_FLOAT_RIGHT     = "float: right; ";
    public static final String STYLE_RED             = "color: red; ";
+   public static final String STYLE_YELLOW = "color: #bfb743; ";
 
 
 

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/session/QSession.java

@@ -142,6 +142,19 @@ public class QSession implements Serializable
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public void removeValue(String key)
+   {
+      if(values != null)
+      {
+         values.remove(key);
+      }
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/memory/MemoryRecordStore.java

@@ -24,6 +24,10 @@ package com.kingsrook.qqq.backend.core.modules.backend.implementations.memory;
 
 import java.io.Serializable;
 import java.math.BigDecimal;
+import java.time.Instant;
+import java.time.ZoneId;
+import java.time.ZonedDateTime;
+import java.time.format.DateTimeFormatter;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
@@ -35,6 +39,7 @@ import java.util.Objects;
 import java.util.Optional;
 import java.util.function.Function;
 import java.util.stream.Collectors;
+import com.kingsrook.qqq.backend.core.actions.dashboard.widgets.DateTimeGroupBy;
 import com.kingsrook.qqq.backend.core.actions.tables.helpers.ValidateRecordSecurityLockHelper;
 import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
@@ -66,6 +71,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 import com.kingsrook.qqq.backend.core.modules.backend.implementations.utils.BackendQueryFilterUtils;
 import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
 import com.kingsrook.qqq.backend.core.utils.ListingHash;
+import com.kingsrook.qqq.backend.core.utils.StringUtils;
 import com.kingsrook.qqq.backend.core.utils.ValueUtils;
 
 
@@ -577,7 +583,11 @@ public class MemoryRecordStore
          for(GroupBy groupBy : groupBys)
          {
             Serializable groupByValue = record.getValue(groupBy.getFieldName());
-            if(groupBy.getType() != null)
+            if(StringUtils.hasContent(groupBy.getFormatString()))
+            {
+               groupByValue = applyFormatString(groupByValue, groupBy);
+            }
+            else if(groupBy.getType() != null)
             {
                groupByValue = ValueUtils.getValueAsFieldType(groupBy.getType(), groupByValue);
             }
@@ -629,7 +639,9 @@ public class MemoryRecordStore
       /////////////////////
       if(aggregateInput.getFilter() != null && CollectionUtils.nullSafeHasContents(aggregateInput.getFilter().getOrderBys()))
       {
-         Comparator<AggregateResult> comparator = null;
+         /////////////////////////////////////////////////////////////////////////////////////
+         // lambda to compare 2 serializables, as we'll assume (& cast) them to Comparables //
+         /////////////////////////////////////////////////////////////////////////////////////
          Comparator<Serializable> serializableComparator = (Serializable a, Serializable b) ->
          {
             if(a == null && b == null)
@@ -647,9 +659,15 @@ public class MemoryRecordStore
             return ((Comparable) a).compareTo(b);
          };
 
+         //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+         // reverse of the lambda above (we had some errors calling .reversed() on the comparator we were building, so this seemed simpler & worked) //
+         //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+         Comparator<Serializable> reverseSerializableComparator = (Serializable a, Serializable b) -> -serializableComparator.compare(a, b);
+
          ////////////////////////////////////////////////
          // build a comparator out of all the orderBys //
          ////////////////////////////////////////////////
+         Comparator<AggregateResult> comparator = null;
          for(QFilterOrderBy orderBy : aggregateInput.getFilter().getOrderBys())
          {
             Function<AggregateResult, Serializable> keyExtractor = aggregateResult ->
@@ -670,16 +688,11 @@ public class MemoryRecordStore
 
             if(comparator == null)
             {
-               comparator = Comparator.comparing(keyExtractor, serializableComparator);
+               comparator = Comparator.comparing(keyExtractor, orderBy.getIsAscending() ? serializableComparator : reverseSerializableComparator);
             }
             else
             {
-               comparator = comparator.thenComparing(keyExtractor, serializableComparator);
-            }
-
-            if(!orderBy.getIsAscending())
-            {
-               comparator = comparator.reversed();
+               comparator = comparator.thenComparing(keyExtractor, orderBy.getIsAscending() ? serializableComparator : reverseSerializableComparator);
             }
          }
 
@@ -696,6 +709,57 @@ public class MemoryRecordStore
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private Serializable applyFormatString(Serializable value, GroupBy groupBy) throws QException
+   {
+      if(value == null)
+      {
+         return (null);
+      }
+
+      String formatString = groupBy.getFormatString();
+
+      try
+      {
+         if(formatString.startsWith("DATE_FORMAT"))
+         {
+            /////////////////////////////////////////////////////////////////////////////
+            // one known-use case we have here looks like this:                        //
+            // DATE_FORMAT(CONVERT_TZ(%s, 'UTC', 'UTC'), '%%Y-%%m-%%dT%%H')            //
+            // ... for now, let's just try to support the formatting bit at the end... //
+            // todo - support the CONVERT_TZ bit too!                                  //
+            /////////////////////////////////////////////////////////////////////////////
+            String            sqlDateTimeFormat = formatString.replaceFirst(".*'%%", "%%").replaceFirst("'.*", "");
+            DateTimeFormatter dateTimeFormatter = DateTimeGroupBy.sqlDateFormatToSelectedDateTimeFormatter(sqlDateTimeFormat);
+            if(dateTimeFormatter == null)
+            {
+               throw (new QException("Unsupported sql dateTime format string [" + sqlDateTimeFormat + "] for MemoryRecordStore"));
+            }
+
+            String        valueAsString  = ValueUtils.getValueAsString(value);
+            Instant       valueAsInstant = ValueUtils.getValueAsInstant(valueAsString);
+            ZonedDateTime zonedDateTime  = valueAsInstant.atZone(ZoneId.systemDefault());
+            return (dateTimeFormatter.format(zonedDateTime));
+         }
+         else
+         {
+            throw (new QException("Unsupported group-by format string [" + formatString + "] for MemoryRecordStore"));
+         }
+      }
+      catch(QException qe)
+      {
+         throw (qe);
+      }
+      catch(Exception e)
+      {
+         throw (new QException("Error applying format string [" + formatString + "] to group by value [" + value + "]", e));
+      }
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/memory/MemoryUpdateAction.java

@@ -22,13 +22,10 @@
 package com.kingsrook.qqq.backend.core.modules.backend.implementations.memory;
 
 
-import java.time.Instant;
 import com.kingsrook.qqq.backend.core.actions.interfaces.UpdateInterface;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateOutput;
-import com.kingsrook.qqq.backend.core.model.data.QRecord;
-import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 
 
 /*******************************************************************************
@@ -45,17 +42,6 @@ public class MemoryUpdateAction extends AbstractMemoryAction implements UpdateIn
    {
       try
       {
-         QTableMetaData table = updateInput.getTable();
-         Instant        now   = Instant.now();
-
-         for(QRecord record : updateInput.getRecords())
-         {
-            ///////////////////////////////////////////
-            // todo .. better (not hard-coded names) //
-            ///////////////////////////////////////////
-            setValueIfTableHasField(record, table, "modifyDate", now, false);
-         }
-
          UpdateOutput updateOutput = new UpdateOutput();
          updateOutput.setRecords(MemoryRecordStore.getInstance().update(updateInput, true));
          return (updateOutput);

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/automation/HealBadRecordAutomationStatusesProcessStep.java

@@ -0,0 +1,298 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.processes.implementations.automation;
+
+
+import java.time.temporal.ChronoUnit;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import com.kingsrook.qqq.backend.core.actions.automation.AutomationStatus;
+import com.kingsrook.qqq.backend.core.actions.processes.BackendStep;
+import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
+import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction;
+import com.kingsrook.qqq.backend.core.context.QContext;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.logging.QLogger;
+import com.kingsrook.qqq.backend.core.model.MetaDataProducerInterface;
+import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
+import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.expressions.NowWithOffset;
+import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
+import com.kingsrook.qqq.backend.core.model.metadata.dashboard.nocode.HtmlWrapper;
+import com.kingsrook.qqq.backend.core.model.metadata.dashboard.nocode.WidgetHtmlLine;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.DisplayFormat;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.DynamicDefaultValueBehavior;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
+import com.kingsrook.qqq.backend.core.model.metadata.processes.NoCodeWidgetFrontendComponentMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.processes.QComponentType;
+import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendComponentMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendStepMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.TablesPossibleValueSourceMetaDataProvider;
+import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
+import com.kingsrook.qqq.backend.core.utils.ExceptionUtils;
+import com.kingsrook.qqq.backend.core.utils.StringUtils;
+import com.kingsrook.qqq.backend.core.utils.collections.MultiLevelMapHelper;
+import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
+
+
+/*******************************************************************************
+ ** Process to find records with a bad automation status, and repair them.
+ **
+ ** Bad status are defined as:
+ ** - failed insert or updates.
+ ** - running insert or updates for more than X minutes (see input field value).
+ **
+ ** Repair in this case means resetting their status to the corresponding (e.g.,
+ ** insert/update) pending status.
+ **
+ *******************************************************************************/
+public class HealBadRecordAutomationStatusesProcessStep implements BackendStep, MetaDataProducerInterface<QProcessMetaData>
+{
+   public static final String NAME = "HealBadRecordAutomationStatusesProcess";
+
+   private static final QLogger LOG = QLogger.getLogger(HealBadRecordAutomationStatusesProcessStep.class);
+
+   private static final Map<Integer, Integer> statusUpdateMap = Map.of(
+      AutomationStatus.FAILED_INSERT_AUTOMATIONS.getId(), AutomationStatus.PENDING_INSERT_AUTOMATIONS.getId(),
+      AutomationStatus.RUNNING_INSERT_AUTOMATIONS.getId(), AutomationStatus.PENDING_INSERT_AUTOMATIONS.getId(),
+      AutomationStatus.FAILED_UPDATE_AUTOMATIONS.getId(), AutomationStatus.PENDING_UPDATE_AUTOMATIONS.getId(),
+      AutomationStatus.RUNNING_UPDATE_AUTOMATIONS.getId(), AutomationStatus.PENDING_UPDATE_AUTOMATIONS.getId()
+   );
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public QProcessMetaData produce(QInstance qInstance) throws QException
+   {
+      QProcessMetaData processMetaData = new QProcessMetaData()
+         .withName(NAME)
+         .withStepList(List.of(
+            new QFrontendStepMetaData()
+               .withName("input")
+               .withComponent(new QFrontendComponentMetaData().withType(QComponentType.EDIT_FORM))
+               .withFormField(new QFieldMetaData("tableName", QFieldType.STRING).withPossibleValueSourceName(TablesPossibleValueSourceMetaDataProvider.NAME))
+               .withFormField(new QFieldMetaData("minutesOldLimit", QFieldType.INTEGER).withDefaultValue(60)),
+            new QBackendStepMetaData()
+               .withName("run")
+               .withCode(new QCodeReference(getClass())),
+            new QFrontendStepMetaData()
+               .withName("output")
+
+               .withComponent(new NoCodeWidgetFrontendComponentMetaData()
+                  .withOutput(new WidgetHtmlLine()
+                     .withCondition(new QFilterCriteria("warningCount", QCriteriaOperator.GREATER_THAN, 0))
+                     .withWrapper(HtmlWrapper.divWithStyles(HtmlWrapper.STYLE_YELLOW))
+                     .withVelocityTemplate("<b>Warning:</b>"))
+                  .withOutput(new WidgetHtmlLine()
+                     .withCondition(new QFilterCriteria("warningCount", QCriteriaOperator.GREATER_THAN, 0))
+                     .withWrapper(HtmlWrapper.divWithStyles(HtmlWrapper.STYLE_INDENT_1))
+                     .withWrapper(HtmlWrapper.divWithStyles(HtmlWrapper.STYLE_YELLOW))
+                     .withVelocityTemplate("""
+                        <ul>
+                           #foreach($string in $warnings)
+                              <li>$string</li>
+                           #end
+                        </ul>
+                        """)))
+
+               .withComponent(new QFrontendComponentMetaData().withType(QComponentType.VIEW_FORM))
+               .withViewField(new QFieldMetaData("totalRecordsUpdated", QFieldType.INTEGER) /* todo - didn't display commas... .withDisplayFormat(DisplayFormat.COMMAS) */)
+
+               .withComponent(new QFrontendComponentMetaData().withType(QComponentType.RECORD_LIST))
+               .withRecordListField(new QFieldMetaData("tableName", QFieldType.STRING))
+               .withRecordListField(new QFieldMetaData("badStatus", QFieldType.STRING))
+               .withRecordListField(new QFieldMetaData("count", QFieldType.INTEGER).withDisplayFormat(DisplayFormat.COMMAS) /* todo - didn't display commas... */)
+
+         ));
+
+      return (processMetaData);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException
+   {
+      int recordsUpdated = 0;
+
+      ////////////////////////////////////////////////////////////////////////
+      // if a table name is given, validate it, and run for just that table //
+      ////////////////////////////////////////////////////////////////////////
+      String            tableName = runBackendStepInput.getValueString("tableName");
+      ArrayList<String> warnings  = new ArrayList<>();
+      if(StringUtils.hasContent(tableName))
+      {
+         if(!QContext.getQInstance().getTables().containsKey(tableName))
+         {
+            throw (new QException("Unrecognized table name: " + tableName));
+         }
+
+         recordsUpdated += processTable(tableName, runBackendStepInput, runBackendStepOutput, warnings);
+      }
+      else
+      {
+         //////////////////////////////////////////////////////////////////////////
+         // else, try to run for all tables that have an automation status field //
+         //////////////////////////////////////////////////////////////////////////
+         for(QTableMetaData table : QContext.getQInstance().getTables().values())
+         {
+            recordsUpdated += processTable(table.getName(), runBackendStepInput, runBackendStepOutput, warnings);
+         }
+      }
+
+      runBackendStepOutput.addValue("totalRecordsUpdated", recordsUpdated);
+      runBackendStepOutput.addValue("warnings", warnings);
+      runBackendStepOutput.addValue("warningCount", warnings.size());
+
+      if(CollectionUtils.nullSafeIsEmpty(runBackendStepOutput.getRecords()))
+      {
+         runBackendStepOutput.addRecord(new QRecord()
+            .withValue("tableName", "--")
+            .withValue("badStatus", "--")
+            .withValue("count", "0"));
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private int processTable(String tableName, RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput, List<String> warnings)
+   {
+      try
+      {
+         Integer        minutesOldLimit = Objects.requireNonNullElse(runBackendStepInput.getValueInteger("minutesOldLimit"), 60);
+         QTableMetaData table           = QContext.getQInstance().getTable(tableName);
+
+         //////////////////////////////////////////////////////////////////////////
+         // only process tables w/ automation details w/ a status tracking field //
+         //////////////////////////////////////////////////////////////////////////
+         if(table != null && table.getAutomationDetails() != null && table.getAutomationDetails().getStatusTracking() != null && StringUtils.hasContent(table.getAutomationDetails().getStatusTracking().getFieldName()))
+         {
+            String automationStatusFieldName = table.getAutomationDetails().getStatusTracking().getFieldName();
+
+            /////////////////////////////////////////////
+            // find the modify-date field on the table //
+            /////////////////////////////////////////////
+            String modifyDateFieldName = null;
+            for(QFieldMetaData field : table.getFields().values())
+            {
+               if(DynamicDefaultValueBehavior.MODIFY_DATE.equals(field.getBehaviorOnlyIfSet(DynamicDefaultValueBehavior.class)))
+               {
+                  modifyDateFieldName = field.getName();
+                  break;
+               }
+            }
+
+            if(modifyDateFieldName == null)
+            {
+               warnings.add("Could not find a Modify Date field on table: " + tableName);
+               LOG.info("Couldn't find a MODIFY_DATE field on table", logPair("tableName", tableName));
+               return 0;
+            }
+
+            ////////////////////////////////////////////////////////////////////////
+            // query for records either FAILED, or RUNNING w/ modify date too old //
+            ////////////////////////////////////////////////////////////////////////
+            QueryInput queryInput = new QueryInput();
+            queryInput.setTableName(tableName);
+            queryInput.setFilter(new QQueryFilter().withBooleanOperator(QQueryFilter.BooleanOperator.OR)
+               .withSubFilter(new QQueryFilter()
+                  .withCriteria(new QFilterCriteria(automationStatusFieldName, QCriteriaOperator.IN, AutomationStatus.FAILED_INSERT_AUTOMATIONS.getId(), AutomationStatus.FAILED_UPDATE_AUTOMATIONS.getId())))
+               .withSubFilter(new QQueryFilter()
+                  .withCriteria(new QFilterCriteria(automationStatusFieldName, QCriteriaOperator.IN, AutomationStatus.RUNNING_INSERT_AUTOMATIONS.getId(), AutomationStatus.RUNNING_UPDATE_AUTOMATIONS.getId()))
+                  .withCriteria(new QFilterCriteria(modifyDateFieldName, QCriteriaOperator.LESS_THAN, NowWithOffset.minus(minutesOldLimit, ChronoUnit.MINUTES))))
+            );
+            QueryOutput queryOutput = new QueryAction().execute(queryInput);
+
+            //////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+            // foreach record found, add it to list of records to be updated - mapping status to appropriate pending status //
+            //////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+            List<QRecord>        recordsToUpdate = new ArrayList<>();
+            Map<String, Integer> countByStatus   = new HashMap<>();
+            for(QRecord record : queryOutput.getRecords())
+            {
+               Integer badAutomationStatusId = record.getValueInteger(automationStatusFieldName);
+               Integer updateStatus          = statusUpdateMap.get(badAutomationStatusId);
+               if(updateStatus != null)
+               {
+                  AutomationStatus badStatus = AutomationStatus.getById(badAutomationStatusId);
+                  if(badStatus != null)
+                  {
+                     MultiLevelMapHelper.getOrPutAndIncrement(countByStatus, badStatus.getLabel());
+                  }
+
+                  recordsToUpdate.add(new QRecord()
+                     .withValue(table.getPrimaryKeyField(), record.getValue(table.getPrimaryKeyField()))
+                     .withValue(automationStatusFieldName, updateStatus));
+               }
+            }
+
+            if(!recordsToUpdate.isEmpty())
+            {
+               LOG.info("Healing bad record automation statuses", logPair("tableName", tableName), logPair("count", recordsToUpdate.size()));
+               new UpdateAction().execute(new UpdateInput(tableName).withRecords(recordsToUpdate).withOmitTriggeringAutomations(true));
+            }
+
+            for(Map.Entry<String, Integer> entry : countByStatus.entrySet())
+            {
+               runBackendStepOutput.addRecord(new QRecord()
+                  .withValue("tableName", tableName)
+                  .withValue("badStatus", entry.getKey())
+                  .withValue("count", entry.getValue()));
+            }
+
+            return (recordsToUpdate.size());
+         }
+      }
+      catch(Exception e)
+      {
+         warnings.add("Error processing table: " + tableName + ": " + ExceptionUtils.getTopAndBottomMessages(e));
+         LOG.warn("Error processing table for bad automation statuses", e, logPair("tableName, name"));
+      }
+
+      return 0;
+   }
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/automation/RunTableAutomationsProcessStep.java

@@ -0,0 +1,149 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.processes.implementations.automation;
+
+
+import java.util.List;
+import java.util.Map;
+import com.kingsrook.qqq.backend.core.actions.automation.polling.PollingAutomationPerTableRunner;
+import com.kingsrook.qqq.backend.core.actions.processes.BackendStep;
+import com.kingsrook.qqq.backend.core.context.QContext;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.MetaDataProducerInterface;
+import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
+import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.automation.QAutomationProviderMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
+import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.processes.QComponentType;
+import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendComponentMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendStepMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.TablesPossibleValueSourceMetaDataProvider;
+import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
+import com.kingsrook.qqq.backend.core.utils.StringUtils;
+
+
+/*******************************************************************************
+ ** Process to manually run table automations, for a table.
+ **
+ ** Useful, maybe, for an e2e test.  Or, if you don't want jobs to be running,
+ ** but want to run automations by-hand, for some reason.
+ **
+ ** In the future, this class could take a param to only do inserts or updates.
+ **
+ ** Also, right now, only records that are Pending automations will be run -
+ ** again, that could be changed, presumably (take a list of records, always run, etc...)
+ *******************************************************************************/
+public class RunTableAutomationsProcessStep implements BackendStep, MetaDataProducerInterface<QProcessMetaData>
+{
+   public static final String NAME = "RunTableAutomationsProcess";
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public QProcessMetaData produce(QInstance qInstance) throws QException
+   {
+      QProcessMetaData processMetaData = new QProcessMetaData()
+         .withName(NAME)
+         .withStepList(List.of(
+            new QFrontendStepMetaData()
+               .withName("input")
+               .withComponent(new QFrontendComponentMetaData().withType(QComponentType.EDIT_FORM))
+               .withFormField(new QFieldMetaData("tableName", QFieldType.STRING).withIsRequired(true).withPossibleValueSourceName(TablesPossibleValueSourceMetaDataProvider.NAME))
+               .withFormField(new QFieldMetaData("automationProviderName", QFieldType.STRING)),
+            new QBackendStepMetaData()
+               .withName("run")
+               .withCode(new QCodeReference(getClass())),
+            new QFrontendStepMetaData()
+               .withName("output")
+               .withComponent(new QFrontendComponentMetaData().withType(QComponentType.VIEW_FORM))
+               .withViewField(new QFieldMetaData("ok", QFieldType.STRING))
+         ));
+
+      return (processMetaData);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException
+   {
+      QInstance qInstance = QContext.getQInstance();
+
+      ////////////////////////////////////////////////////////////////////
+      // get tableName param (since this process is not table-specific) //
+      ////////////////////////////////////////////////////////////////////
+      String tableName = runBackendStepInput.getValueString("tableName");
+      if(!StringUtils.hasContent(tableName))
+      {
+         throw (new QException("Missing required input value: tableName"));
+      }
+
+      if(!QContext.getQInstance().getTables().containsKey(tableName))
+      {
+         throw (new QException("Unrecognized table name: " + tableName));
+      }
+
+      ////////////////////////////////////////////////////////////////////////////////////////////////
+      // get the automation provider name to use - either as the only-one-in-instance, or via param //
+      ////////////////////////////////////////////////////////////////////////////////////////////////
+      String automationProviderName = runBackendStepInput.getValueString("automationProviderName");
+      if(!StringUtils.hasContent(automationProviderName))
+      {
+         Map<String, QAutomationProviderMetaData> automationProviders = CollectionUtils.nonNullMap(qInstance.getAutomationProviders());
+         if(automationProviders.size() == 1)
+         {
+            automationProviderName = automationProviders.keySet().iterator().next();
+         }
+         else
+         {
+            throw (new QException("Missing required input value: automationProviderName (and there is not exactly 1 in the active instance)"));
+         }
+      }
+
+      /////////////////////////////////////////////
+      // run automations for the requested table //
+      /////////////////////////////////////////////
+      List<PollingAutomationPerTableRunner.TableActionsInterface> tableActions = PollingAutomationPerTableRunner.getTableActions(qInstance, automationProviderName);
+      for(PollingAutomationPerTableRunner.TableActionsInterface tableAction : tableActions)
+      {
+         if(tableName.equals(tableAction.tableName()))
+         {
+            PollingAutomationPerTableRunner pollingAutomationPerTableRunner = new PollingAutomationPerTableRunner(qInstance, automationProviderName, () -> QContext.getQSession(), tableAction);
+            pollingAutomationPerTableRunner.processTableInsertOrUpdate(qInstance.getTable(tableAction.tableName()), tableAction.status());
+         }
+      }
+
+      runBackendStepOutput.addValue("ok", "true");
+   }
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/columnstats/ColumnStatsStep.java

@@ -25,6 +25,8 @@ package com.kingsrook.qqq.backend.core.processes.implementations.columnstats;
 import java.io.Serializable;
 import java.math.BigDecimal;
 import java.math.RoundingMode;
+import java.time.Instant;
+import java.time.ZoneId;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.Comparator;
@@ -173,11 +175,10 @@ public class ColumnStatsStep implements BackendStep
          Aggregate aggregate = new Aggregate(table.getPrimaryKeyField(), AggregateOperator.COUNT).withFieldType(QFieldType.DECIMAL);
          GroupBy   groupBy   = new GroupBy(field.getType(), fieldName);
 
-         // todo - something here about "by-date, not time"
+         // todo - something here about an input param to specify how you want dates & date-times grouped
          if(field.getType().equals(QFieldType.DATE_TIME))
          {
-            // groupBy = new GroupBy(field.getType(), fieldName, "DATE(%s)");
-            String sqlExpression = DateTimeGroupBy.HOUR.getSqlExpression();
+            String sqlExpression = DateTimeGroupBy.HOUR.getSqlExpression(ZoneId.systemDefault());
             groupBy = new GroupBy(QFieldType.STRING, fieldName, sqlExpression);
          }
 
@@ -230,6 +231,12 @@ public class ColumnStatsStep implements BackendStep
          for(AggregateResult result : aggregateOutput.getResults())
          {
             Serializable value = result.getGroupByValue(groupBy);
+
+            if(field.getType().equals(QFieldType.DATE_TIME) && value != null)
+            {
+               value = Instant.parse(value + ":00:00Z");
+            }
+
             Integer      count = ValueUtils.getValueAsInteger(result.getAggregateValue(aggregate));
             valueCounts.add(new QRecord().withValue(fieldName, value).withValue("count", count));
          }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/scripts/RunRecordScriptLoadStep.java

@@ -26,7 +26,6 @@ import java.io.Serializable;
 import java.util.ArrayList;
 import java.util.List;
 import com.kingsrook.qqq.backend.core.actions.audits.AuditAction;
-import com.kingsrook.qqq.backend.core.actions.audits.DMLAuditAction;
 import com.kingsrook.qqq.backend.core.actions.scripts.RunAdHocRecordScriptAction;
 import com.kingsrook.qqq.backend.core.actions.scripts.logging.StoreScriptLogAndScriptLogLineExecutionLogger;
 import com.kingsrook.qqq.backend.core.actions.tables.GetAction;
@@ -133,11 +132,6 @@ public class RunRecordScriptLoadStep extends AbstractLoadStep implements Process
          throw (new QException("Could not find script by id: " + scriptId));
       }
 
-      //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-      // set an "audit context" - so any DML executed during the script will include the note of what script was running. //
-      //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-      runBackendStepInput.addValue(DMLAuditAction.AUDIT_CONTEXT_FIELD_NAME, "via Script \"" + script.getValue("name") + "\"");
-
       String tableName = script.getValueString("tableName");
 
       RunAdHocRecordScriptInput input = new RunAdHocRecordScriptInput();

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/ExceptionUtils.java

@@ -164,4 +164,23 @@ public class ExceptionUtils
 
       return (StringUtils.join("; ", messages));
    }
+
+
+
+   /*******************************************************************************
+    ** Get the messages from the top & bottoms (root) of an exception.
+    **
+    ** If there's no root, just return the top (e.g., parameter)'s message.
+    ** If they are both found, put ": " between them.
+    *******************************************************************************/
+   public static String getTopAndBottomMessages(Exception e)
+   {
+      String    rs            = e.getMessage();
+      Throwable rootException = getRootException(e);
+      if(rootException != e)
+      {
+         rs += ": " + rootException.getMessage();
+      }
+      return (rs);
+   }
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/memoization/Memoization.java

@@ -30,6 +30,7 @@ import java.util.LinkedHashMap;
 import java.util.Map;
 import java.util.Optional;
 import com.kingsrook.qqq.backend.core.logging.QLogger;
+import com.kingsrook.qqq.backend.core.utils.lambdas.UnsafeFunction;
 
 
 /*******************************************************************************
@@ -58,8 +59,16 @@ public class Memoization<K, V>
 
 
    /*******************************************************************************
+    ** Get the memoized Value for a given input Key.
     **
+    ** But note, this looks the same to the caller, whether the key just wasn't in
+    ** the internal map (e.g., had never been looked up), or if it was previously looked
+    ** up, and that returned null.  In either case, the optional will be empty.
+    **
+    ** See getMemoizedResult for where we can tell the difference (and we would
+    ** generally want to call that.
     *******************************************************************************/
+   @Deprecated
    public Optional<V> getResult(K key)
    {
       MemoizedResult<V> result = map.get(key);
@@ -77,7 +86,57 @@ public class Memoization<K, V>
 
 
    /*******************************************************************************
+    ** Get the memoized Value for a given input Key - computing it if it wasn't previously
+    ** memoized (or expired).
     **
+    ** In here, if the optional is empty, it means the value is null (whether that
+    ** came form memoization, or from the lookupFunction, you don't care - the answer
+    ** is null).
+    *******************************************************************************/
+   public Optional<V> getResult(K key, UnsafeFunction<K, V, ?> lookupFunction)
+   {
+      MemoizedResult<V> result = map.get(key);
+      if(result != null)
+      {
+         if(result.getTime().isAfter(Instant.now().minus(timeout)))
+         {
+            //////////////////////////////////////////////////////////////////////////////
+            // ok, we have a memoized value, and it's not expired, so we can return it. //
+            // of course, it might be a memoized null, so we use .ofNullable.           //
+            //////////////////////////////////////////////////////////////////////////////
+            return (Optional.ofNullable(result.getResult()));
+         }
+      }
+
+      /////////////////////////////////////////////////////////////////////////////////////////////
+      // ok - either we never memoized this key, or it's expired, so, apply the lookup function, //
+      // store the result, and then return the value (in an Optional.ofNullable)                 //
+      /////////////////////////////////////////////////////////////////////////////////////////////
+      try
+      {
+         V value = lookupFunction.apply(key);
+         storeResult(key, value);
+         return (Optional.ofNullable(value));
+      }
+      catch(Exception e)
+      {
+         LOG.warn("Uncaught Exception while executing a Memoization lookupFunction (to avoid this log, add a catch in the lookupFunction)", e);
+         storeResult(key, null);
+         return (Optional.empty());
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** Get a memoized result, optionally containing a Value, for a given input Key.
+    **
+    ** In this method (contrasted with getResult), if the returned Optional is empty,
+    ** it means that we haven't ever looked up or memoized the key (or it's expired).
+    **
+    ** If the returned Optional is not empty, then it means we've memoized something
+    ** (and it's not expired) - so if the Value from the MemoizedResult is null,
+    ** then null is the proper memoized value.
     *******************************************************************************/
    public Optional<MemoizedResult<V>> getMemoizedResult(K key)
    {
@@ -86,7 +145,7 @@ public class Memoization<K, V>
       {
          if(result.getTime().isAfter(Instant.now().minus(timeout)))
          {
-            return (Optional.ofNullable(result));
+            return (Optional.of(result));
          }
       }
 
@@ -181,4 +240,47 @@ public class Memoization<K, V>
    {
       return map;
    }
+
+
+
+   /*******************************************************************************
+    ** Getter for timeout
+    *******************************************************************************/
+   public Duration getTimeout()
+   {
+      return (this.timeout);
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for timeout
+    *******************************************************************************/
+   public Memoization<K, V> withTimeout(Duration timeout)
+   {
+      this.timeout = timeout;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for maxSize
+    *******************************************************************************/
+   public Integer getMaxSize()
+   {
+      return (this.maxSize);
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for maxSize
+    *******************************************************************************/
+   public Memoization<K, V> withMaxSize(Integer maxSize)
+   {
+      this.maxSize = maxSize;
+      return (this);
+   }
+
 }

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/automation/polling/PollingAutomationPerTableRunnerAutomtationUpdatingSelfAvoidInfiniteLoopTest.java

@@ -0,0 +1,220 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.actions.automation.polling;
+
+
+import java.math.BigDecimal;
+import java.util.List;
+import java.util.Objects;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.ScheduledExecutorService;
+import com.kingsrook.qqq.backend.core.BaseTest;
+import com.kingsrook.qqq.backend.core.actions.automation.AutomationStatus;
+import com.kingsrook.qqq.backend.core.actions.automation.RecordAutomationHandler;
+import com.kingsrook.qqq.backend.core.actions.tables.GetAction;
+import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
+import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction;
+import com.kingsrook.qqq.backend.core.context.CapturedContext;
+import com.kingsrook.qqq.backend.core.context.QContext;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
+import com.kingsrook.qqq.backend.core.model.automation.RecordAutomationInput;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.TableAutomationAction;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.TriggerEvent;
+import com.kingsrook.qqq.backend.core.utils.TestUtils;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import static com.kingsrook.qqq.backend.core.actions.automation.polling.PollingAutomationPerTableRunnerTest.runAllTableActions;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNotEquals;
+
+
+/*******************************************************************************
+ ** Test for the case where:
+ ** - inserting into a main table and a child table, and the child table has a
+ ** post-insert customizer, which mo
+ *******************************************************************************/
+public class PollingAutomationPerTableRunnerAutomtationUpdatingSelfAvoidInfiniteLoopTest extends BaseTest
+{
+   private static boolean didFailInThread = false;
+
+   static
+   {
+      ///////////////////////////////////////////////////////////////////////////////////////////////////
+      // we can set this property to revert to the behavior that existed before this test was written. //
+      ///////////////////////////////////////////////////////////////////////////////////////////////////
+      // System.setProperty("qqq.recordAutomationStatusUpdater.skipPreUpdateFetch", "true");
+   }
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @BeforeEach
+   void beforeEach()
+   {
+      didFailInThread = false;
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void test() throws QException
+   {
+      ////////////////////////////////////
+      // add automations to order table //
+      ////////////////////////////////////
+      QContext.getQInstance().getTable(TestUtils.TABLE_NAME_ORDER)
+         .withField(TestUtils.standardQqqAutomationStatusField())
+         .withAutomationDetails(TestUtils.defineStandardAutomationDetails()
+            .withAction(new TableAutomationAction()
+               .withName("orderPostInsertAction")
+               .withTriggerEvent(TriggerEvent.POST_INSERT)
+               .withCodeReference(new QCodeReference(OrderPostInsertAndUpdateAction.class)))
+            .withAction(new TableAutomationAction()
+               .withName("orderPostUpdateAction")
+               .withTriggerEvent(TriggerEvent.POST_UPDATE)
+               .withCodeReference(new QCodeReference(OrderPostInsertAndUpdateAction.class))));
+
+      QContext.getQSession().withSecurityKeyValue(TestUtils.SECURITY_KEY_TYPE_STORE_ALL_ACCESS, true);
+
+      InsertInput insertInput = new InsertInput();
+      insertInput.setTableName(TestUtils.TABLE_NAME_ORDER);
+      insertInput.setRecords(List.of(new QRecord().withValue("orderNo", "10101").withValue("total", new BigDecimal(1))));
+      new InsertAction().execute(insertInput);
+
+      //////////////////////////////////////////////////////
+      // make sure the order is in pending-inserts status //
+      //////////////////////////////////////////////////////
+      {
+         QRecord order = new GetAction().executeForRecord(new GetInput(TestUtils.TABLE_NAME_ORDER).withPrimaryKey(1));
+         assertEquals(AutomationStatus.PENDING_INSERT_AUTOMATIONS.getId(), order.getValue(TestUtils.standardQqqAutomationStatusField().getName()));
+         assertEquals(new BigDecimal(1), order.getValueBigDecimal("total"));
+      }
+
+      ////////////////////////////////////////////////////////////////////////////////////////////////
+      // run automations - that should update the order via the automation - but leave status as OK //
+      ////////////////////////////////////////////////////////////////////////////////////////////////
+      runAllTableActions(QContext.getQInstance());
+      assertFalse(didFailInThread, "A failure condition happened in the automation sub-thread.  Check System.out for message.");
+
+      {
+         QRecord order = new GetAction().executeForRecord(new GetInput(TestUtils.TABLE_NAME_ORDER).withPrimaryKey(1));
+         assertEquals(AutomationStatus.OK.getId(), order.getValue(TestUtils.standardQqqAutomationStatusField().getName()));
+         assertEquals(new BigDecimal(2), order.getValueBigDecimal("total"));
+      }
+
+      //////////////////////////////////////////////////////////////////
+      // now update the order, verify status moves to pending-updates //
+      //////////////////////////////////////////////////////////////////
+      new UpdateAction().execute(new UpdateInput(TestUtils.TABLE_NAME_ORDER).withRecord(new QRecord()
+         .withValue("id", 1)
+         .withValue("storeId", "x")));
+
+      {
+         QRecord order = new GetAction().executeForRecord(new GetInput(TestUtils.TABLE_NAME_ORDER).withPrimaryKey(1));
+         assertEquals(AutomationStatus.PENDING_UPDATE_AUTOMATIONS.getId(), order.getValue(TestUtils.standardQqqAutomationStatusField().getName()));
+         assertEquals(new BigDecimal(2), order.getValueBigDecimal("total"));
+         assertEquals("x", order.getValueString("storeId"));
+      }
+
+      ////////////////////////////////////////////////////////////////////////////////////////////////
+      // run automations - that should update the order via the automation - but leave status as OK //
+      ////////////////////////////////////////////////////////////////////////////////////////////////
+      runAllTableActions(QContext.getQInstance());
+      assertFalse(didFailInThread, "A failure condition happened in the automation sub-thread.  Check System.out for message.");
+
+      {
+         QRecord order = new GetAction().executeForRecord(new GetInput(TestUtils.TABLE_NAME_ORDER).withPrimaryKey(1));
+         assertEquals(AutomationStatus.OK.getId(), order.getValue(TestUtils.standardQqqAutomationStatusField().getName()));
+         assertEquals(new BigDecimal(3), order.getValueBigDecimal("total"));
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public static class OrderPostInsertAndUpdateAction extends RecordAutomationHandler
+   {
+
+      /*******************************************************************************
+       **
+       *******************************************************************************/
+      @Override
+      public void execute(RecordAutomationInput recordAutomationInput) throws QException
+      {
+         ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+         // launch a new thread, to make sure we avoid the "stack contains automations" check in RecordAutomationStatusUpdater //
+         ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+         CapturedContext capturedContext = QContext.capture();
+         for(QRecord record : recordAutomationInput.getRecordList())
+         {
+            ScheduledExecutorService service = Executors.newSingleThreadScheduledExecutor();
+            Future<?> submit = service.submit(() ->
+            {
+               QContext.init(capturedContext);
+               try
+               {
+                  new UpdateAction().execute(new UpdateInput(TestUtils.TABLE_NAME_ORDER).withRecord(new QRecord()
+                     .withValue("id", record.getValue("id"))
+                     .withValue("total", record.getValueBigDecimal("total").add(new BigDecimal(1)))
+                  ));
+
+                  ///////////////////////////////////////////////////////////////////
+                  // make sure that update action didn't change the order's status //
+                  ///////////////////////////////////////////////////////////////////
+                  QRecord order = new GetAction().executeForRecord(new GetInput(TestUtils.TABLE_NAME_ORDER).withPrimaryKey(1));
+                  if(Objects.equals(AutomationStatus.PENDING_UPDATE_AUTOMATIONS.getId(), order.getValue(TestUtils.standardQqqAutomationStatusField().getName())))
+                  {
+                     System.out.println("Failing test - expected status to not be [PENDING_UPDATE_AUTOMATIONS], but it was.");
+                     didFailInThread = true;
+                  }
+                  assertNotEquals(AutomationStatus.PENDING_UPDATE_AUTOMATIONS.getId(), order.getValue(TestUtils.standardQqqAutomationStatusField().getName()));
+               }
+               catch(QException e)
+               {
+                  e.printStackTrace();
+               }
+               finally
+               {
+                  QContext.clear();
+               }
+            });
+
+            while(!submit.isDone())
+            {
+            }
+         }
+      }
+   }
+
+}

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/automation/polling/PollingAutomationPerTableRunnerChildPostInsertCustomizerTest.java

@@ -0,0 +1,224 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.actions.automation.polling;
+
+
+import java.io.Serializable;
+import java.math.BigDecimal;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.UUID;
+import java.util.stream.Collectors;
+import com.kingsrook.qqq.backend.core.BaseTest;
+import com.kingsrook.qqq.backend.core.actions.automation.AutomationStatus;
+import com.kingsrook.qqq.backend.core.actions.automation.RecordAutomationHandler;
+import com.kingsrook.qqq.backend.core.actions.customizers.AbstractPostInsertCustomizer;
+import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizers;
+import com.kingsrook.qqq.backend.core.actions.tables.AggregateAction;
+import com.kingsrook.qqq.backend.core.actions.tables.GetAction;
+import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
+import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction;
+import com.kingsrook.qqq.backend.core.context.QContext;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.Aggregate;
+import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateOperator;
+import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateOutput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateResult;
+import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.GroupBy;
+import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertOutput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
+import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
+import com.kingsrook.qqq.backend.core.model.automation.RecordAutomationInput;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.TableAutomationAction;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.TriggerEvent;
+import com.kingsrook.qqq.backend.core.utils.TestUtils;
+import com.kingsrook.qqq.backend.core.utils.ValueUtils;
+import org.junit.jupiter.api.Test;
+import static com.kingsrook.qqq.backend.core.actions.automation.polling.PollingAutomationPerTableRunnerTest.runAllTableActions;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+
+/*******************************************************************************
+ ** Test for the case where:
+ ** - inserting into a main table and a child table, and the child table has a
+ ** post-insert customizer, which mo
+ *******************************************************************************/
+public class PollingAutomationPerTableRunnerChildPostInsertCustomizerTest extends BaseTest
+{
+
+   static
+   {
+      ///////////////////////////////////////////////////////////////////////////////////////////////////
+      // we can set this property to revert to the behavior that existed before this test was written. //
+      ///////////////////////////////////////////////////////////////////////////////////////////////////
+      // System.setProperty("qqq.recordAutomationStatusUpdater.skipPreUpdateFetch", "true");
+   }
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void test() throws QException
+   {
+      ////////////////////////////////////
+      // add automations to order table //
+      ////////////////////////////////////
+      QContext.getQInstance().getTable(TestUtils.TABLE_NAME_ORDER)
+         .withField(TestUtils.standardQqqAutomationStatusField())
+         .withAutomationDetails(TestUtils.defineStandardAutomationDetails()
+            .withAction(new TableAutomationAction()
+               .withName("orderPostInsertAction")
+               .withTriggerEvent(TriggerEvent.POST_INSERT)
+               .withCodeReference(new QCodeReference(OrderPostInsertAction.class))
+            ));
+
+      ///////////////////////////////////////////////////////////////////////////
+      // add a post-insert customizer to line-ite table (child of order table) //
+      ///////////////////////////////////////////////////////////////////////////
+      QContext.getQInstance().getTable(TestUtils.TABLE_NAME_LINE_ITEM)
+         .withCustomizer(TableCustomizers.POST_INSERT_RECORD, new QCodeReference(LineItemPostInsertCustomizer.class));
+
+      QContext.getQSession().withSecurityKeyValue(TestUtils.SECURITY_KEY_TYPE_STORE_ALL_ACCESS, true);
+
+      InsertInput insertInput = new InsertInput();
+      insertInput.setTableName(TestUtils.TABLE_NAME_ORDER);
+      insertInput.setRecords(List.of(new QRecord()
+         .withValue("orderNo", "10101")
+         .withAssociatedRecord("orderLine", new QRecord()
+            .withValue("sku", "ABC")
+            .withValue("quantity", 1))));
+      InsertOutput insertOutput = new InsertAction().execute(insertInput);
+
+      ///////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      // make sure the order is in pending-inserts status (at one time, a bug meant that it wouldn't have been...) //
+      ///////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      {
+         QRecord order = new GetAction().executeForRecord(new GetInput(TestUtils.TABLE_NAME_ORDER).withPrimaryKey(1));
+         assertEquals(AutomationStatus.PENDING_INSERT_AUTOMATIONS.getId(), order.getValue(TestUtils.standardQqqAutomationStatusField().getName()));
+         assertEquals(new BigDecimal(1), order.getValueBigDecimal("total"));
+      }
+
+      ///////////////////////////////////////////////////////////////////////////////////////////////
+      // run automations - that should... insert a second line item, but should leave the order in //
+      // automation-status = OK, to avoid perpetual re-running                                     //
+      // the line-item post-inserter should run a second time, making the order's total = 2        //
+      ///////////////////////////////////////////////////////////////////////////////////////////////
+      runAllTableActions(QContext.getQInstance());
+
+      {
+         QRecord order = new GetAction().executeForRecord(new GetInput(TestUtils.TABLE_NAME_ORDER).withPrimaryKey(1));
+         assertEquals(AutomationStatus.OK.getId(), order.getValue(TestUtils.standardQqqAutomationStatusField().getName()));
+         assertEquals(new BigDecimal(2), order.getValueBigDecimal("total"));
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public static class OrderPostInsertAction extends RecordAutomationHandler
+   {
+
+      /*******************************************************************************
+       **
+       *******************************************************************************/
+      @Override
+      public void execute(RecordAutomationInput recordAutomationInput) throws QException
+      {
+         ///////////////////////////////////////
+         // add a new line item to the orders //
+         ///////////////////////////////////////
+         List<QRecord> lineItemsToInsert = new ArrayList<>();
+         for(QRecord record : recordAutomationInput.getRecordList())
+         {
+            lineItemsToInsert.add(new QRecord()
+               .withValue("orderId", record.getValue("id"))
+               .withValue("sku", UUID.randomUUID())
+               .withValue("quantity", 1)
+            );
+         }
+         new InsertAction().execute(new InsertInput(TestUtils.TABLE_NAME_LINE_ITEM).withRecords(lineItemsToInsert));
+
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public static class LineItemPostInsertCustomizer extends AbstractPostInsertCustomizer
+   {
+      @Override
+      public List<QRecord> apply(List<QRecord> records) throws QException
+      {
+         //////////////////////////////////
+         // count line items by order id //
+         //////////////////////////////////
+         Set<Serializable> orderIds = records.stream().map(r -> r.getValue("orderId")).collect(Collectors.toSet());
+
+         GroupBy   groupByOrderId = new GroupBy(QFieldType.STRING, "orderId");
+         Aggregate countId        = new Aggregate("id", AggregateOperator.COUNT);
+
+         AggregateInput aggregateInput = new AggregateInput();
+         aggregateInput.setTableName(TestUtils.TABLE_NAME_LINE_ITEM);
+         aggregateInput.setFilter(new QQueryFilter(new QFilterCriteria("orderId", QCriteriaOperator.IN, orderIds)));
+         aggregateInput.withGroupBy(groupByOrderId);
+         aggregateInput.withAggregate(countId);
+         AggregateOutput       aggregateOutput = new AggregateAction().execute(aggregateInput);
+         Map<Integer, Integer> countByOrderId  = new HashMap<>();
+         for(AggregateResult result : aggregateOutput.getResults())
+         {
+            countByOrderId.put(ValueUtils.getValueAsInteger(result.getGroupByValue(groupByOrderId)), ValueUtils.getValueAsInteger(result.getAggregateValue(countId)));
+         }
+
+         ///////////////////////////////////
+         // update the order total fields //
+         // s/b in bulk, but, meh         //
+         ///////////////////////////////////
+         for(Integer orderId : countByOrderId.keySet())
+         {
+            UpdateInput updateInput = new UpdateInput();
+            updateInput.setTableName(TestUtils.TABLE_NAME_ORDER);
+            updateInput.setRecords(List.of(new QRecord()
+               .withValue("id", orderId)
+               .withValue("total", new BigDecimal(countByOrderId.get(orderId)))));
+            new UpdateAction().execute(updateInput);
+         }
+
+         return (records);
+      }
+   }
+
+}

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/automation/polling/PollingAutomationPerTableRunnerTest.java

@@ -56,14 +56,11 @@ import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.QTableAut
 import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.TableAutomationAction;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.TriggerEvent;
 import com.kingsrook.qqq.backend.core.model.session.QSession;
-import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryRecordStore;
 import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.ExtractViaQueryStep;
 import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.LoadViaInsertStep;
 import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess;
 import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcessTest;
 import com.kingsrook.qqq.backend.core.utils.TestUtils;
-import org.junit.jupiter.api.AfterEach;
-import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
 import static org.assertj.core.api.Assertions.assertThat;
 import static org.assertj.core.api.Assertions.assertThatThrownBy;
@@ -77,18 +74,6 @@ import static org.junit.jupiter.api.Assertions.assertFalse;
 class PollingAutomationPerTableRunnerTest extends BaseTest
 {
 
-   /*******************************************************************************
-    **
-    *******************************************************************************/
-   @BeforeEach
-   @AfterEach
-   void beforeAndAfterEach()
-   {
-      MemoryRecordStore.getInstance().reset();
-   }
-
-
-
    /*******************************************************************************
     ** Test a cycle that does an insert, some automations, then and an update, and more automations.
     *******************************************************************************/
@@ -200,7 +185,7 @@ class PollingAutomationPerTableRunnerTest extends BaseTest
    /*******************************************************************************
     **
     *******************************************************************************/
-   private void runAllTableActions(QInstance qInstance) throws QException
+   static void runAllTableActions(QInstance qInstance) throws QException
    {
       List<PollingAutomationPerTableRunner.TableActionsInterface> tableActions = PollingAutomationPerTableRunner.getTableActions(qInstance, TestUtils.POLLING_AUTOMATION);
       for(PollingAutomationPerTableRunner.TableActionsInterface tableAction : tableActions)
@@ -210,7 +195,7 @@ class PollingAutomationPerTableRunnerTest extends BaseTest
          /////////////////////////////////////////////////////////////////////////////////////////////////////
          // note - don't call run - it is meant to be called async - e.g., it sets & clears thread context. //
          /////////////////////////////////////////////////////////////////////////////////////////////////////
-         pollingAutomationPerTableRunner.processTableInsertOrUpdate(qInstance.getTable(tableAction.tableName()), QContext.getQSession(), tableAction.status());
+         pollingAutomationPerTableRunner.processTableInsertOrUpdate(qInstance.getTable(tableAction.tableName()), tableAction.status());
       }
    }
 
@@ -512,7 +497,7 @@ class PollingAutomationPerTableRunnerTest extends BaseTest
             /////////////////////////////////////////////////////////////////////////////////////////////////////
             // note - don't call run - it is meant to be called async - e.g., it sets & clears thread context. //
             /////////////////////////////////////////////////////////////////////////////////////////////////////
-            pollingAutomationPerTableRunner.processTableInsertOrUpdate(qInstance.getTable(tableAction.tableName()), QContext.getQSession(), tableAction.status());
+            pollingAutomationPerTableRunner.processTableInsertOrUpdate(qInstance.getTable(tableAction.tableName()), tableAction.status());
          }
       }).hasMessage(PollingAutomationPerTableRunnerThatShouldSimulateServerShutdownMidRun.EXCEPTION_MESSAGE);
 

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/data/QRecordTest.java

@@ -26,6 +26,7 @@ import java.math.BigDecimal;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.LinkedHashMap;
+import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
 import com.kingsrook.qqq.backend.core.BaseTest;
@@ -175,7 +176,7 @@ class QRecordTest extends BaseTest
     **
     *******************************************************************************/
    @Test
-   void testListAsValue()
+   void testArrayListAsValue()
    {
       ArrayList<Integer> originalArrayList        = new ArrayList<>(List.of(1, 2, 3));
       QRecord            recordWithArrayListValue = new QRecord().withValue("myList", originalArrayList);
@@ -196,6 +197,31 @@ class QRecordTest extends BaseTest
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testLinkedListAsValue()
+   {
+      LinkedList<Integer> originalLinkedList        = new LinkedList<>(List.of(1, 2, 3));
+      QRecord             recordWithLinkedListValue = new QRecord().withValue("myList", originalLinkedList);
+      QRecord             cloneWithLinkedListValue  = new QRecord(recordWithLinkedListValue);
+
+      ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      // the clone list and original list should be equals (have contents that are equals), but not be the same (reference) //
+      ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      assertEquals(List.of(1, 2, 3), cloneWithLinkedListValue.getValue("myList"));
+      assertNotSame(originalLinkedList, cloneWithLinkedListValue.getValue("myList"));
+
+      //////////////////////////////////////////////////////////////////////////////////////////////////////
+      // make sure a change to the original list doesn't change the cloned list (as it was cloned deeply) //
+      //////////////////////////////////////////////////////////////////////////////////////////////////////
+      originalLinkedList.add(4);
+      assertNotEquals(originalLinkedList, cloneWithLinkedListValue.getValue("myList"));
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/automation/HealBadRecordAutomationStatusesProcessStepTest.java

@@ -0,0 +1,204 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.processes.implementations.automation;
+
+
+import java.time.Instant;
+import java.time.temporal.ChronoUnit;
+import java.util.List;
+import java.util.Map;
+import com.kingsrook.qqq.backend.core.BaseTest;
+import com.kingsrook.qqq.backend.core.actions.automation.AutomationStatus;
+import com.kingsrook.qqq.backend.core.actions.automation.RecordAutomationStatusUpdater;
+import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
+import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
+import com.kingsrook.qqq.backend.core.context.QContext;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
+import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.DynamicDefaultValueBehavior;
+import com.kingsrook.qqq.backend.core.utils.TestUtils;
+import org.junit.jupiter.api.Test;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+
+/*******************************************************************************
+ ** Unit test for HealBadRecordAutomationStatusesProcessStep 
+ *******************************************************************************/
+class HealBadRecordAutomationStatusesProcessStepTest extends BaseTest
+{
+   private static String tableName = TestUtils.TABLE_NAME_PERSON_MEMORY;
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testTwoFailedUpdates() throws QException
+   {
+      new InsertAction().execute(new InsertInput(tableName).withRecords(List.of(new QRecord(), new QRecord())));
+      List<QRecord> records = queryAllRecords();
+      RecordAutomationStatusUpdater.setAutomationStatusInRecordsAndUpdate(QContext.getQInstance().getTable(tableName), records, AutomationStatus.FAILED_UPDATE_AUTOMATIONS, null);
+
+      assertThat(queryAllRecords()).allMatch(r -> AutomationStatus.FAILED_UPDATE_AUTOMATIONS.getId().equals(getAutomationStatus(r)));
+
+      RunBackendStepOutput output = runProcessStep();
+
+      assertEquals(2, output.getValueInteger("totalRecordsUpdated"));
+      assertThat(queryAllRecords()).allMatch(r -> AutomationStatus.PENDING_UPDATE_AUTOMATIONS.getId().equals(getAutomationStatus(r)));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testOneFailedUpdateOneFailedInsert() throws QException
+   {
+      new InsertAction().execute(new InsertInput(tableName).withRecords(List.of(new QRecord(), new QRecord())));
+      List<QRecord> records = queryAllRecords();
+      RecordAutomationStatusUpdater.setAutomationStatusInRecordsAndUpdate(QContext.getQInstance().getTable(tableName), records.subList(0, 1), AutomationStatus.FAILED_UPDATE_AUTOMATIONS, null);
+      RecordAutomationStatusUpdater.setAutomationStatusInRecordsAndUpdate(QContext.getQInstance().getTable(tableName), records.subList(1, 2), AutomationStatus.FAILED_INSERT_AUTOMATIONS, null);
+
+      assertThat(queryAllRecords())
+         .anyMatch(r -> AutomationStatus.FAILED_UPDATE_AUTOMATIONS.getId().equals(getAutomationStatus(r)))
+         .anyMatch(r -> AutomationStatus.FAILED_INSERT_AUTOMATIONS.getId().equals(getAutomationStatus(r)));
+
+      RunBackendStepOutput output = runProcessStep();
+
+      assertEquals(2, output.getValueInteger("totalRecordsUpdated"));
+      assertThat(queryAllRecords())
+         .anyMatch(r -> AutomationStatus.PENDING_UPDATE_AUTOMATIONS.getId().equals(getAutomationStatus(r)))
+         .anyMatch(r -> AutomationStatus.PENDING_INSERT_AUTOMATIONS.getId().equals(getAutomationStatus(r)));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testOldRunning() throws QException
+   {
+      /////////////////////////////////////////////////
+      // temporarily remove the modify-date behavior //
+      /////////////////////////////////////////////////
+      QContext.getQInstance().getTable(tableName).getField("modifyDate").withBehavior(DynamicDefaultValueBehavior.NONE);
+
+      //////////////////////////////////////////////////////////////////////////
+      // insert 2 records, one with an old modifyDate, one with 6 minutes ago //
+      //////////////////////////////////////////////////////////////////////////
+      new InsertAction().execute(new InsertInput(tableName).withRecords(List.of(
+         new QRecord().withValue("firstName", "Darin").withValue("modifyDate", Instant.parse("2023-01-01T12:00:00Z")),
+         new QRecord().withValue("firstName", "Tim").withValue("modifyDate", Instant.now().minus(6, ChronoUnit.MINUTES))
+      )));
+      List<QRecord> records = queryAllRecords();
+
+      ///////////////////////////////////////////////////////
+      // put those records both in status: running-updates //
+      ///////////////////////////////////////////////////////
+      RecordAutomationStatusUpdater.setAutomationStatusInRecordsAndUpdate(QContext.getQInstance().getTable(tableName), records, AutomationStatus.RUNNING_UPDATE_AUTOMATIONS, null);
+
+      assertThat(queryAllRecords())
+         .allMatch(r -> AutomationStatus.RUNNING_UPDATE_AUTOMATIONS.getId().equals(getAutomationStatus(r)));
+
+      /////////////////////////////////////
+      // restore the modifyDate behavior //
+      /////////////////////////////////////
+      QContext.getQInstance().getTable(tableName).getField("modifyDate").withBehavior(DynamicDefaultValueBehavior.MODIFY_DATE);
+
+      /////////////////////////
+      // run code under test //
+      /////////////////////////
+      RunBackendStepOutput output = runProcessStep();
+
+      /////////////////////////////////////////////////////////////////////////////////////////////
+      // assert we updated 1 (the old one) to pending-updates, the other left as running-updates //
+      /////////////////////////////////////////////////////////////////////////////////////////////
+      assertEquals(1, output.getValueInteger("totalRecordsUpdated"));
+      assertThat(queryAllRecords())
+         .anyMatch(r -> AutomationStatus.PENDING_UPDATE_AUTOMATIONS.getId().equals(getAutomationStatus(r)))
+         .anyMatch(r -> AutomationStatus.RUNNING_UPDATE_AUTOMATIONS.getId().equals(getAutomationStatus(r)));
+
+      /////////////////////////////////
+      // re-run, with 3-minute limit //
+      /////////////////////////////////
+      output = runProcessStep(new RunBackendStepInput().withValues(Map.of("minutesOldLimit", 3)));
+
+      /////////////////////////////////////////////////////////////////
+      // assert that one updated too, and all are now pending-update //
+      /////////////////////////////////////////////////////////////////
+      assertEquals(1, output.getValueInteger("totalRecordsUpdated"));
+      assertThat(queryAllRecords())
+         .allMatch(r -> AutomationStatus.PENDING_UPDATE_AUTOMATIONS.getId().equals(getAutomationStatus(r)));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private static Integer getAutomationStatus(QRecord r)
+   {
+      return r.getValueInteger(TestUtils.standardQqqAutomationStatusField().getName());
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private static List<QRecord> queryAllRecords() throws QException
+   {
+      return new QueryAction().execute(new QueryInput(tableName)).getRecords();
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private static RunBackendStepOutput runProcessStep() throws QException
+   {
+      RunBackendStepInput input = new RunBackendStepInput();
+      return runProcessStep(input);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private static RunBackendStepOutput runProcessStep(RunBackendStepInput input) throws QException
+   {
+      RunBackendStepOutput output = new RunBackendStepOutput();
+      new HealBadRecordAutomationStatusesProcessStep().run(input, output);
+      return output;
+   }
+
+}

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/automation/RunTableAutomationsProcessStepTest.java

@@ -0,0 +1,97 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.processes.implementations.automation;
+
+
+import com.kingsrook.qqq.backend.core.BaseTest;
+import com.kingsrook.qqq.backend.core.actions.automation.AutomationStatus;
+import com.kingsrook.qqq.backend.core.actions.tables.GetAction;
+import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
+import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.utils.TestUtils;
+import com.kingsrook.qqq.backend.core.utils.lambdas.UnsafeSupplier;
+import org.junit.jupiter.api.Test;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+
+/*******************************************************************************
+ ** Unit test for RunTableAutomationsProcessStep 
+ *******************************************************************************/
+class RunTableAutomationsProcessStepTest extends BaseTest
+{
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void test() throws Exception
+   {
+      UnsafeSupplier<Integer, ?> getAutomationStatus = () -> new GetAction().executeForRecord(new GetInput(TestUtils.TABLE_NAME_PERSON_MEMORY).withPrimaryKey(1)).getValueInteger("qqqAutomationStatus");
+
+      new InsertAction().execute(new InsertInput(TestUtils.TABLE_NAME_PERSON_MEMORY).withRecord(new QRecord()));
+      assertEquals(AutomationStatus.PENDING_INSERT_AUTOMATIONS.getId(), getAutomationStatus.get());
+
+      RunBackendStepInput input = new RunBackendStepInput();
+      input.addValue("tableName", TestUtils.TABLE_NAME_PERSON_MEMORY);
+      RunBackendStepOutput output = new RunBackendStepOutput();
+      new RunTableAutomationsProcessStep().run(input, output);
+      assertEquals("true", output.getValue("ok"));
+
+      assertEquals(AutomationStatus.OK.getId(), getAutomationStatus.get());
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testThrowsWithoutTableName() throws QException
+   {
+      RunBackendStepInput  input  = new RunBackendStepInput();
+      RunBackendStepOutput output = new RunBackendStepOutput();
+      assertThatThrownBy(() -> new RunTableAutomationsProcessStep().run(input, output))
+         .hasMessageContaining("Missing required input value: tableName");
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testThrowsWithInvalidTableName() throws QException
+   {
+      RunBackendStepInput  input  = new RunBackendStepInput();
+      RunBackendStepOutput output = new RunBackendStepOutput();
+      input.addValue("tableName", "asdf");
+      assertThatThrownBy(() -> new RunTableAutomationsProcessStep().run(input, output))
+         .hasMessageContaining("Unrecognized table name: asdf");
+   }
+
+}

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/columnstats/ColumnStatsStepTest.java

@@ -24,6 +24,7 @@ package com.kingsrook.qqq.backend.core.processes.implementations.columnstats;
 
 import java.io.Serializable;
 import java.math.BigDecimal;
+import java.time.Instant;
 import java.util.List;
 import java.util.Map;
 import com.kingsrook.qqq.backend.core.BaseTest;
@@ -91,4 +92,50 @@ class ColumnStatsStepTest extends BaseTest
          .hasFieldOrPropertyWithValue("percent", new BigDecimal("16.67"));
    }
 
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testDateTimesRollupByHour() throws QException
+   {
+      InsertInput insertInput = new InsertInput();
+      insertInput.setTableName(TestUtils.TABLE_NAME_PERSON_MEMORY);
+      insertInput.setRecords(List.of(
+         new QRecord().withValue("timestamp", Instant.parse("2024-01-31T09:59:01Z")),
+         new QRecord().withValue("timestamp", Instant.parse("2024-01-31T09:59:59Z")),
+         new QRecord().withValue("timestamp", Instant.parse("2024-01-31T10:00:00Z")),
+         new QRecord().withValue("timestamp", Instant.parse("2024-01-31T10:01:01Z")),
+         new QRecord().withValue("timestamp", Instant.parse("2024-01-31T10:59:59Z")),
+         new QRecord().withValue("timestamp", null)
+      ));
+      new InsertAction().execute(insertInput);
+
+      RunBackendStepInput input = new RunBackendStepInput();
+      input.addValue("tableName", TestUtils.TABLE_NAME_PERSON_MEMORY);
+      input.addValue("fieldName", "timestamp");
+      input.addValue("orderBy", "count.desc");
+
+      RunBackendStepOutput output = new RunBackendStepOutput();
+      new ColumnStatsStep().run(input, output);
+
+      Map<String, Serializable> values = output.getValues();
+
+      @SuppressWarnings("unchecked")
+      List<QRecord> valueCounts = (List<QRecord>) values.get("valueCounts");
+
+      assertThat(valueCounts.get(0).getValues())
+         .hasFieldOrPropertyWithValue("timestamp", Instant.parse("2024-01-31T10:00:00Z"))
+         .hasFieldOrPropertyWithValue("count", 3);
+
+      assertThat(valueCounts.get(1).getValues())
+         .hasFieldOrPropertyWithValue("timestamp", Instant.parse("2024-01-31T09:00:00Z"))
+         .hasFieldOrPropertyWithValue("count", 2);
+
+      assertThat(valueCounts.get(2).getValues())
+         .hasFieldOrPropertyWithValue("timestamp", null)
+         .hasFieldOrPropertyWithValue("count", 1);
+   }
+
 }

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/ExceptionUtilsTest.java

@@ -116,6 +116,19 @@ class ExceptionUtilsTest extends BaseTest
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testGetTopAndBottomMessages()
+   {
+      assertEquals("foo", ExceptionUtils.getTopAndBottomMessages(new Exception("foo")));
+      assertEquals("foo: bar", ExceptionUtils.getTopAndBottomMessages(new Exception("foo", new Exception("bar"))));
+      assertEquals("foo: baz", ExceptionUtils.getTopAndBottomMessages(new Exception("foo", new Exception("bar", new Exception("baz")))));
+   }
+
+
+
    /*******************************************************************************
     ** Test exception class - lets you set the cause, easier to create a loop.
     *******************************************************************************/

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java

@@ -720,7 +720,7 @@ public class TestUtils
    /*******************************************************************************
     **
     *******************************************************************************/
-   private static QTableAutomationDetails defineStandardAutomationDetails()
+   public static QTableAutomationDetails defineStandardAutomationDetails()
    {
       return (new QTableAutomationDetails()
          .withProviderName(POLLING_AUTOMATION)

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/memoization/MemoizationTest.java

@@ -32,11 +32,14 @@ import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
 import java.util.concurrent.Future;
 import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
 import com.kingsrook.qqq.backend.core.BaseTest;
 import com.kingsrook.qqq.backend.core.utils.SleepUtils;
+import com.kingsrook.qqq.backend.core.utils.lambdas.UnsafeFunction;
 import org.junit.jupiter.api.Disabled;
 import org.junit.jupiter.api.Test;
 import static org.assertj.core.api.Assertions.assertThat;
+import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertNotNull;
 import static org.junit.jupiter.api.Assertions.assertNull;
 import static org.junit.jupiter.api.Assertions.assertTrue;
@@ -122,6 +125,58 @@ class MemoizationTest extends BaseTest
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testLookupFunction()
+   {
+      AtomicInteger lookupFunctionCallCounter = new AtomicInteger(0);
+
+      Memoization<String, Integer> memoization = new Memoization<>();
+
+      UnsafeFunction<String, Integer, Exception> lookupFunction = numberString ->
+      {
+         lookupFunctionCallCounter.getAndIncrement();
+
+         if(numberString.equals("null"))
+         {
+            return (null);
+         }
+
+         return Integer.parseInt(numberString);
+      };
+
+      //////////////////////////////////////////////////////////////////////////////////////////
+      // get "1" twice - should return 1 each time, and call the lookup function exactly once //
+      //////////////////////////////////////////////////////////////////////////////////////////
+      assertThat(memoization.getResult("1", lookupFunction)).isPresent().contains(1);
+      assertEquals(1, lookupFunctionCallCounter.get());
+
+      assertThat(memoization.getResult("1", lookupFunction)).isPresent().contains(1);
+      assertEquals(1, lookupFunctionCallCounter.get());
+
+      /////////////////////////////////////////////////////////////////////////////////////////////////////////
+      // now get "null" twice - should return null each time, and call the lookup function exactly once more //
+      /////////////////////////////////////////////////////////////////////////////////////////////////////////
+      assertThat(memoization.getResult("null", lookupFunction)).isEmpty();
+      assertEquals(2, lookupFunctionCallCounter.get());
+
+      assertThat(memoization.getResult("null", lookupFunction)).isEmpty();
+      assertEquals(2, lookupFunctionCallCounter.get());
+
+      ////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      // now make a call that throws twice - again, should return null each time, and only do one more loookup call //
+      ////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      assertThat(memoization.getResult(null, lookupFunction)).isEmpty();
+      assertEquals(3, lookupFunctionCallCounter.get());
+
+      assertThat(memoization.getResult(null, lookupFunction)).isEmpty();
+      assertEquals(3, lookupFunctionCallCounter.get());
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/

qqq-middleware-api/src/main/java/com/kingsrook/qqq/api/actions/ApiImplementation.java

@@ -992,6 +992,8 @@ public class ApiImplementation
       runProcessInput.setProcessUUID(processUUID);
       // todo i don't think runProcessInput.setAsyncJobCallback();
 
+      PermissionsHelper.checkProcessPermissionThrowing(runProcessInput, processName);
+
       //////////////////////
       // map input values //
       //////////////////////

qqq-middleware-api/src/test/java/com/kingsrook/qqq/api/javalin/QJavalinApiHandlerPermissionsTest.java

@@ -0,0 +1,138 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2023.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.api.javalin;
+
+
+import com.kingsrook.qqq.api.BaseTest;
+import com.kingsrook.qqq.api.TestUtils;
+import com.kingsrook.qqq.api.actions.ApiImplementation;
+import com.kingsrook.qqq.backend.core.exceptions.QInstanceValidationException;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.permissions.PermissionLevel;
+import com.kingsrook.qqq.backend.core.model.metadata.permissions.QPermissionRules;
+import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+import com.kingsrook.qqq.backend.javalin.QJavalinImplementation;
+import io.javalin.apibuilder.EndpointGroup;
+import kong.unirest.Unirest;
+import org.eclipse.jetty.http.HttpStatus;
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import static com.kingsrook.qqq.api.javalin.QJavalinApiHandlerTest.assertErrorResponse;
+
+
+/*******************************************************************************
+ ** Unit test permissions within QJavalinApiHandler
+ *******************************************************************************/
+class QJavalinApiHandlerPermissionsTest extends BaseTest
+{
+   private static final   int    PORT     = 6263;
+   protected static final String BASE_URL = "http://localhost:" + PORT;
+
+   private static final String VERSION = "2023.Q1";
+
+   protected static QJavalinImplementation qJavalinImplementation;
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @BeforeAll
+   static void beforeAll() throws QInstanceValidationException
+   {
+      QInstance qInstance = TestUtils.defineInstance();
+
+      ///////////////////////////////////////////////////
+      // turn on permissions on all tables & processes //
+      ///////////////////////////////////////////////////
+      for(QTableMetaData table : qInstance.getTables().values())
+      {
+         table.withPermissionRules(new QPermissionRules().withLevel(PermissionLevel.READ_INSERT_EDIT_DELETE_PERMISSIONS));
+      }
+
+      for(QProcessMetaData process : qInstance.getProcesses().values())
+      {
+         process.withPermissionRules(new QPermissionRules().withLevel(PermissionLevel.HAS_ACCESS_PERMISSION));
+      }
+
+      qJavalinImplementation = new QJavalinImplementation(qInstance);
+      qJavalinImplementation.startJavalinServer(PORT);
+      EndpointGroup routes = new QJavalinApiHandler(qInstance).getRoutes();
+      qJavalinImplementation.getJavalinService().routes(routes);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @BeforeEach
+   @AfterEach
+   void beforeAndAfterEach()
+   {
+      ApiImplementation.clearCaches();
+   }
+
+
+
+   /*******************************************************************************
+    ** Before the class (all) runs, start a javalin server.
+    **
+    *******************************************************************************/
+   @AfterAll
+   static void afterAll()
+   {
+      qJavalinImplementation.stopJavalinServer();
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void test403s()
+   {
+      ////////////////////////////
+      // tables - single & bulk //
+      ////////////////////////////
+      assertErrorResponse(HttpStatus.FORBIDDEN_403, "You do not have permission", Unirest.get(BASE_URL + "/api/" + VERSION + "/order/query").asString());
+      assertErrorResponse(HttpStatus.FORBIDDEN_403, "You do not have permission", Unirest.get(BASE_URL + "/api/" + VERSION + "/order/1").asString());
+      assertErrorResponse(HttpStatus.FORBIDDEN_403, "You do not have permission", Unirest.post(BASE_URL + "/api/" + VERSION + "/person").asString());
+      assertErrorResponse(HttpStatus.FORBIDDEN_403, "You do not have permission", Unirest.post(BASE_URL + "/api/" + VERSION + "/person/bulk").asString());
+      assertErrorResponse(HttpStatus.FORBIDDEN_403, "You do not have permission", Unirest.delete(BASE_URL + "/api/" + VERSION + "/person/1").asString());
+      assertErrorResponse(HttpStatus.FORBIDDEN_403, "You do not have permission", Unirest.delete(BASE_URL + "/api/" + VERSION + "/person/bulk").asString());
+      assertErrorResponse(HttpStatus.FORBIDDEN_403, "You do not have permission", Unirest.patch(BASE_URL + "/api/" + VERSION + "/person/1").asString());
+      assertErrorResponse(HttpStatus.FORBIDDEN_403, "You do not have permission", Unirest.patch(BASE_URL + "/api/" + VERSION + "/person/bulk").asString());
+
+      ///////////////
+      // processes //
+      ///////////////
+      assertErrorResponse(HttpStatus.FORBIDDEN_403, "You do not have permission", Unirest.get(BASE_URL + "/api/" + VERSION + "/person/getPersonInfo").asString());
+      assertErrorResponse(HttpStatus.FORBIDDEN_403, "You do not have permission", Unirest.post(BASE_URL + "/api/" + VERSION + "/person/transformPeople").asString());
+   }
+
+}

qqq-middleware-api/src/test/java/com/kingsrook/qqq/api/javalin/QJavalinApiHandlerTest.java

@@ -1548,7 +1548,7 @@ class QJavalinApiHandlerTest extends BaseTest
    /*******************************************************************************
     **
     *******************************************************************************/
-   private void assertErrorResponse(Integer expectedStatusCode, String expectedErrorMessage, HttpResponse<String> response)
+   static void assertErrorResponse(Integer expectedStatusCode, String expectedErrorMessage, HttpResponse<String> response)
    {
       if(expectedStatusCode != null)
       {