Increasing logging for s3 importer archive errors

CE-798 - Add de-duplication of (some) redundant criteria in dashboard…

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/audits/DMLAuditAction.java

@@ -78,6 +78,7 @@ public class DMLAuditAction extends AbstractQActionFunction<DMLAuditInput, DMLAu
    private static Set<String> loggedUnauditableTableNames = new HashSet<>();
 
 
+
    /*******************************************************************************
     **
     *******************************************************************************/
@@ -210,6 +211,19 @@ public class DMLAuditAction extends AbstractQActionFunction<DMLAuditInput, DMLAu
          contextSuffix.append(" ").append(input.getAuditContext());
       }
 
+      //////////////////////////////////////////////////////////////
+      // look for a context value place directly into the session //
+      //////////////////////////////////////////////////////////////
+      QSession qSession = QContext.getQSession();
+      if(qSession != null)
+      {
+         String sessionContext = qSession.getValue(AUDIT_CONTEXT_FIELD_NAME);
+         if(StringUtils.hasContent(sessionContext))
+         {
+            contextSuffix.append(" ").append(sessionContext);
+         }
+      }
+
       /////////////////////////////////////////////////////////////////////////////////////
       // note process label (and a possible context from the process's state) if present //
       /////////////////////////////////////////////////////////////////////////////////////
@@ -233,17 +247,20 @@ public class DMLAuditAction extends AbstractQActionFunction<DMLAuditInput, DMLAu
       ///////////////////////////////////////////////////
       // use api label & version if present in session //
       ///////////////////////////////////////////////////
-      QSession qSession   = QContext.getQSession();
-      String   apiVersion = qSession.getValue("apiVersion");
-      if(apiVersion != null)
+      if(qSession != null)
       {
-         String apiLabel = qSession.getValue("apiLabel");
-         if(!StringUtils.hasContent(apiLabel))
+         String apiVersion = qSession.getValue("apiVersion");
+         if(apiVersion != null)
          {
-            apiLabel = "API";
+            String apiLabel = qSession.getValue("apiLabel");
+            if(!StringUtils.hasContent(apiLabel))
+            {
+               apiLabel = "API";
+            }
+            contextSuffix.append(" via ").append(apiLabel).append(" Version: ").append(apiVersion);
          }
-         contextSuffix.append(" via ").append(apiLabel).append(" Version: ").append(apiVersion);
       }
+
       return (contextSuffix.toString());
    }
 

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/automation/AutomationStatus.java

@@ -22,6 +22,7 @@
 package com.kingsrook.qqq.backend.core.actions.automation;
 
 
+import java.util.Objects;
 import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.PossibleValueEnum;
 
 
@@ -55,6 +56,30 @@ public enum AutomationStatus implements PossibleValueEnum<Integer>
 
 
 
+   /*******************************************************************************
+    ** Get instance by id
+    **
+    *******************************************************************************/
+   public static AutomationStatus getById(Integer id)
+   {
+      if(id == null)
+      {
+         return (null);
+      }
+
+      for(AutomationStatus value : AutomationStatus.values())
+      {
+         if(Objects.equals(value.id, id))
+         {
+            return (value);
+         }
+      }
+
+      return (null);
+   }
+
+
+
    /*******************************************************************************
     ** Getter for id
     **
@@ -106,10 +131,10 @@ public enum AutomationStatus implements PossibleValueEnum<Integer>
    public String getInsertOrUpdate()
    {
       return switch(this)
-         {
-            case PENDING_INSERT_AUTOMATIONS, RUNNING_INSERT_AUTOMATIONS, FAILED_INSERT_AUTOMATIONS -> "Insert";
-            case PENDING_UPDATE_AUTOMATIONS, RUNNING_UPDATE_AUTOMATIONS, FAILED_UPDATE_AUTOMATIONS -> "Update";
-            case OK -> "";
-         };
+      {
+         case PENDING_INSERT_AUTOMATIONS, RUNNING_INSERT_AUTOMATIONS, FAILED_INSERT_AUTOMATIONS -> "Insert";
+         case PENDING_UPDATE_AUTOMATIONS, RUNNING_UPDATE_AUTOMATIONS, FAILED_UPDATE_AUTOMATIONS -> "Update";
+         case OK -> "";
+      };
    }
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/automation/RecordAutomationStatusUpdater.java

@@ -22,30 +22,40 @@
 package com.kingsrook.qqq.backend.core.actions.automation;
 
 
+import java.io.Serializable;
+import java.time.Duration;
+import java.time.temporal.ChronoUnit;
 import java.util.Collections;
+import java.util.HashSet;
 import java.util.List;
+import java.util.Optional;
+import java.util.Set;
+import com.kingsrook.qqq.backend.core.actions.QBackendTransaction;
 import com.kingsrook.qqq.backend.core.actions.tables.CountAction;
+import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
 import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction;
 import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.instances.QMetaDataVariableInterpreter;
 import com.kingsrook.qqq.backend.core.logging.QLogger;
 import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountOutput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
 import com.kingsrook.qqq.backend.core.model.automation.TableTrigger;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
-import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.AutomationStatusTrackingType;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.QTableAutomationDetails;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.TableAutomationAction;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.TriggerEvent;
-import com.kingsrook.qqq.backend.core.model.session.QSession;
 import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
+import com.kingsrook.qqq.backend.core.utils.memoization.Memoization;
 import org.apache.commons.lang.NotImplementedException;
+import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
 
 
 /*******************************************************************************
@@ -55,19 +65,37 @@ public class RecordAutomationStatusUpdater
 {
    private static final QLogger LOG = QLogger.getLogger(RecordAutomationStatusUpdater.class);
 
+   ///////////////////////////////////////////////////////////////////////////////////////////////////////
+   // feature flag - by default, will be true - before setting records to PENDING_UPDATE_AUTOMATIONS,   //
+   // we will fetch them (if we didn't take them in from the caller, which, UpdateAction does if its    //
+   // backend supports it), to check their current automationStatus - and if they are currently PENDING //
+   // or RUNNING inserts or updates, we won't update them.                                              //
+   ///////////////////////////////////////////////////////////////////////////////////////////////////////
+   private static boolean allowPreUpdateFetch = new QMetaDataVariableInterpreter().getBooleanFromPropertyOrEnvironment("qqq.recordAutomationStatusUpdater.allowPreUpdateFetch", "QQQ_RECORD_AUTOMATION_STATUS_UPDATER_ALLOW_PRE_UPDATE_FETCH", true);
+
+   ///////////////////////////////////////////////////////////////////////////////////////////////
+   // feature flag - by default, we'll memoize the check for triggers - but we can turn it off. //
+   ///////////////////////////////////////////////////////////////////////////////////////////////
+   private static boolean memoizeCheckForTriggers = new QMetaDataVariableInterpreter().getBooleanFromPropertyOrEnvironment("qqq.recordAutomationStatusUpdater.memoizeCheckForTriggers", "QQQ_RECORD_AUTOMATION_STATUS_UPDATER_MEMOIZE_CHECK_FOR_TRIGGERS", true);
+
+   private static Memoization<Key, Boolean> areThereTableTriggersForTableMemoization = new Memoization<Key, Boolean>().withTimeout(Duration.of(60, ChronoUnit.SECONDS));
+
 
 
    /*******************************************************************************
     ** for a list of records from a table, set their automation status - based on
     ** how the table is configured.
     *******************************************************************************/
-   public static boolean setAutomationStatusInRecords(QSession session, QTableMetaData table, List<QRecord> records, AutomationStatus automationStatus)
+   public static boolean setAutomationStatusInRecords(QTableMetaData table, List<QRecord> records, AutomationStatus automationStatus, QBackendTransaction transaction, List<QRecord> oldRecordList)
    {
       if(table == null || table.getAutomationDetails() == null || CollectionUtils.nullSafeIsEmpty(records))
       {
          return (false);
       }
 
+      QTableAutomationDetails automationDetails   = table.getAutomationDetails();
+      Set<Serializable>       pkeysWeMayNotUpdate = new HashSet<>();
+
       ///////////////////////////////////////////////////////////////////////////////////////////////////
       // In case an automation is running, and it updates records - don't let those records be marked  //
       // as PENDING_UPDATE_AUTOMATIONS... this is meant to avoid having a record's automation update   //
@@ -81,12 +109,60 @@ public class RecordAutomationStatusUpdater
          for(StackTraceElement stackTraceElement : e.getStackTrace())
          {
             String className = stackTraceElement.getClassName();
-            if(className.contains("com.kingsrook.qqq.backend.core.actions.automation") && !className.equals(RecordAutomationStatusUpdater.class.getName()) && !className.endsWith("Test"))
+            if(className.contains(RecordAutomationStatusUpdater.class.getPackageName()) && !className.equals(RecordAutomationStatusUpdater.class.getName()) && !className.endsWith("Test") && !className.contains("Test$"))
             {
                LOG.debug("Avoiding re-setting automation status to PENDING_UPDATE while running an automation");
                return (false);
             }
          }
+
+         ///////////////////////////////////////////////////////////////////////////////
+         // if table uses field-in-table status tracking, then check the old records, //
+         // before we set them to pending-updates, to avoid losing other pending or   //
+         // running status information.  We will allow moving  from OK or the 2       //
+         // failed statuses into pending-updates - which seems right.                 //
+         // This is added to fix cases where an update that comes in before insert    //
+         // -automations have run, will cause the pending-insert status to be missed. //
+         ///////////////////////////////////////////////////////////////////////////////
+         if(automationDetails.getStatusTracking() != null && AutomationStatusTrackingType.FIELD_IN_TABLE.equals(automationDetails.getStatusTracking().getType()))
+         {
+            try
+            {
+               if(CollectionUtils.nullSafeIsEmpty(oldRecordList))
+               {
+                  ///////////////////////////////////////////////////////////////////////////////////////////////
+                  // if we didn't get the oldRecordList as input (though UpdateAction should usually pass it?) //
+                  // then check feature-flag if we're allowed to do a lookup here & now.  If so, then do.      //
+                  ///////////////////////////////////////////////////////////////////////////////////////////////
+                  if(allowPreUpdateFetch)
+                  {
+                     List<Serializable> pkeysToLookup = records.stream().map(r -> r.getValue(table.getPrimaryKeyField())).toList();
+                     oldRecordList = new QueryAction().execute(new QueryInput(table.getName())
+                        .withFilter(new QQueryFilter(new QFilterCriteria(table.getPrimaryKeyField(), QCriteriaOperator.IN, pkeysToLookup)))
+                        .withTransaction(transaction)
+                     ).getRecords();
+                  }
+               }
+
+               for(QRecord freshRecord : CollectionUtils.nonNullList(oldRecordList))
+               {
+                  Serializable recordStatus = freshRecord.getValue(automationDetails.getStatusTracking().getFieldName());
+                  if(AutomationStatus.PENDING_INSERT_AUTOMATIONS.getId().equals(recordStatus)
+                     || AutomationStatus.PENDING_UPDATE_AUTOMATIONS.getId().equals(recordStatus)
+                     || AutomationStatus.RUNNING_INSERT_AUTOMATIONS.getId().equals(recordStatus)
+                     || AutomationStatus.RUNNING_UPDATE_AUTOMATIONS.getId().equals(recordStatus))
+                  {
+                     Serializable primaryKey = freshRecord.getValue(table.getPrimaryKeyField());
+                     LOG.debug("May not update automation status", logPair("table", table.getName()), logPair("id", primaryKey), logPair("currentStatus", recordStatus), logPair("requestedStatus", automationStatus.getId()));
+                     pkeysWeMayNotUpdate.add(primaryKey);
+                  }
+               }
+            }
+            catch(QException qe)
+            {
+               LOG.error("Error checking existing automation status before setting new automation status - more records will be updated than maybe should be...", qe);
+            }
+         }
       }
 
       ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
@@ -98,19 +174,15 @@ public class RecordAutomationStatusUpdater
          automationStatus = AutomationStatus.OK;
       }
 
-      QTableAutomationDetails automationDetails = table.getAutomationDetails();
       if(automationDetails.getStatusTracking() != null && AutomationStatusTrackingType.FIELD_IN_TABLE.equals(automationDetails.getStatusTracking().getType()))
       {
          for(QRecord record : records)
          {
-            ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-            // todo - seems like there's some case here, where if an order was in PENDING_INSERT, but then some other job updated the record, that we'd  //
-            //  lose that pending status, which would be a Bad Thing™...                                                                                 //
-            //  problem is - we may not have the full record in here, so we can't necessarily check the record to see what status it's currently in...   //
-            ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-
-            record.setValue(automationDetails.getStatusTracking().getFieldName(), automationStatus.getId());
-            // todo - another field - for the automation timestamp??
+            if(!pkeysWeMayNotUpdate.contains(record.getValue(table.getPrimaryKeyField())))
+            {
+               record.setValue(automationDetails.getStatusTracking().getFieldName(), automationStatus.getId());
+               // todo - another field - for the automation timestamp??
+            }
          }
       }
 
@@ -188,11 +260,29 @@ public class RecordAutomationStatusUpdater
          return (false);
       }
 
+      if(memoizeCheckForTriggers)
+      {
+         ///////////////////////////////////////////////////////////////////////////////////////
+         // as within the lookup method, error on the side of "yes, maybe there are triggers" //
+         ///////////////////////////////////////////////////////////////////////////////////////
+         Optional<Boolean> result = areThereTableTriggersForTableMemoization.getResult(new Key(table, triggerEvent), key -> lookupIfThereAreTriggersForTable(table, triggerEvent));
+         return result.orElse(true);
+      }
+      else
+      {
+         return lookupIfThereAreTriggersForTable(table, triggerEvent);
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private static Boolean lookupIfThereAreTriggersForTable(QTableMetaData table, TriggerEvent triggerEvent)
+   {
       try
       {
-         ///////////////////
-         // todo - cache? //
-         ///////////////////
          CountInput countInput = new CountInput();
          countInput.setTableName(TableTrigger.TABLE_NAME);
          countInput.setFilter(new QQueryFilter(
@@ -207,6 +297,7 @@ public class RecordAutomationStatusUpdater
          ///////////////////////////////////////////////////////////////////////////////////////////////////////////
          // if the count query failed, we're a bit safer to err on the side of "yeah, there might be automations" //
          ///////////////////////////////////////////////////////////////////////////////////////////////////////////
+         LOG.warn("Error looking if there are triggers for table", e, logPair("tableName", table.getName()));
          return (true);
       }
    }
@@ -217,12 +308,12 @@ public class RecordAutomationStatusUpdater
     ** for a list of records, update their automation status and actually Update the
     ** backend as well.
     *******************************************************************************/
-   public static void setAutomationStatusInRecordsAndUpdate(QInstance instance, QSession session, QTableMetaData table, List<QRecord> records, AutomationStatus automationStatus) throws QException
+   public static void setAutomationStatusInRecordsAndUpdate(QTableMetaData table, List<QRecord> records, AutomationStatus automationStatus, QBackendTransaction transaction) throws QException
    {
       QTableAutomationDetails automationDetails = table.getAutomationDetails();
       if(automationDetails != null && AutomationStatusTrackingType.FIELD_IN_TABLE.equals(automationDetails.getStatusTracking().getType()))
       {
-         boolean didSetStatusField = setAutomationStatusInRecords(session, table, records, automationStatus);
+         boolean didSetStatusField = setAutomationStatusInRecords(table, records, automationStatus, transaction, null);
          if(didSetStatusField)
          {
             UpdateInput updateInput = new UpdateInput();
@@ -237,6 +328,7 @@ public class RecordAutomationStatusUpdater
                .withValue(table.getPrimaryKeyField(), r.getValue(table.getPrimaryKeyField()))
                .withValue(automationDetails.getStatusTracking().getFieldName(), r.getValue(automationDetails.getStatusTracking().getFieldName()))).toList());
             updateInput.setAreAllValuesBeingUpdatedTheSame(true);
+            updateInput.setTransaction(transaction);
             updateInput.setOmitDmlAudit(true);
 
             new UpdateAction().execute(updateInput);
@@ -250,4 +342,8 @@ public class RecordAutomationStatusUpdater
       }
    }
 
+
+
+   private record Key(QTableMetaData table, TriggerEvent triggerEvent) {}
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/automation/polling/PollingAutomationPerTableRunner.java

@@ -28,6 +28,7 @@ import java.util.Comparator;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
+import java.util.Optional;
 import java.util.function.Supplier;
 import java.util.stream.Collectors;
 import com.kingsrook.qqq.backend.core.actions.async.AsyncRecordPipeLoop;
@@ -60,18 +61,21 @@ import com.kingsrook.qqq.backend.core.model.automation.TableTrigger;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
 import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
 import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.DynamicDefaultValueBehavior;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.AutomationStatusTrackingType;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.QTableAutomationDetails;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.TableAutomationAction;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.TriggerEvent;
-import com.kingsrook.qqq.backend.core.model.savedfilters.SavedFilter;
+import com.kingsrook.qqq.backend.core.model.savedviews.SavedView;
 import com.kingsrook.qqq.backend.core.model.session.QSession;
 import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
 import com.kingsrook.qqq.backend.core.utils.JsonUtils;
 import com.kingsrook.qqq.backend.core.utils.StringUtils;
 import com.kingsrook.qqq.backend.core.utils.collections.MapBuilder;
 import org.apache.commons.lang.NotImplementedException;
+import org.json.JSONObject;
 import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
 
 
@@ -251,12 +255,11 @@ public class PollingAutomationPerTableRunner implements Runnable
 
       try
       {
-         QSession session = sessionSupplier != null ? sessionSupplier.get() : new QSession();
-         processTableInsertOrUpdate(instance.getTable(tableActions.tableName()), session, tableActions.status());
+         processTableInsertOrUpdate(instance.getTable(tableActions.tableName()), tableActions.status());
       }
       catch(Exception e)
       {
-         LOG.warn("Error running automations", e);
+         LOG.warn("Error running automations", e, logPair("tableName", tableActions.tableName()), logPair("status", tableActions.status()));
       }
       finally
       {
@@ -270,7 +273,7 @@ public class PollingAutomationPerTableRunner implements Runnable
    /*******************************************************************************
     ** Query for and process records that have a PENDING_INSERT or PENDING_UPDATE status on a given table.
     *******************************************************************************/
-   public void processTableInsertOrUpdate(QTableMetaData table, QSession session, AutomationStatus automationStatus) throws QException
+   public void processTableInsertOrUpdate(QTableMetaData table, AutomationStatus automationStatus) throws QException
    {
       /////////////////////////////////////////////////////////////////////////
       // get the actions to run against this table in this automation status //
@@ -300,7 +303,9 @@ public class PollingAutomationPerTableRunner implements Runnable
             AutomationStatusTrackingType statusTrackingType = automationDetails.getStatusTracking().getType();
             if(AutomationStatusTrackingType.FIELD_IN_TABLE.equals(statusTrackingType))
             {
-               queryInput.setFilter(new QQueryFilter().withCriteria(new QFilterCriteria(automationDetails.getStatusTracking().getFieldName(), QCriteriaOperator.EQUALS, List.of(automationStatus.getId()))));
+               QQueryFilter filter = new QQueryFilter().withCriteria(new QFilterCriteria(automationDetails.getStatusTracking().getFieldName(), QCriteriaOperator.EQUALS, List.of(automationStatus.getId())));
+               addOrderByToQueryFilter(table, automationStatus, filter);
+               queryInput.setFilter(filter);
             }
             else
             {
@@ -321,7 +326,7 @@ public class PollingAutomationPerTableRunner implements Runnable
          }, () ->
          {
             List<QRecord> records = recordPipe.consumeAvailableRecords();
-            applyActionsToRecords(session, table, records, actions, automationStatus);
+            applyActionsToRecords(table, records, actions, automationStatus);
             return (records.size());
          }
       );
@@ -329,6 +334,38 @@ public class PollingAutomationPerTableRunner implements Runnable
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   static void addOrderByToQueryFilter(QTableMetaData table, AutomationStatus automationStatus, QQueryFilter filter)
+   {
+      ////////////////////////////////////////////////////////////////////////////////////
+      // look for a field in the table with either create-date or modify-date behavior, //
+      // based on if doing insert or update automations                                 //
+      ////////////////////////////////////////////////////////////////////////////////////
+      DynamicDefaultValueBehavior dynamicDefaultValueBehavior = automationStatus.equals(AutomationStatus.PENDING_INSERT_AUTOMATIONS) ? DynamicDefaultValueBehavior.CREATE_DATE : DynamicDefaultValueBehavior.MODIFY_DATE;
+      Optional<QFieldMetaData> field = table.getFields().values().stream()
+         .filter(f -> dynamicDefaultValueBehavior.equals(f.getBehaviorOrDefault(QContext.getQInstance(), DynamicDefaultValueBehavior.class)))
+         .findFirst();
+
+      if(field.isPresent())
+      {
+         //////////////////////////////////////////////////////////////////////
+         // if a create/modify date field was found, order by it (ascending) //
+         //////////////////////////////////////////////////////////////////////
+         filter.addOrderBy(new QFilterOrderBy(field.get().getName()));
+      }
+      else
+      {
+         ////////////////////////////////////
+         // else, order by the primary key //
+         ////////////////////////////////////
+         filter.addOrderBy(new QFilterOrderBy(table.getPrimaryKeyField()));
+      }
+   }
+
+
+
    /*******************************************************************************
     ** get the actions to run against a table in an automation status.  both from
     ** metaData and tableTriggers/data.
@@ -388,13 +425,15 @@ public class PollingAutomationPerTableRunner implements Runnable
                if(filterId != null)
                {
                   GetInput getInput = new GetInput();
-                  getInput.setTableName(SavedFilter.TABLE_NAME);
+                  getInput.setTableName(SavedView.TABLE_NAME);
                   getInput.setPrimaryKey(filterId);
                   GetOutput getOutput = new GetAction().execute(getInput);
                   if(getOutput.getRecord() != null)
                   {
-                     SavedFilter savedFilter = new SavedFilter(getOutput.getRecord());
-                     filter = JsonUtils.toObject(savedFilter.getFilterJson(), QQueryFilter.class);
+                     SavedView  savedView   = new SavedView(getOutput.getRecord());
+                     JSONObject viewJson    = new JSONObject(savedView.getViewJson());
+                     JSONObject queryFilter = viewJson.getJSONObject("queryFilter");
+                     filter = JsonUtils.toObject(queryFilter.toString(), QQueryFilter.class);
                   }
                }
 
@@ -427,7 +466,7 @@ public class PollingAutomationPerTableRunner implements Runnable
     ** table's actions against them - IF they are found to match the action's filter
     ** (assuming it has one - if it doesn't, then all records match).
     *******************************************************************************/
-   private void applyActionsToRecords(QSession session, QTableMetaData table, List<QRecord> records, List<TableAutomationAction> actions, AutomationStatus automationStatus) throws QException
+   private void applyActionsToRecords(QTableMetaData table, List<QRecord> records, List<TableAutomationAction> actions, AutomationStatus automationStatus) throws QException
    {
       if(CollectionUtils.nullSafeIsEmpty(records))
       {
@@ -437,7 +476,7 @@ public class PollingAutomationPerTableRunner implements Runnable
       ///////////////////////////////////////////////////
       // mark the records as RUNNING their automations //
       ///////////////////////////////////////////////////
-      RecordAutomationStatusUpdater.setAutomationStatusInRecordsAndUpdate(instance, session, table, records, pendingToRunningStatusMap.get(automationStatus));
+      RecordAutomationStatusUpdater.setAutomationStatusInRecordsAndUpdate(table, records, pendingToRunningStatusMap.get(automationStatus), null);
 
       ///////////////////////////////////////////////////////////////////////////////////////////////////////////////
       // foreach action - run it against the records (but only if they match the action's filter, if there is one) //
@@ -455,13 +494,15 @@ public class PollingAutomationPerTableRunner implements Runnable
       ////////////////////////////////////////
       // update status on all these records //
       ////////////////////////////////////////
-      if(anyActionsFailed)
+      AutomationStatus statusToUpdateTo = anyActionsFailed ? pendingToFailedStatusMap.get(automationStatus) : AutomationStatus.OK;
+      try
       {
-         RecordAutomationStatusUpdater.setAutomationStatusInRecordsAndUpdate(instance, session, table, records, pendingToFailedStatusMap.get(automationStatus));
+         RecordAutomationStatusUpdater.setAutomationStatusInRecordsAndUpdate(table, records, statusToUpdateTo, null);
       }
-      else
+      catch(Exception e)
       {
-         RecordAutomationStatusUpdater.setAutomationStatusInRecordsAndUpdate(instance, session, table, records, AutomationStatus.OK);
+         LOG.warn("Error updating automationStatus after running automations", logPair("tableName", table), logPair("count", records.size()), logPair("status", statusToUpdateTo));
+         throw (e);
       }
    }
 
@@ -491,7 +532,7 @@ public class PollingAutomationPerTableRunner implements Runnable
       }
       catch(Exception e)
       {
-         LOG.warn("Caught exception processing records on " + table + " for action " + action, e);
+         LOG.warn("Caught exception processing automations", e, logPair("tableName", table), logPair("action", action.getName()));
          return (true);
       }
    }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/dashboard/AbstractHTMLWidgetRenderer.java

@@ -41,6 +41,7 @@ import com.kingsrook.qqq.backend.core.model.actions.widgets.RenderWidgetInput;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.DisplayFormat;
 import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
 import com.kingsrook.qqq.backend.core.utils.JsonUtils;
+import com.kingsrook.qqq.backend.core.utils.QQueryFilterDeduper;
 import com.kingsrook.qqq.backend.core.utils.StringUtils;
 
 
@@ -176,11 +177,13 @@ public abstract class AbstractHTMLWidgetRenderer extends AbstractWidgetRenderer
       {
          return (totalString);
       }
+      filter = QQueryFilterDeduper.dedupeFilter(filter);
       return ("<a href='" + tablePath + "?filter=" + JsonUtils.toJson(filter) + "'>" + totalString + "</a>");
    }
 
 
 
+
    /*******************************************************************************
     **
     *******************************************************************************/
@@ -192,6 +195,7 @@ public abstract class AbstractHTMLWidgetRenderer extends AbstractWidgetRenderer
          return;
       }
 
+      filter = QQueryFilterDeduper.dedupeFilter(filter);
       urls.add(tablePath + "?filter=" + JsonUtils.toJson(filter));
    }
 
@@ -208,6 +212,7 @@ public abstract class AbstractHTMLWidgetRenderer extends AbstractWidgetRenderer
          return (null);
       }
 
+      filter = QQueryFilterDeduper.dedupeFilter(filter);
       return (tablePath + "?filter=" + JsonUtils.toJson(filter));
    }
 
@@ -224,6 +229,7 @@ public abstract class AbstractHTMLWidgetRenderer extends AbstractWidgetRenderer
          return (null);
       }
 
+      filter = QQueryFilterDeduper.dedupeFilter(filter);
       return (tablePath + "?filter=" + URLEncoder.encode(JsonUtils.toJson(filter), Charset.defaultCharset()));
    }
 
@@ -326,6 +332,7 @@ public abstract class AbstractHTMLWidgetRenderer extends AbstractWidgetRenderer
       }
 
       String tablePath = QContext.getQInstance().getTablePath(tableName);
+      filter = QQueryFilterDeduper.dedupeFilter(filter);
       return (tablePath + "/" + processName + "?recordsParam=filterJSON&filterJSON=" + URLEncoder.encode(JsonUtils.toJson(filter), StandardCharsets.UTF_8));
    }
 

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/dashboard/widgets/DateTimeGroupBy.java

@@ -297,4 +297,21 @@ public enum DateTimeGroupBy
       ZonedDateTime zoned = instant.atZone(zoneId);
       return (zoned.plus(noOfChronoUnitsToAdd, chronoUnitToAdd).toInstant());
    }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public static DateTimeFormatter sqlDateFormatToSelectedDateTimeFormatter(String sqlDateFormat)
+   {
+      for(DateTimeGroupBy value : values())
+      {
+         if(value.sqlDateFormat.equals(sqlDateFormat))
+         {
+            return (value.selectedStringFormatter);
+         }
+      }
+      return null;
+   }
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/scripts/RunAdHocRecordScriptAction.java

@@ -30,6 +30,7 @@ import java.util.List;
 import java.util.Map;
 import java.util.Optional;
 import com.kingsrook.qqq.backend.core.actions.ActionHelper;
+import com.kingsrook.qqq.backend.core.actions.audits.DMLAuditAction;
 import com.kingsrook.qqq.backend.core.actions.tables.GetAction;
 import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
 import com.kingsrook.qqq.backend.core.context.QContext;
@@ -47,12 +48,14 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryJoin;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
 import com.kingsrook.qqq.backend.core.model.metadata.code.AdHocScriptCodeReference;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 import com.kingsrook.qqq.backend.core.model.scripts.Script;
 import com.kingsrook.qqq.backend.core.model.scripts.ScriptRevision;
 import com.kingsrook.qqq.backend.core.model.scripts.ScriptsMetaDataProvider;
 import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
+import com.kingsrook.qqq.backend.core.utils.memoization.Memoization;
 
 
 /*******************************************************************************
@@ -65,6 +68,8 @@ public class RunAdHocRecordScriptAction
    private Map<Integer, ScriptRevision> scriptRevisionCacheByScriptRevisionId = new HashMap<>();
    private Map<Integer, ScriptRevision> scriptRevisionCacheByScriptId         = new HashMap<>();
 
+   private static Memoization<Integer, Script> scriptMemoizationById = new Memoization<>();
+
 
 
    /*******************************************************************************
@@ -85,6 +90,12 @@ public class RunAdHocRecordScriptAction
             throw (new QException("Script revision was not found."));
          }
 
+         Optional<Script> script = getScript(scriptRevision);
+
+         QContext.getQSession().setValue(DMLAuditAction.AUDIT_CONTEXT_FIELD_NAME, script.isPresent()
+            ? "via Script \"%s\"".formatted(script.get().getName())
+            : "via Script id " + scriptRevision.getScriptId());
+
          ////////////////////////////
          // figure out the records //
          ////////////////////////////
@@ -124,6 +135,10 @@ public class RunAdHocRecordScriptAction
       {
          output.setException(Optional.of(e));
       }
+      finally
+      {
+         QContext.getQSession().removeValue(DMLAuditAction.AUDIT_CONTEXT_FIELD_NAME);
+      }
    }
 
 
@@ -217,4 +232,44 @@ public class RunAdHocRecordScriptAction
       throw (new QException("Code reference did not contain a scriptRevision, scriptRevisionId, or scriptId"));
    }
 
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private Optional<Script> getScript(ScriptRevision scriptRevision)
+   {
+      if(scriptRevision == null || scriptRevision.getScriptId() == null)
+      {
+         return (Optional.empty());
+      }
+
+      try
+      {
+         return scriptMemoizationById.getResult(scriptRevision.getScriptId(), scriptId ->
+         {
+            try
+            {
+               QRecord scriptRecord = new GetAction().executeForRecord(new GetInput(Script.TABLE_NAME).withPrimaryKey(scriptRevision.getScriptId()));
+               if(scriptRecord != null)
+               {
+                  Script script = new Script(scriptRecord);
+                  scriptMemoizationById.storeResult(scriptRevision.getScriptId(), script);
+                  return (script);
+               }
+            }
+            catch(Exception e)
+            {
+               LOG.info("");
+            }
+
+            return (null);
+         });
+      }
+      catch(Exception e)
+      {
+         return (Optional.empty());
+      }
+   }
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/InsertAction.java

@@ -23,7 +23,6 @@ package com.kingsrook.qqq.backend.core.actions.tables;
 
 
 import java.io.Serializable;
-import java.time.Instant;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.HashSet;
@@ -207,17 +206,6 @@ public class InsertAction extends AbstractQActionFunction<InsertInput, InsertOut
          return (rs);
       }
 
-      /////////////////////////////////////////////
-      // set values in create date & modify date //
-      // todo .. better (not hard-coded names)   //
-      /////////////////////////////////////////////
-      Instant now = Instant.now();
-      for(QRecord record : insertInput.getRecords())
-      {
-         setValueIfTableHasField(record, insertInput.getTable(), "createDate", now);
-         setValueIfTableHasField(record, insertInput.getTable(), "modifyDate", now);
-      }
-
       //////////////////////////////////////////////////////
       // load the backend module and its insert interface //
       //////////////////////////////////////////////////////
@@ -233,29 +221,6 @@ public class InsertAction extends AbstractQActionFunction<InsertInput, InsertOut
 
 
 
-   /*******************************************************************************
-    ** If the table has a field with the given name, then set the given value in the
-    ** given record.
-    *******************************************************************************/
-   private static void setValueIfTableHasField(QRecord record, QTableMetaData table, String fieldName, Serializable value)
-   {
-      try
-      {
-         if(table.getFields().containsKey(fieldName))
-         {
-            record.setValue(fieldName, value);
-         }
-      }
-      catch(Exception e)
-      {
-         /////////////////////////////////////////////////
-         // this means field doesn't exist, so, ignore. //
-         /////////////////////////////////////////////////
-      }
-   }
-
-
-
    /*******************************************************************************
     **
     *******************************************************************************/
@@ -277,7 +242,7 @@ public class InsertAction extends AbstractQActionFunction<InsertInput, InsertOut
 
       setDefaultValuesInRecords(table, insertInput.getRecords());
 
-      ValueBehaviorApplier.applyFieldBehaviors(insertInput.getInstance(), table, insertInput.getRecords());
+      ValueBehaviorApplier.applyFieldBehaviors(ValueBehaviorApplier.Action.INSERT, insertInput.getInstance(), table, insertInput.getRecords());
 
       runPreInsertCustomizerIfItIsTime(insertInput, preInsertCustomizer, AbstractPreInsertCustomizer.WhenToRun.BEFORE_UNIQUE_KEY_CHECKS);
       setErrorsIfUniqueKeyErrors(insertInput, table);
@@ -479,7 +444,7 @@ public class InsertAction extends AbstractQActionFunction<InsertInput, InsertOut
     *******************************************************************************/
    private void setAutomationStatusField(InsertInput insertInput)
    {
-      RecordAutomationStatusUpdater.setAutomationStatusInRecords(insertInput.getSession(), insertInput.getTable(), insertInput.getRecords(), AutomationStatus.PENDING_INSERT_AUTOMATIONS);
+      RecordAutomationStatusUpdater.setAutomationStatusInRecords(insertInput.getTable(), insertInput.getRecords(), AutomationStatus.PENDING_INSERT_AUTOMATIONS, insertInput.getTransaction(), null);
    }
 
 

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/UpdateAction.java

@@ -113,7 +113,6 @@ public class UpdateAction
    public UpdateOutput execute(UpdateInput updateInput) throws QException
    {
       ActionHelper.validateSession(updateInput);
-      setAutomationStatusField(updateInput);
 
       QTableMetaData table = updateInput.getTable();
 
@@ -130,6 +129,16 @@ public class UpdateAction
       ////////////////////////////////////////////////////////////////////////////////
       Optional<List<QRecord>> oldRecordList = fetchOldRecords(updateInput, updateInterface);
 
+      ///////////////////////////////////////////////////////////////////////////////////////
+      // allow caller to specify that we don't want to trigger automations. this isn't     //
+      // isn't expected to be used much - by design, only for the process that is meant to //
+      // heal automation status, so that it can force us into status=Pending-inserts       //
+      ///////////////////////////////////////////////////////////////////////////////////////
+      if(!updateInput.getOmitTriggeringAutomations())
+      {
+         setAutomationStatusField(updateInput, oldRecordList);
+      }
+
       performValidations(updateInput, oldRecordList, false);
 
       ////////////////////////////////////
@@ -235,7 +244,7 @@ public class UpdateAction
       /////////////////////////////
       // run standard validators //
       /////////////////////////////
-      ValueBehaviorApplier.applyFieldBehaviors(updateInput.getInstance(), table, updateInput.getRecords());
+      ValueBehaviorApplier.applyFieldBehaviors(ValueBehaviorApplier.Action.UPDATE, updateInput.getInstance(), table, updateInput.getRecords());
       validatePrimaryKeysAreGiven(updateInput);
 
       if(oldRecordList.isPresent())
@@ -561,9 +570,9 @@ public class UpdateAction
    /*******************************************************************************
     ** If the table being updated uses an automation-status field, populate it now.
     *******************************************************************************/
-   private void setAutomationStatusField(UpdateInput updateInput)
+   private void setAutomationStatusField(UpdateInput updateInput, Optional<List<QRecord>> oldRecordList)
    {
-      RecordAutomationStatusUpdater.setAutomationStatusInRecords(updateInput.getSession(), updateInput.getTable(), updateInput.getRecords(), AutomationStatus.PENDING_UPDATE_AUTOMATIONS);
+      RecordAutomationStatusUpdater.setAutomationStatusInRecords(updateInput.getTable(), updateInput.getRecords(), AutomationStatus.PENDING_UPDATE_AUTOMATIONS, updateInput.getTransaction(), oldRecordList.orElse(null));
    }
 
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/helpers/UpdateActionRecordSplitHelper.java

@@ -22,7 +22,6 @@
 package com.kingsrook.qqq.backend.core.actions.tables.helpers;
 
 
-import java.io.Serializable;
 import java.time.Instant;
 import java.util.ArrayList;
 import java.util.List;
@@ -61,11 +60,6 @@ public class UpdateActionRecordSplitHelper
 
       for(QRecord record : updateInput.getRecords())
       {
-         ////////////////////////////////////////////
-         // todo .. better (not a hard-coded name) //
-         ////////////////////////////////////////////
-         setValueIfTableHasField(record, table, "modifyDate", now);
-
          List<String> updatableFields = table.getFields().values().stream()
             .map(QFieldMetaData::getName)
             // todo - intent here is to avoid non-updateable fields - but this
@@ -147,29 +141,6 @@ public class UpdateActionRecordSplitHelper
 
 
 
-   /*******************************************************************************
-    ** If the table has a field with the given name, then set the given value in the
-    ** given record.
-    *******************************************************************************/
-   protected void setValueIfTableHasField(QRecord record, QTableMetaData table, String fieldName, Serializable value)
-   {
-      try
-      {
-         if(table.getFields().containsKey(fieldName))
-         {
-            record.setValue(fieldName, value);
-         }
-      }
-      catch(Exception e)
-      {
-         /////////////////////////////////////////////////
-         // this means field doesn't exist, so, ignore. //
-         /////////////////////////////////////////////////
-      }
-   }
-
-
-
    /*******************************************************************************
     ** Getter for haveAnyWithoutErrors
     **

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/ValueBehaviorApplier.java

@@ -25,12 +25,10 @@ package com.kingsrook.qqq.backend.core.actions.values;
 import java.util.List;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
 import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.FieldBehavior;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
-import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
-import com.kingsrook.qqq.backend.core.model.metadata.fields.ValueTooLongBehavior;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
-import com.kingsrook.qqq.backend.core.model.statusmessages.BadInputStatusMessage;
-import com.kingsrook.qqq.backend.core.utils.StringUtils;
+import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
 
 
 /*******************************************************************************
@@ -42,16 +40,10 @@ public class ValueBehaviorApplier
    /*******************************************************************************
     **
     *******************************************************************************/
-   public static void applyFieldBehaviors(QInstance instance, QTableMetaData table, List<QRecord> recordList)
+   public enum Action
    {
-      for(QFieldMetaData field : table.getFields().values())
-      {
-         String fieldName = field.getName();
-         if(field.getType().equals(QFieldType.STRING) && field.getMaxLength() != null)
-         {
-            applyValueTooLongBehavior(instance, recordList, field, fieldName);
-         }
-      }
+      INSERT,
+      UPDATE
    }
 
 
@@ -59,31 +51,18 @@ public class ValueBehaviorApplier
    /*******************************************************************************
     **
     *******************************************************************************/
-   private static void applyValueTooLongBehavior(QInstance instance, List<QRecord> recordList, QFieldMetaData field, String fieldName)
+   public static void applyFieldBehaviors(Action action, QInstance instance, QTableMetaData table, List<QRecord> recordList)
    {
-      ValueTooLongBehavior valueTooLongBehavior = field.getBehavior(instance, ValueTooLongBehavior.class);
-
-      ////////////////////////////////////////////////////////////////////////////////////////////////////
-      // don't process PASS_THROUGH - so we don't have to iterate over the whole record list to do noop //
-      ////////////////////////////////////////////////////////////////////////////////////////////////////
-      if(valueTooLongBehavior != null && !valueTooLongBehavior.equals(ValueTooLongBehavior.PASS_THROUGH))
+      if(CollectionUtils.nullSafeIsEmpty(recordList))
       {
-         for(QRecord record : recordList)
+         return;
+      }
+
+      for(QFieldMetaData field : table.getFields().values())
+      {
+         for(FieldBehavior<?> fieldBehavior : CollectionUtils.nonNullCollection(field.getBehaviors()))
          {
-            String value = record.getValueString(fieldName);
-            if(value != null && value.length() > field.getMaxLength())
-            {
-               switch(valueTooLongBehavior)
-               {
-                  case TRUNCATE -> record.setValue(fieldName, StringUtils.safeTruncate(value, field.getMaxLength()));
-                  case TRUNCATE_ELLIPSIS -> record.setValue(fieldName, StringUtils.safeTruncate(value, field.getMaxLength(), "..."));
-                  case ERROR -> record.addError(new BadInputStatusMessage("The value for " + field.getLabel() + " is too long (max allowed length=" + field.getMaxLength() + ")"));
-                  case PASS_THROUGH ->
-                  {
-                  }
-                  default -> throw new IllegalStateException("Unexpected valueTooLongBehavior: " + valueTooLongBehavior);
-               }
-            }
+            fieldBehavior.apply(action, recordList, instance, table, field);
          }
       }
    }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java

@@ -42,6 +42,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
 import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
 import com.kingsrook.qqq.backend.core.model.metadata.dashboard.QWidgetMetaDataInterface;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.AdornmentType;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.DynamicDefaultValueBehavior;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.FieldAdornment;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
@@ -94,10 +95,8 @@ public class QInstanceEnricher
 
    private JoinGraph joinGraph;
 
-   //////////////////////////////////////////////////////////
-   // todo - come up w/ a way for app devs to set configs! //
-   //////////////////////////////////////////////////////////
-   private boolean configRemoveIdFromNameWhenCreatingPossibleValueFieldLabels = true;
+   private boolean configRemoveIdFromNameWhenCreatingPossibleValueFieldLabels        = true;
+   private boolean configAddDynamicDefaultValuesToFieldsNamedCreateDateAndModifyDate = true;
 
    //////////////////////////////////////////////////////////////////////////////////////////////////
    // let an instance define mappings to be applied during name-to-label enrichments,              //
@@ -464,6 +463,22 @@ public class QInstanceEnricher
             }
          }
       }
+
+      /////////////////////////////////////////////////////////////////////////
+      // add field behaviors for create date & modify date, if so configured //
+      /////////////////////////////////////////////////////////////////////////
+      if(configAddDynamicDefaultValuesToFieldsNamedCreateDateAndModifyDate)
+      {
+         if("createDate".equals(field.getName()) && field.getBehaviorOnlyIfSet(DynamicDefaultValueBehavior.class) == null)
+         {
+            field.withBehavior(DynamicDefaultValueBehavior.CREATE_DATE);
+         }
+
+         if("modifyDate".equals(field.getName()) && field.getBehaviorOnlyIfSet(DynamicDefaultValueBehavior.class) == null)
+         {
+            field.withBehavior(DynamicDefaultValueBehavior.MODIFY_DATE);
+         }
+      }
    }
 
 
@@ -1220,4 +1235,66 @@ public class QInstanceEnricher
       labelMappings.clear();
    }
 
+
+
+   /*******************************************************************************
+    ** Getter for configRemoveIdFromNameWhenCreatingPossibleValueFieldLabels
+    *******************************************************************************/
+   public boolean getConfigRemoveIdFromNameWhenCreatingPossibleValueFieldLabels()
+   {
+      return (this.configRemoveIdFromNameWhenCreatingPossibleValueFieldLabels);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for configRemoveIdFromNameWhenCreatingPossibleValueFieldLabels
+    *******************************************************************************/
+   public void setConfigRemoveIdFromNameWhenCreatingPossibleValueFieldLabels(boolean configRemoveIdFromNameWhenCreatingPossibleValueFieldLabels)
+   {
+      this.configRemoveIdFromNameWhenCreatingPossibleValueFieldLabels = configRemoveIdFromNameWhenCreatingPossibleValueFieldLabels;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for configRemoveIdFromNameWhenCreatingPossibleValueFieldLabels
+    *******************************************************************************/
+   public QInstanceEnricher withConfigRemoveIdFromNameWhenCreatingPossibleValueFieldLabels(boolean configRemoveIdFromNameWhenCreatingPossibleValueFieldLabels)
+   {
+      this.configRemoveIdFromNameWhenCreatingPossibleValueFieldLabels = configRemoveIdFromNameWhenCreatingPossibleValueFieldLabels;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for configAddDynamicDefaultValuesToFieldsNamedCreateDateAndModifyDate
+    *******************************************************************************/
+   public boolean getConfigAddDynamicDefaultValuesToFieldsNamedCreateDateAndModifyDate()
+   {
+      return (this.configAddDynamicDefaultValuesToFieldsNamedCreateDateAndModifyDate);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for configAddDynamicDefaultValuesToFieldsNamedCreateDateAndModifyDate
+    *******************************************************************************/
+   public void setConfigAddDynamicDefaultValuesToFieldsNamedCreateDateAndModifyDate(boolean configAddDynamicDefaultValuesToFieldsNamedCreateDateAndModifyDate)
+   {
+      this.configAddDynamicDefaultValuesToFieldsNamedCreateDateAndModifyDate = configAddDynamicDefaultValuesToFieldsNamedCreateDateAndModifyDate;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for configAddDynamicDefaultValuesToFieldsNamedCreateDateAndModifyDate
+    *******************************************************************************/
+   public QInstanceEnricher withConfigAddDynamicDefaultValuesToFieldsNamedCreateDateAndModifyDate(boolean configAddDynamicDefaultValuesToFieldsNamedCreateDateAndModifyDate)
+   {
+      this.configAddDynamicDefaultValuesToFieldsNamedCreateDateAndModifyDate = configAddDynamicDefaultValuesToFieldsNamedCreateDateAndModifyDate;
+      return (this);
+   }
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidator.java

@@ -76,6 +76,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.tables.AssociatedScript;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.Association;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.ExposedJoin;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QFieldSection;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QSupplementalTableMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.Tier;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.UniqueKey;
@@ -87,6 +88,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.tables.cache.CacheUseCase;
 import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
 import com.kingsrook.qqq.backend.core.utils.StringUtils;
 import com.kingsrook.qqq.backend.core.utils.ValueUtils;
+import com.kingsrook.qqq.backend.core.utils.lambdas.UnsafeLambda;
 
 
 /*******************************************************************************
@@ -493,6 +495,11 @@ public class QInstanceValidator
             validateTableRecordSecurityLocks(qInstance, table);
             validateTableAssociations(qInstance, table);
             validateExposedJoins(qInstance, joinGraph, table);
+
+            for(QSupplementalTableMetaData supplementalTableMetaData : CollectionUtils.nonNullMap(table.getSupplementalMetaData()).values())
+            {
+               supplementalTableMetaData.validate(qInstance, table, this);
+            }
          });
       }
    }
@@ -692,7 +699,7 @@ public class QInstanceValidator
 
       String prefix = "Field " + fieldName + " in table " + tableName + " ";
 
-      ValueTooLongBehavior behavior = field.getBehavior(qInstance, ValueTooLongBehavior.class);
+      ValueTooLongBehavior behavior = field.getBehaviorOrDefault(qInstance, ValueTooLongBehavior.class);
       if(behavior != null && !behavior.equals(ValueTooLongBehavior.PASS_THROUGH))
       {
          assertCondition(field.getMaxLength() != null, prefix + "specifies a ValueTooLongBehavior, but not a maxLength.");
@@ -1784,20 +1791,6 @@ public class QInstanceValidator
 
 
 
-   /*******************************************************************************
-    **
-    *******************************************************************************/
-   @FunctionalInterface
-   interface UnsafeLambda
-   {
-      /*******************************************************************************
-       **
-       *******************************************************************************/
-      void run() throws Exception;
-   }
-
-
-
    /*******************************************************************************
     **
     *******************************************************************************/

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QFilterCriteria.java

@@ -27,6 +27,7 @@ import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.List;
+import java.util.Objects;
 import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
 import com.kingsrook.qqq.backend.core.logging.QLogger;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.serialization.QFilterCriteriaDeserializer;
@@ -346,4 +347,37 @@ public class QFilterCriteria implements Serializable, Cloneable
       return (rs.toString());
    }
 
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public boolean equals(Object o)
+   {
+      if(this == o)
+      {
+         return true;
+      }
+
+      if(o == null || getClass() != o.getClass())
+      {
+         return false;
+      }
+
+      QFilterCriteria that = (QFilterCriteria) o;
+      return Objects.equals(fieldName, that.fieldName) && operator == that.operator && Objects.equals(values, that.values) && Objects.equals(otherFieldName, that.otherFieldName);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public int hashCode()
+   {
+      return Objects.hash(fieldName, operator, values, otherFieldName);
+   }
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QQueryFilter.java

@@ -27,6 +27,7 @@ import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
 import java.util.Map;
+import java.util.Objects;
 import com.kingsrook.qqq.backend.core.instances.QMetaDataVariableInterpreter;
 import com.kingsrook.qqq.backend.core.logging.QLogger;
 import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
@@ -323,6 +324,18 @@ public class QQueryFilter implements Serializable, Cloneable
 
 
 
+   /*******************************************************************************
+    ** Fluent setter for adding a single subFilter
+    **
+    *******************************************************************************/
+   public QQueryFilter withSubFilter(QQueryFilter subFilter)
+   {
+      addSubFilter(subFilter);
+      return (this);
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/
@@ -467,4 +480,36 @@ public class QQueryFilter implements Serializable, Cloneable
       return (this);
    }
 
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public boolean equals(Object o)
+   {
+      if(this == o)
+      {
+         return true;
+      }
+
+      if(o == null || getClass() != o.getClass())
+      {
+         return false;
+      }
+
+      QQueryFilter that = (QQueryFilter) o;
+      return Objects.equals(criteria, that.criteria) && Objects.equals(orderBys, that.orderBys) && booleanOperator == that.booleanOperator && Objects.equals(subFilters, that.subFilters) && Objects.equals(skip, that.skip) && Objects.equals(limit, that.limit);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public int hashCode()
+   {
+      return Objects.hash(criteria, orderBys, booleanOperator, subFilters, skip, limit);
+   }
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/update/UpdateInput.java

@@ -51,6 +51,7 @@ public class UpdateInput extends AbstractTableActionInput
    ////////////////////////////////////////////////////////////////////////////////////////////
    private Boolean areAllValuesBeingUpdatedTheSame = null;
 
+   private boolean omitTriggeringAutomations = false;
    private boolean omitDmlAudit = false;
    private String  auditContext = null;
 
@@ -321,4 +322,35 @@ public class UpdateInput extends AbstractTableActionInput
       return (this);
    }
 
+
+
+   /*******************************************************************************
+    ** Getter for omitTriggeringAutomations
+    *******************************************************************************/
+   public boolean getOmitTriggeringAutomations()
+   {
+      return (this.omitTriggeringAutomations);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for omitTriggeringAutomations
+    *******************************************************************************/
+   public void setOmitTriggeringAutomations(boolean omitTriggeringAutomations)
+   {
+      this.omitTriggeringAutomations = omitTriggeringAutomations;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for omitTriggeringAutomations
+    *******************************************************************************/
+   public UpdateInput withOmitTriggeringAutomations(boolean omitTriggeringAutomations)
+   {
+      this.omitTriggeringAutomations = omitTriggeringAutomations;
+      return (this);
+   }
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/automation/TableTrigger.java

@@ -28,7 +28,7 @@ import com.kingsrook.qqq.backend.core.model.data.QField;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
 import com.kingsrook.qqq.backend.core.model.data.QRecordEntity;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.TablesPossibleValueSourceMetaDataProvider;
-import com.kingsrook.qqq.backend.core.model.savedfilters.SavedFilter;
+import com.kingsrook.qqq.backend.core.model.savedviews.SavedView;
 import com.kingsrook.qqq.backend.core.model.scripts.Script;
 
 
@@ -51,7 +51,7 @@ public class TableTrigger extends QRecordEntity
    @QField(possibleValueSourceName = TablesPossibleValueSourceMetaDataProvider.NAME)
    private String tableName;
 
-   @QField(possibleValueSourceName = SavedFilter.TABLE_NAME)
+   @QField(possibleValueSourceName = SavedView.TABLE_NAME)
    private Integer filterId;
 
    @QField(possibleValueSourceName = Script.TABLE_NAME)

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/data/QRecord.java

@@ -32,6 +32,7 @@ import java.util.ArrayList;
 import java.util.Date;
 import java.util.HashMap;
 import java.util.LinkedHashMap;
+import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
@@ -167,6 +168,11 @@ public class QRecord implements Serializable
             ArrayList<?> cloneList = new ArrayList<>(arrayList);
             clone.put(entry.getKey(), (V) cloneList);
          }
+         else if(entry.getValue() instanceof LinkedList<?> linkedList)
+         {
+            LinkedList<?> cloneList = new LinkedList<>(linkedList);
+            clone.put(entry.getKey(), (V) cloneList);
+         }
          else if(entry.getValue() instanceof LinkedHashMap<?, ?> linkedHashMap)
          {
             LinkedHashMap<?, ?> cloneMap = new LinkedHashMap<>(linkedHashMap);
@@ -186,7 +192,7 @@ public class QRecord implements Serializable
             //////////////////////////////////////////////////////////////////////////////
             // we know entry is serializable at this point, based on type param's bound //
             //////////////////////////////////////////////////////////////////////////////
-            LOG.info("Non-primitive serializable value in QRecord - calling SerializationUtils.clone...", logPair("key", entry.getKey()), logPair("type", value.getClass()));
+            LOG.debug("Non-primitive serializable value in QRecord - calling SerializationUtils.clone...", logPair("key", entry.getKey()), logPair("type", value.getClass()));
             clone.put(entry.getKey(), (V) SerializationUtils.clone(entry.getValue()));
          }
       }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/dashboard/nocode/HtmlWrapper.java

@@ -51,6 +51,7 @@ public class HtmlWrapper implements Serializable
    public static final String STYLE_INDENT_2        = "padding-left: 2rem; ";
    public static final String STYLE_FLOAT_RIGHT     = "float: right; ";
    public static final String STYLE_RED             = "color: red; ";
+   public static final String STYLE_YELLOW = "color: #bfb743; ";
 
 
 

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/DynamicDefaultValueBehavior.java

@@ -0,0 +1,164 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.model.metadata.fields;
+
+
+import java.io.Serializable;
+import java.time.Instant;
+import java.time.LocalDate;
+import java.util.List;
+import com.kingsrook.qqq.backend.core.actions.values.ValueBehaviorApplier;
+import com.kingsrook.qqq.backend.core.logging.QLogger;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
+import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
+
+
+/*******************************************************************************
+ ** Field behavior that sets a default value for a field dynamically.
+ ** e.g., create-date fields get set to 'now' on insert.
+ ** e.g., modify-date fields get set to 'now' on insert and on update.
+ *******************************************************************************/
+public enum DynamicDefaultValueBehavior implements FieldBehavior<DynamicDefaultValueBehavior>
+{
+   CREATE_DATE,
+   MODIFY_DATE,
+   NONE;
+
+   private static final QLogger LOG = QLogger.getLogger(ValueTooLongBehavior.class);
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public DynamicDefaultValueBehavior getDefault()
+   {
+      return (NONE);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public void apply(ValueBehaviorApplier.Action action, List<QRecord> recordList, QInstance instance, QTableMetaData table, QFieldMetaData field)
+   {
+      if(this.equals(NONE))
+      {
+         return;
+      }
+
+      switch(this)
+      {
+         case CREATE_DATE -> applyCreateDate(action, recordList, table, field);
+         case MODIFY_DATE -> applyModifyDate(action, recordList, table, field);
+         default -> throw new IllegalStateException("Unexpected enum value: " + this);
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private void applyCreateDate(ValueBehaviorApplier.Action action, List<QRecord> recordList, QTableMetaData table, QFieldMetaData field)
+   {
+      if(!ValueBehaviorApplier.Action.INSERT.equals(action))
+      {
+         return;
+      }
+
+      setCreateDateOrModifyDateOnList(recordList, table, field);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private void applyModifyDate(ValueBehaviorApplier.Action action, List<QRecord> recordList, QTableMetaData table, QFieldMetaData field)
+   {
+      ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      // check both of these (even though they're the only 2 values at the time of this writing), just in case more enum values are added in the future //
+      ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      if(!ValueBehaviorApplier.Action.INSERT.equals(action) && !ValueBehaviorApplier.Action.UPDATE.equals(action))
+      {
+         return;
+      }
+
+      setCreateDateOrModifyDateOnList(recordList, table, field);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private void setCreateDateOrModifyDateOnList(List<QRecord> recordList, QTableMetaData table, QFieldMetaData field)
+   {
+      String       fieldName = field.getName();
+      Serializable value     = getNow(table, field);
+
+      for(QRecord record : CollectionUtils.nonNullList(recordList))
+      {
+         record.setValue(fieldName, value);
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private Serializable getNow(QTableMetaData table, QFieldMetaData field)
+   {
+      if(QFieldType.DATE_TIME.equals(field.getType()))
+      {
+         return (Instant.now());
+      }
+      else if(QFieldType.DATE.equals(field.getType()))
+      {
+         return (LocalDate.now());
+      }
+      else
+      {
+         LOG.debug("Request to apply a " + this.name() + " DynamicDefaultValueBehavior to a non-date or date-time field", logPair("table", table.getName()), logPair("field", field.getName()));
+         return (null);
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private void noop()
+   {
+
+   }
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/FieldBehavior.java

@@ -22,10 +22,41 @@
 package com.kingsrook.qqq.backend.core.model.metadata.fields;
 
 
+import java.util.List;
+import com.kingsrook.qqq.backend.core.actions.values.ValueBehaviorApplier;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+
+
 /*******************************************************************************
+ ** Interface for (expected to be?) enums which define behaviors that get applied
+ ** to fields.
+ **
+ ** At the present, these behaviors get applied before a field is stored (insert
+ ** or update), through the ValueBehaviorApplier class.
  **
  *******************************************************************************/
-public interface FieldBehavior
+public interface FieldBehavior<T extends FieldBehavior<T>>
 {
 
+   /*******************************************************************************
+    ** In case a behavior of this type wasn't set on the field, what should the
+    ** default of this type be?
+    *******************************************************************************/
+   T getDefault();
+
+   /*******************************************************************************
+    ** Apply this behavior to a list of records
+    *******************************************************************************/
+   void apply(ValueBehaviorApplier.Action action, List<QRecord> recordList, QInstance instance, QTableMetaData table, QFieldMetaData field);
+
+   /*******************************************************************************
+    ** control if multiple behaviors of this type should be allowed together on a field.
+    *******************************************************************************/
+   default boolean allowMultipleBehaviorsOfThisType()
+   {
+      return (false);
+   }
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/QFieldMetaData.java

@@ -35,6 +35,7 @@ import com.fasterxml.jackson.annotation.JsonIgnore;
 import com.github.hervian.reflection.Fun;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.instances.QInstanceHelpContentManager;
+import com.kingsrook.qqq.backend.core.logging.QLogger;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
 import com.kingsrook.qqq.backend.core.model.data.QField;
 import com.kingsrook.qqq.backend.core.model.data.QRecordEntity;
@@ -44,6 +45,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.help.QHelpContent;
 import com.kingsrook.qqq.backend.core.model.metadata.security.FieldSecurityLock;
 import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
 import com.kingsrook.qqq.backend.core.utils.StringUtils;
+import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
 
 
 /*******************************************************************************
@@ -52,6 +54,8 @@ import com.kingsrook.qqq.backend.core.utils.StringUtils;
  *******************************************************************************/
 public class QFieldMetaData implements Cloneable
 {
+   private static final QLogger LOG = QLogger.getLogger(QFieldMetaData.class);
+
    private String     name;
    private String     label;
    private String     backendName;
@@ -73,8 +77,8 @@ public class QFieldMetaData implements Cloneable
    private String       possibleValueSourceName;
    private QQueryFilter possibleValueSourceFilter;
 
-   private Integer            maxLength;
-   private Set<FieldBehavior> behaviors;
+   private Integer               maxLength;
+   private Set<FieldBehavior<?>> behaviors;
 
    ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
    // w/ longer-term vision for FieldBehaviors                                                                           //
@@ -674,7 +678,7 @@ public class QFieldMetaData implements Cloneable
     ** Getter for behaviors
     **
     *******************************************************************************/
-   public Set<FieldBehavior> getBehaviors()
+   public Set<FieldBehavior<?>> getBehaviors()
    {
       return behaviors;
    }
@@ -682,11 +686,12 @@ public class QFieldMetaData implements Cloneable
 
 
    /*******************************************************************************
-    **
+    ** Get the FieldBehavior object of a given behaviorType (class) - but - if one
+    ** isn't set, then use the default from that type.
     *******************************************************************************/
-   public <T extends FieldBehavior> T getBehavior(QInstance instance, Class<T> behaviorType)
+   public <T extends FieldBehavior<T>> T getBehaviorOrDefault(QInstance instance, Class<T> behaviorType)
    {
-      for(FieldBehavior fieldBehavior : CollectionUtils.nonNullCollection(behaviors))
+      for(FieldBehavior<?> fieldBehavior : CollectionUtils.nonNullCollection(behaviors))
       {
          if(behaviorType.isInstance(fieldBehavior))
          {
@@ -701,9 +706,33 @@ public class QFieldMetaData implements Cloneable
       ///////////////////////////////////////////
       // return default behavior for this type //
       ///////////////////////////////////////////
-      if(behaviorType.equals(ValueTooLongBehavior.class))
+      if(behaviorType.isEnum())
       {
-         return behaviorType.cast(ValueTooLongBehavior.getDefault());
+         return (behaviorType.getEnumConstants()[0].getDefault());
+      }
+
+      return (null);
+   }
+
+
+
+   /*******************************************************************************
+    ** Get the FieldBehavior object of a given behaviorType (class) - and if one
+    ** isn't set, then return null.
+    *******************************************************************************/
+   public <T extends FieldBehavior<T>> T getBehaviorOnlyIfSet(Class<T> behaviorType)
+   {
+      if(behaviors == null)
+      {
+         return (null);
+      }
+
+      for(FieldBehavior<?> fieldBehavior : CollectionUtils.nonNullCollection(behaviors))
+      {
+         if(behaviorType.isInstance(fieldBehavior))
+         {
+            return (behaviorType.cast(fieldBehavior));
+         }
       }
 
       return (null);
@@ -715,7 +744,7 @@ public class QFieldMetaData implements Cloneable
     ** Setter for behaviors
     **
     *******************************************************************************/
-   public void setBehaviors(Set<FieldBehavior> behaviors)
+   public void setBehaviors(Set<FieldBehavior<?>> behaviors)
    {
       this.behaviors = behaviors;
    }
@@ -726,7 +755,7 @@ public class QFieldMetaData implements Cloneable
     ** Fluent setter for behaviors
     **
     *******************************************************************************/
-   public QFieldMetaData withBehaviors(Set<FieldBehavior> behaviors)
+   public QFieldMetaData withBehaviors(Set<FieldBehavior<?>> behaviors)
    {
       this.behaviors = behaviors;
       return (this);
@@ -738,12 +767,30 @@ public class QFieldMetaData implements Cloneable
     ** Fluent setter for behaviors
     **
     *******************************************************************************/
-   public QFieldMetaData withBehavior(FieldBehavior behavior)
+   public QFieldMetaData withBehavior(FieldBehavior<?> behavior)
    {
+      if(behavior == null)
+      {
+         LOG.debug("Skipping request to add null behavior", logPair("fieldName", getName()));
+         return (this);
+      }
+
       if(behaviors == null)
       {
          behaviors = new HashSet<>();
       }
+
+      if(!behavior.allowMultipleBehaviorsOfThisType())
+      {
+         @SuppressWarnings("unchecked")
+         FieldBehavior<?> existingBehaviorOfThisType = getBehaviorOnlyIfSet(behavior.getClass());
+         if(existingBehaviorOfThisType != null)
+         {
+            LOG.debug("Replacing a field behavior", logPair("fieldName", getName()), logPair("oldBehavior", existingBehaviorOfThisType), logPair("newBehavior", behavior));
+            this.behaviors.remove(existingBehaviorOfThisType);
+         }
+      }
+
       this.behaviors.add(behavior);
       return (this);
    }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/ValueTooLongBehavior.java

@@ -22,23 +22,85 @@
 package com.kingsrook.qqq.backend.core.model.metadata.fields;
 
 
+import java.util.List;
+import com.kingsrook.qqq.backend.core.actions.values.ValueBehaviorApplier;
+import com.kingsrook.qqq.backend.core.logging.QLogger;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+import com.kingsrook.qqq.backend.core.model.statusmessages.BadInputStatusMessage;
+import com.kingsrook.qqq.backend.core.utils.StringUtils;
+import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
+
+
 /*******************************************************************************
+ ** Behaviors for string fields, if their value is too long.
  **
+ ** Note:  This was the first implementation of a FieldBehavior, so its test
+ ** coverage is provided in ValueBehaviorApplierTest.
  *******************************************************************************/
-public enum ValueTooLongBehavior implements FieldBehavior
+public enum ValueTooLongBehavior implements FieldBehavior<ValueTooLongBehavior>
 {
    TRUNCATE,
    TRUNCATE_ELLIPSIS,
    ERROR,
    PASS_THROUGH;
 
+   private static final QLogger LOG = QLogger.getLogger(ValueTooLongBehavior.class);
+
 
 
    /*******************************************************************************
     **
     *******************************************************************************/
-   public static FieldBehavior getDefault()
+   @Override
+   public ValueTooLongBehavior getDefault()
    {
-      return PASS_THROUGH;
+      return (PASS_THROUGH);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public void apply(ValueBehaviorApplier.Action action, List<QRecord> recordList, QInstance instance, QTableMetaData table, QFieldMetaData field)
+   {
+      if(this.equals(PASS_THROUGH))
+      {
+         return;
+      }
+
+      String fieldName = field.getName();
+      if(!QFieldType.STRING.equals(field.getType()))
+      {
+         LOG.debug("Request to apply a ValueTooLongBehavior to a non-string field", logPair("table", table.getName()), logPair("field", fieldName));
+         return;
+      }
+
+      if(field.getMaxLength() == null)
+      {
+         LOG.debug("Request to apply a ValueTooLongBehavior to string field without a maxLength", logPair("table", table.getName()), logPair("field", fieldName));
+         return;
+      }
+
+      for(QRecord record : recordList)
+      {
+         String value = record.getValueString(fieldName);
+         if(value != null && value.length() > field.getMaxLength())
+         {
+            switch(this)
+            {
+               case TRUNCATE -> record.setValue(fieldName, StringUtils.safeTruncate(value, field.getMaxLength()));
+               case TRUNCATE_ELLIPSIS -> record.setValue(fieldName, StringUtils.safeTruncate(value, field.getMaxLength(), "..."));
+               case ERROR -> record.addError(new BadInputStatusMessage("The value for " + field.getLabel() + " is too long (max allowed length=" + field.getMaxLength() + ")"));
+               ///////////////////////////////////
+               // PASS_THROUGH is handled above //
+               ///////////////////////////////////
+               default -> throw new IllegalStateException("Unexpected enum value: " + this);
+            }
+         }
+      }
    }
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/QSupplementalTableMetaData.java

@@ -22,6 +22,7 @@
 package com.kingsrook.qqq.backend.core.model.metadata.tables;
 
 
+import com.kingsrook.qqq.backend.core.instances.QInstanceValidator;
 import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
 
 
@@ -69,4 +70,16 @@ public abstract class QSupplementalTableMetaData
       // noop in base class //
       ////////////////////////
    }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public void validate(QInstance qInstance, QTableMetaData tableMetaData, QInstanceValidator qInstanceValidator)
+   {
+      ////////////////////////
+      // noop in base class //
+      ////////////////////////
+   }
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/savedviews/SavedView.java

@@ -1,6 +1,6 @@
 /*
  * QQQ - Low-code Application Framework for Engineers.
- * Copyright (C) 2021-2023.  Kingsrook, LLC
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
  * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
  * contact@kingsrook.com
  * https://github.com/Kingsrook/
@@ -19,7 +19,7 @@
  * along with this program.  If not, see <https://www.gnu.org/licenses/>.
  */
 
-package com.kingsrook.qqq.backend.core.model.savedfilters;
+package com.kingsrook.qqq.backend.core.model.savedviews;
 
 
 import java.time.Instant;
@@ -32,9 +32,9 @@ import com.kingsrook.qqq.backend.core.model.data.QRecordEntity;
 /*******************************************************************************
  ** Entity bean for the saved filter table
  *******************************************************************************/
-public class SavedFilter extends QRecordEntity
+public class SavedView extends QRecordEntity
 {
-   public static final String TABLE_NAME = "savedFilter";
+   public static final String TABLE_NAME = "savedView";
 
    @QField(isEditable = false)
    private Integer id;
@@ -55,7 +55,7 @@ public class SavedFilter extends QRecordEntity
    private String userId;
 
    @QField(isEditable = false)
-   private String filterJson;
+   private String viewJson;
 
 
 
@@ -63,7 +63,7 @@ public class SavedFilter extends QRecordEntity
     ** Constructor
     **
     *******************************************************************************/
-   public SavedFilter()
+   public SavedView()
    {
    }
 
@@ -73,7 +73,7 @@ public class SavedFilter extends QRecordEntity
     ** Constructor
     **
     *******************************************************************************/
-   public SavedFilter(QRecord qRecord) throws QException
+   public SavedView(QRecord qRecord) throws QException
    {
       populateFromQRecord(qRecord);
    }
@@ -172,7 +172,7 @@ public class SavedFilter extends QRecordEntity
     ** Fluent setter for label
     **
     *******************************************************************************/
-   public SavedFilter withLabel(String label)
+   public SavedView withLabel(String label)
    {
       this.label = label;
       return (this);
@@ -206,7 +206,7 @@ public class SavedFilter extends QRecordEntity
     ** Fluent setter for tableName
     **
     *******************************************************************************/
-   public SavedFilter withTableName(String tableName)
+   public SavedView withTableName(String tableName)
    {
       this.tableName = tableName;
       return (this);
@@ -240,7 +240,7 @@ public class SavedFilter extends QRecordEntity
     ** Fluent setter for userId
     **
     *******************************************************************************/
-   public SavedFilter withUserId(String userId)
+   public SavedView withUserId(String userId)
    {
       this.userId = userId;
       return (this);
@@ -249,34 +249,31 @@ public class SavedFilter extends QRecordEntity
 
 
    /*******************************************************************************
-    ** Getter for filterJson
-    **
+    ** Getter for viewJson
     *******************************************************************************/
-   public String getFilterJson()
+   public String getViewJson()
    {
-      return filterJson;
+      return (this.viewJson);
    }
 
 
 
    /*******************************************************************************
-    ** Setter for filterJson
-    **
+    ** Setter for viewJson
     *******************************************************************************/
-   public void setFilterJson(String filterJson)
+   public void setViewJson(String viewJson)
    {
-      this.filterJson = filterJson;
+      this.viewJson = viewJson;
    }
 
 
 
    /*******************************************************************************
-    ** Fluent setter for filterJson
-    **
+    ** Fluent setter for viewJson
     *******************************************************************************/
-   public SavedFilter withFilterJson(String filterJson)
+   public SavedView withViewJson(String viewJson)
    {
-      this.filterJson = filterJson;
+      this.viewJson = viewJson;
       return (this);
    }
 

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/savedviews/SavedViewsMetaDataProvider.java

@@ -1,6 +1,6 @@
 /*
  * QQQ - Low-code Application Framework for Engineers.
- * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
  * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
  * contact@kingsrook.com
  * https://github.com/Kingsrook/
@@ -19,25 +19,31 @@
  * along with this program.  If not, see <https://www.gnu.org/licenses/>.
  */
 
-package com.kingsrook.qqq.backend.core.model.savedfilters;
+package com.kingsrook.qqq.backend.core.model.savedviews;
 
 
+import java.util.List;
 import java.util.function.Consumer;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.AdornmentType;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.FieldAdornment;
+import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon;
 import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.PVSValueFormatAndFields;
 import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSource;
 import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSourceType;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QFieldSection;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
-import com.kingsrook.qqq.backend.core.processes.implementations.savedfilters.DeleteSavedFilterProcess;
-import com.kingsrook.qqq.backend.core.processes.implementations.savedfilters.QuerySavedFilterProcess;
-import com.kingsrook.qqq.backend.core.processes.implementations.savedfilters.StoreSavedFilterProcess;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.Tier;
+import com.kingsrook.qqq.backend.core.processes.implementations.savedviews.DeleteSavedViewProcess;
+import com.kingsrook.qqq.backend.core.processes.implementations.savedviews.QuerySavedViewProcess;
+import com.kingsrook.qqq.backend.core.processes.implementations.savedviews.StoreSavedViewProcess;
 
 
 /*******************************************************************************
  **
  *******************************************************************************/
-public class SavedFiltersMetaDataProvider
+public class SavedViewsMetaDataProvider
 {
 
 
@@ -46,11 +52,11 @@ public class SavedFiltersMetaDataProvider
     *******************************************************************************/
    public void defineAll(QInstance instance, String backendName, Consumer<QTableMetaData> backendDetailEnricher) throws QException
    {
-      instance.addTable(defineSavedFilterTable(backendName, backendDetailEnricher));
-      instance.addPossibleValueSource(defineSavedFilterPossibleValueSource());
-      instance.addProcess(QuerySavedFilterProcess.getProcessMetaData());
-      instance.addProcess(StoreSavedFilterProcess.getProcessMetaData());
-      instance.addProcess(DeleteSavedFilterProcess.getProcessMetaData());
+      instance.addTable(defineSavedViewTable(backendName, backendDetailEnricher));
+      instance.addPossibleValueSource(defineSavedViewPossibleValueSource());
+      instance.addProcess(QuerySavedViewProcess.getProcessMetaData());
+      instance.addProcess(StoreSavedViewProcess.getProcessMetaData());
+      instance.addProcess(DeleteSavedViewProcess.getProcessMetaData());
    }
 
 
@@ -58,16 +64,21 @@ public class SavedFiltersMetaDataProvider
    /*******************************************************************************
     **
     *******************************************************************************/
-   private QTableMetaData defineSavedFilterTable(String backendName, Consumer<QTableMetaData> backendDetailEnricher) throws QException
+   public QTableMetaData defineSavedViewTable(String backendName, Consumer<QTableMetaData> backendDetailEnricher) throws QException
    {
       QTableMetaData table = new QTableMetaData()
-         .withName(SavedFilter.TABLE_NAME)
-         .withLabel("Saved Filter")
+         .withName(SavedView.TABLE_NAME)
+         .withLabel("Saved View")
          .withRecordLabelFormat("%s")
          .withRecordLabelFields("label")
          .withBackendName(backendName)
          .withPrimaryKeyField("id")
-         .withFieldsFromEntity(SavedFilter.class);
+         .withFieldsFromEntity(SavedView.class)
+         .withSection(new QFieldSection("identity", new QIcon().withName("badge"), Tier.T1, List.of("id", "label")))
+         .withSection(new QFieldSection("data", new QIcon().withName("text_snippet"), Tier.T2, List.of("userId", "tableName", "viewJson")))
+         .withSection(new QFieldSection("dates", new QIcon().withName("calendar_month"), Tier.T3, List.of("createDate", "modifyDate")));
+
+      table.getField("viewJson").withFieldAdornment(new FieldAdornment(AdornmentType.CODE_EDITOR).withValue(AdornmentType.CodeEditorValues.languageMode("json")));
 
       if(backendDetailEnricher != null)
       {
@@ -82,12 +93,12 @@ public class SavedFiltersMetaDataProvider
    /*******************************************************************************
     **
     *******************************************************************************/
-   private QPossibleValueSource defineSavedFilterPossibleValueSource()
+   private QPossibleValueSource defineSavedViewPossibleValueSource()
    {
       return new QPossibleValueSource()
-         .withName(SavedFilter.TABLE_NAME)
+         .withName(SavedView.TABLE_NAME)
          .withType(QPossibleValueSourceType.TABLE)
-         .withTableName(SavedFilter.TABLE_NAME)
+         .withTableName(SavedView.TABLE_NAME)
          .withValueFormatAndFields(PVSValueFormatAndFields.LABEL_ONLY)
          .withOrderByField("label");
    }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/session/QSession.java

@@ -142,6 +142,19 @@ public class QSession implements Serializable
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public void removeValue(String key)
+   {
+      if(values != null)
+      {
+         values.remove(key);
+      }
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/authentication/implementations/Auth0AuthenticationModule.java

@@ -176,7 +176,7 @@ public class Auth0AuthenticationModule implements QAuthenticationModuleInterface
             // process a sessionUUID - looks up userSession record - cannot create token this way. //
             /////////////////////////////////////////////////////////////////////////////////////////
             String sessionUUID = context.get(SESSION_UUID_KEY);
-            LOG.info("Creating session from sessionUUID (userSession)", logPair("sessionUUID", maskForLog(sessionUUID)));
+            LOG.debug("Creating session from sessionUUID (userSession)", logPair("sessionUUID", maskForLog(sessionUUID)));
             if(sessionUUID != null)
             {
                accessToken = getAccessTokenFromSessionUUID(metaData, sessionUUID);

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/memory/MemoryRecordStore.java

@@ -24,6 +24,10 @@ package com.kingsrook.qqq.backend.core.modules.backend.implementations.memory;
 
 import java.io.Serializable;
 import java.math.BigDecimal;
+import java.time.Instant;
+import java.time.ZoneId;
+import java.time.ZonedDateTime;
+import java.time.format.DateTimeFormatter;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
@@ -35,6 +39,7 @@ import java.util.Objects;
 import java.util.Optional;
 import java.util.function.Function;
 import java.util.stream.Collectors;
+import com.kingsrook.qqq.backend.core.actions.dashboard.widgets.DateTimeGroupBy;
 import com.kingsrook.qqq.backend.core.actions.tables.helpers.ValidateRecordSecurityLockHelper;
 import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
@@ -66,6 +71,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 import com.kingsrook.qqq.backend.core.modules.backend.implementations.utils.BackendQueryFilterUtils;
 import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
 import com.kingsrook.qqq.backend.core.utils.ListingHash;
+import com.kingsrook.qqq.backend.core.utils.StringUtils;
 import com.kingsrook.qqq.backend.core.utils.ValueUtils;
 
 
@@ -577,7 +583,11 @@ public class MemoryRecordStore
          for(GroupBy groupBy : groupBys)
          {
             Serializable groupByValue = record.getValue(groupBy.getFieldName());
-            if(groupBy.getType() != null)
+            if(StringUtils.hasContent(groupBy.getFormatString()))
+            {
+               groupByValue = applyFormatString(groupByValue, groupBy);
+            }
+            else if(groupBy.getType() != null)
             {
                groupByValue = ValueUtils.getValueAsFieldType(groupBy.getType(), groupByValue);
             }
@@ -629,7 +639,9 @@ public class MemoryRecordStore
       /////////////////////
       if(aggregateInput.getFilter() != null && CollectionUtils.nullSafeHasContents(aggregateInput.getFilter().getOrderBys()))
       {
-         Comparator<AggregateResult> comparator = null;
+         /////////////////////////////////////////////////////////////////////////////////////
+         // lambda to compare 2 serializables, as we'll assume (& cast) them to Comparables //
+         /////////////////////////////////////////////////////////////////////////////////////
          Comparator<Serializable> serializableComparator = (Serializable a, Serializable b) ->
          {
             if(a == null && b == null)
@@ -647,9 +659,15 @@ public class MemoryRecordStore
             return ((Comparable) a).compareTo(b);
          };
 
+         //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+         // reverse of the lambda above (we had some errors calling .reversed() on the comparator we were building, so this seemed simpler & worked) //
+         //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+         Comparator<Serializable> reverseSerializableComparator = (Serializable a, Serializable b) -> -serializableComparator.compare(a, b);
+
          ////////////////////////////////////////////////
          // build a comparator out of all the orderBys //
          ////////////////////////////////////////////////
+         Comparator<AggregateResult> comparator = null;
          for(QFilterOrderBy orderBy : aggregateInput.getFilter().getOrderBys())
          {
             Function<AggregateResult, Serializable> keyExtractor = aggregateResult ->
@@ -670,16 +688,11 @@ public class MemoryRecordStore
 
             if(comparator == null)
             {
-               comparator = Comparator.comparing(keyExtractor, serializableComparator);
+               comparator = Comparator.comparing(keyExtractor, orderBy.getIsAscending() ? serializableComparator : reverseSerializableComparator);
             }
             else
             {
-               comparator = comparator.thenComparing(keyExtractor, serializableComparator);
-            }
-
-            if(!orderBy.getIsAscending())
-            {
-               comparator = comparator.reversed();
+               comparator = comparator.thenComparing(keyExtractor, orderBy.getIsAscending() ? serializableComparator : reverseSerializableComparator);
             }
          }
 
@@ -696,6 +709,57 @@ public class MemoryRecordStore
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private Serializable applyFormatString(Serializable value, GroupBy groupBy) throws QException
+   {
+      if(value == null)
+      {
+         return (null);
+      }
+
+      String formatString = groupBy.getFormatString();
+
+      try
+      {
+         if(formatString.startsWith("DATE_FORMAT"))
+         {
+            /////////////////////////////////////////////////////////////////////////////
+            // one known-use case we have here looks like this:                        //
+            // DATE_FORMAT(CONVERT_TZ(%s, 'UTC', 'UTC'), '%%Y-%%m-%%dT%%H')            //
+            // ... for now, let's just try to support the formatting bit at the end... //
+            // todo - support the CONVERT_TZ bit too!                                  //
+            /////////////////////////////////////////////////////////////////////////////
+            String            sqlDateTimeFormat = formatString.replaceFirst(".*'%%", "%%").replaceFirst("'.*", "");
+            DateTimeFormatter dateTimeFormatter = DateTimeGroupBy.sqlDateFormatToSelectedDateTimeFormatter(sqlDateTimeFormat);
+            if(dateTimeFormatter == null)
+            {
+               throw (new QException("Unsupported sql dateTime format string [" + sqlDateTimeFormat + "] for MemoryRecordStore"));
+            }
+
+            String        valueAsString  = ValueUtils.getValueAsString(value);
+            Instant       valueAsInstant = ValueUtils.getValueAsInstant(valueAsString);
+            ZonedDateTime zonedDateTime  = valueAsInstant.atZone(ZoneId.systemDefault());
+            return (dateTimeFormatter.format(zonedDateTime));
+         }
+         else
+         {
+            throw (new QException("Unsupported group-by format string [" + formatString + "] for MemoryRecordStore"));
+         }
+      }
+      catch(QException qe)
+      {
+         throw (qe);
+      }
+      catch(Exception e)
+      {
+         throw (new QException("Error applying format string [" + formatString + "] to group by value [" + value + "]", e));
+      }
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/memory/MemoryUpdateAction.java

@@ -22,13 +22,10 @@
 package com.kingsrook.qqq.backend.core.modules.backend.implementations.memory;
 
 
-import java.time.Instant;
 import com.kingsrook.qqq.backend.core.actions.interfaces.UpdateInterface;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateOutput;
-import com.kingsrook.qqq.backend.core.model.data.QRecord;
-import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 
 
 /*******************************************************************************
@@ -45,17 +42,6 @@ public class MemoryUpdateAction extends AbstractMemoryAction implements UpdateIn
    {
       try
       {
-         QTableMetaData table = updateInput.getTable();
-         Instant        now   = Instant.now();
-
-         for(QRecord record : updateInput.getRecords())
-         {
-            ///////////////////////////////////////////
-            // todo .. better (not hard-coded names) //
-            ///////////////////////////////////////////
-            setValueIfTableHasField(record, table, "modifyDate", now, false);
-         }
-
          UpdateOutput updateOutput = new UpdateOutput();
          updateOutput.setRecords(MemoryRecordStore.getInstance().update(updateInput, true));
          return (updateOutput);

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/automation/HealBadRecordAutomationStatusesProcessStep.java

@@ -0,0 +1,298 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.processes.implementations.automation;
+
+
+import java.time.temporal.ChronoUnit;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import com.kingsrook.qqq.backend.core.actions.automation.AutomationStatus;
+import com.kingsrook.qqq.backend.core.actions.processes.BackendStep;
+import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
+import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction;
+import com.kingsrook.qqq.backend.core.context.QContext;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.logging.QLogger;
+import com.kingsrook.qqq.backend.core.model.MetaDataProducerInterface;
+import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
+import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.expressions.NowWithOffset;
+import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
+import com.kingsrook.qqq.backend.core.model.metadata.dashboard.nocode.HtmlWrapper;
+import com.kingsrook.qqq.backend.core.model.metadata.dashboard.nocode.WidgetHtmlLine;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.DisplayFormat;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.DynamicDefaultValueBehavior;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
+import com.kingsrook.qqq.backend.core.model.metadata.processes.NoCodeWidgetFrontendComponentMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.processes.QComponentType;
+import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendComponentMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendStepMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.TablesPossibleValueSourceMetaDataProvider;
+import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
+import com.kingsrook.qqq.backend.core.utils.ExceptionUtils;
+import com.kingsrook.qqq.backend.core.utils.StringUtils;
+import com.kingsrook.qqq.backend.core.utils.collections.MultiLevelMapHelper;
+import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
+
+
+/*******************************************************************************
+ ** Process to find records with a bad automation status, and repair them.
+ **
+ ** Bad status are defined as:
+ ** - failed insert or updates.
+ ** - running insert or updates for more than X minutes (see input field value).
+ **
+ ** Repair in this case means resetting their status to the corresponding (e.g.,
+ ** insert/update) pending status.
+ **
+ *******************************************************************************/
+public class HealBadRecordAutomationStatusesProcessStep implements BackendStep, MetaDataProducerInterface<QProcessMetaData>
+{
+   public static final String NAME = "HealBadRecordAutomationStatusesProcess";
+
+   private static final QLogger LOG = QLogger.getLogger(HealBadRecordAutomationStatusesProcessStep.class);
+
+   private static final Map<Integer, Integer> statusUpdateMap = Map.of(
+      AutomationStatus.FAILED_INSERT_AUTOMATIONS.getId(), AutomationStatus.PENDING_INSERT_AUTOMATIONS.getId(),
+      AutomationStatus.RUNNING_INSERT_AUTOMATIONS.getId(), AutomationStatus.PENDING_INSERT_AUTOMATIONS.getId(),
+      AutomationStatus.FAILED_UPDATE_AUTOMATIONS.getId(), AutomationStatus.PENDING_UPDATE_AUTOMATIONS.getId(),
+      AutomationStatus.RUNNING_UPDATE_AUTOMATIONS.getId(), AutomationStatus.PENDING_UPDATE_AUTOMATIONS.getId()
+   );
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public QProcessMetaData produce(QInstance qInstance) throws QException
+   {
+      QProcessMetaData processMetaData = new QProcessMetaData()
+         .withName(NAME)
+         .withStepList(List.of(
+            new QFrontendStepMetaData()
+               .withName("input")
+               .withComponent(new QFrontendComponentMetaData().withType(QComponentType.EDIT_FORM))
+               .withFormField(new QFieldMetaData("tableName", QFieldType.STRING).withPossibleValueSourceName(TablesPossibleValueSourceMetaDataProvider.NAME))
+               .withFormField(new QFieldMetaData("minutesOldLimit", QFieldType.INTEGER).withDefaultValue(60)),
+            new QBackendStepMetaData()
+               .withName("run")
+               .withCode(new QCodeReference(getClass())),
+            new QFrontendStepMetaData()
+               .withName("output")
+
+               .withComponent(new NoCodeWidgetFrontendComponentMetaData()
+                  .withOutput(new WidgetHtmlLine()
+                     .withCondition(new QFilterCriteria("warningCount", QCriteriaOperator.GREATER_THAN, 0))
+                     .withWrapper(HtmlWrapper.divWithStyles(HtmlWrapper.STYLE_YELLOW))
+                     .withVelocityTemplate("<b>Warning:</b>"))
+                  .withOutput(new WidgetHtmlLine()
+                     .withCondition(new QFilterCriteria("warningCount", QCriteriaOperator.GREATER_THAN, 0))
+                     .withWrapper(HtmlWrapper.divWithStyles(HtmlWrapper.STYLE_INDENT_1))
+                     .withWrapper(HtmlWrapper.divWithStyles(HtmlWrapper.STYLE_YELLOW))
+                     .withVelocityTemplate("""
+                        <ul>
+                           #foreach($string in $warnings)
+                              <li>$string</li>
+                           #end
+                        </ul>
+                        """)))
+
+               .withComponent(new QFrontendComponentMetaData().withType(QComponentType.VIEW_FORM))
+               .withViewField(new QFieldMetaData("totalRecordsUpdated", QFieldType.INTEGER) /* todo - didn't display commas... .withDisplayFormat(DisplayFormat.COMMAS) */)
+
+               .withComponent(new QFrontendComponentMetaData().withType(QComponentType.RECORD_LIST))
+               .withRecordListField(new QFieldMetaData("tableName", QFieldType.STRING))
+               .withRecordListField(new QFieldMetaData("badStatus", QFieldType.STRING))
+               .withRecordListField(new QFieldMetaData("count", QFieldType.INTEGER).withDisplayFormat(DisplayFormat.COMMAS) /* todo - didn't display commas... */)
+
+         ));
+
+      return (processMetaData);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException
+   {
+      int recordsUpdated = 0;
+
+      ////////////////////////////////////////////////////////////////////////
+      // if a table name is given, validate it, and run for just that table //
+      ////////////////////////////////////////////////////////////////////////
+      String            tableName = runBackendStepInput.getValueString("tableName");
+      ArrayList<String> warnings  = new ArrayList<>();
+      if(StringUtils.hasContent(tableName))
+      {
+         if(!QContext.getQInstance().getTables().containsKey(tableName))
+         {
+            throw (new QException("Unrecognized table name: " + tableName));
+         }
+
+         recordsUpdated += processTable(tableName, runBackendStepInput, runBackendStepOutput, warnings);
+      }
+      else
+      {
+         //////////////////////////////////////////////////////////////////////////
+         // else, try to run for all tables that have an automation status field //
+         //////////////////////////////////////////////////////////////////////////
+         for(QTableMetaData table : QContext.getQInstance().getTables().values())
+         {
+            recordsUpdated += processTable(table.getName(), runBackendStepInput, runBackendStepOutput, warnings);
+         }
+      }
+
+      runBackendStepOutput.addValue("totalRecordsUpdated", recordsUpdated);
+      runBackendStepOutput.addValue("warnings", warnings);
+      runBackendStepOutput.addValue("warningCount", warnings.size());
+
+      if(CollectionUtils.nullSafeIsEmpty(runBackendStepOutput.getRecords()))
+      {
+         runBackendStepOutput.addRecord(new QRecord()
+            .withValue("tableName", "--")
+            .withValue("badStatus", "--")
+            .withValue("count", "0"));
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private int processTable(String tableName, RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput, List<String> warnings)
+   {
+      try
+      {
+         Integer        minutesOldLimit = Objects.requireNonNullElse(runBackendStepInput.getValueInteger("minutesOldLimit"), 60);
+         QTableMetaData table           = QContext.getQInstance().getTable(tableName);
+
+         //////////////////////////////////////////////////////////////////////////
+         // only process tables w/ automation details w/ a status tracking field //
+         //////////////////////////////////////////////////////////////////////////
+         if(table != null && table.getAutomationDetails() != null && table.getAutomationDetails().getStatusTracking() != null && StringUtils.hasContent(table.getAutomationDetails().getStatusTracking().getFieldName()))
+         {
+            String automationStatusFieldName = table.getAutomationDetails().getStatusTracking().getFieldName();
+
+            /////////////////////////////////////////////
+            // find the modify-date field on the table //
+            /////////////////////////////////////////////
+            String modifyDateFieldName = null;
+            for(QFieldMetaData field : table.getFields().values())
+            {
+               if(DynamicDefaultValueBehavior.MODIFY_DATE.equals(field.getBehaviorOnlyIfSet(DynamicDefaultValueBehavior.class)))
+               {
+                  modifyDateFieldName = field.getName();
+                  break;
+               }
+            }
+
+            if(modifyDateFieldName == null)
+            {
+               warnings.add("Could not find a Modify Date field on table: " + tableName);
+               LOG.info("Couldn't find a MODIFY_DATE field on table", logPair("tableName", tableName));
+               return 0;
+            }
+
+            ////////////////////////////////////////////////////////////////////////
+            // query for records either FAILED, or RUNNING w/ modify date too old //
+            ////////////////////////////////////////////////////////////////////////
+            QueryInput queryInput = new QueryInput();
+            queryInput.setTableName(tableName);
+            queryInput.setFilter(new QQueryFilter().withBooleanOperator(QQueryFilter.BooleanOperator.OR)
+               .withSubFilter(new QQueryFilter()
+                  .withCriteria(new QFilterCriteria(automationStatusFieldName, QCriteriaOperator.IN, AutomationStatus.FAILED_INSERT_AUTOMATIONS.getId(), AutomationStatus.FAILED_UPDATE_AUTOMATIONS.getId())))
+               .withSubFilter(new QQueryFilter()
+                  .withCriteria(new QFilterCriteria(automationStatusFieldName, QCriteriaOperator.IN, AutomationStatus.RUNNING_INSERT_AUTOMATIONS.getId(), AutomationStatus.RUNNING_UPDATE_AUTOMATIONS.getId()))
+                  .withCriteria(new QFilterCriteria(modifyDateFieldName, QCriteriaOperator.LESS_THAN, NowWithOffset.minus(minutesOldLimit, ChronoUnit.MINUTES))))
+            );
+            QueryOutput queryOutput = new QueryAction().execute(queryInput);
+
+            //////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+            // foreach record found, add it to list of records to be updated - mapping status to appropriate pending status //
+            //////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+            List<QRecord>        recordsToUpdate = new ArrayList<>();
+            Map<String, Integer> countByStatus   = new HashMap<>();
+            for(QRecord record : queryOutput.getRecords())
+            {
+               Integer badAutomationStatusId = record.getValueInteger(automationStatusFieldName);
+               Integer updateStatus          = statusUpdateMap.get(badAutomationStatusId);
+               if(updateStatus != null)
+               {
+                  AutomationStatus badStatus = AutomationStatus.getById(badAutomationStatusId);
+                  if(badStatus != null)
+                  {
+                     MultiLevelMapHelper.getOrPutAndIncrement(countByStatus, badStatus.getLabel());
+                  }
+
+                  recordsToUpdate.add(new QRecord()
+                     .withValue(table.getPrimaryKeyField(), record.getValue(table.getPrimaryKeyField()))
+                     .withValue(automationStatusFieldName, updateStatus));
+               }
+            }
+
+            if(!recordsToUpdate.isEmpty())
+            {
+               LOG.info("Healing bad record automation statuses", logPair("tableName", tableName), logPair("count", recordsToUpdate.size()));
+               new UpdateAction().execute(new UpdateInput(tableName).withRecords(recordsToUpdate).withOmitTriggeringAutomations(true));
+            }
+
+            for(Map.Entry<String, Integer> entry : countByStatus.entrySet())
+            {
+               runBackendStepOutput.addRecord(new QRecord()
+                  .withValue("tableName", tableName)
+                  .withValue("badStatus", entry.getKey())
+                  .withValue("count", entry.getValue()));
+            }
+
+            return (recordsToUpdate.size());
+         }
+      }
+      catch(Exception e)
+      {
+         warnings.add("Error processing table: " + tableName + ": " + ExceptionUtils.getTopAndBottomMessages(e));
+         LOG.warn("Error processing table for bad automation statuses", e, logPair("tableName, name"));
+      }
+
+      return 0;
+   }
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/automation/RunTableAutomationsProcessStep.java

@@ -0,0 +1,149 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.processes.implementations.automation;
+
+
+import java.util.List;
+import java.util.Map;
+import com.kingsrook.qqq.backend.core.actions.automation.polling.PollingAutomationPerTableRunner;
+import com.kingsrook.qqq.backend.core.actions.processes.BackendStep;
+import com.kingsrook.qqq.backend.core.context.QContext;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.MetaDataProducerInterface;
+import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
+import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.automation.QAutomationProviderMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
+import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.processes.QComponentType;
+import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendComponentMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendStepMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.TablesPossibleValueSourceMetaDataProvider;
+import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
+import com.kingsrook.qqq.backend.core.utils.StringUtils;
+
+
+/*******************************************************************************
+ ** Process to manually run table automations, for a table.
+ **
+ ** Useful, maybe, for an e2e test.  Or, if you don't want jobs to be running,
+ ** but want to run automations by-hand, for some reason.
+ **
+ ** In the future, this class could take a param to only do inserts or updates.
+ **
+ ** Also, right now, only records that are Pending automations will be run -
+ ** again, that could be changed, presumably (take a list of records, always run, etc...)
+ *******************************************************************************/
+public class RunTableAutomationsProcessStep implements BackendStep, MetaDataProducerInterface<QProcessMetaData>
+{
+   public static final String NAME = "RunTableAutomationsProcess";
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public QProcessMetaData produce(QInstance qInstance) throws QException
+   {
+      QProcessMetaData processMetaData = new QProcessMetaData()
+         .withName(NAME)
+         .withStepList(List.of(
+            new QFrontendStepMetaData()
+               .withName("input")
+               .withComponent(new QFrontendComponentMetaData().withType(QComponentType.EDIT_FORM))
+               .withFormField(new QFieldMetaData("tableName", QFieldType.STRING).withIsRequired(true).withPossibleValueSourceName(TablesPossibleValueSourceMetaDataProvider.NAME))
+               .withFormField(new QFieldMetaData("automationProviderName", QFieldType.STRING)),
+            new QBackendStepMetaData()
+               .withName("run")
+               .withCode(new QCodeReference(getClass())),
+            new QFrontendStepMetaData()
+               .withName("output")
+               .withComponent(new QFrontendComponentMetaData().withType(QComponentType.VIEW_FORM))
+               .withViewField(new QFieldMetaData("ok", QFieldType.STRING))
+         ));
+
+      return (processMetaData);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException
+   {
+      QInstance qInstance = QContext.getQInstance();
+
+      ////////////////////////////////////////////////////////////////////
+      // get tableName param (since this process is not table-specific) //
+      ////////////////////////////////////////////////////////////////////
+      String tableName = runBackendStepInput.getValueString("tableName");
+      if(!StringUtils.hasContent(tableName))
+      {
+         throw (new QException("Missing required input value: tableName"));
+      }
+
+      if(!QContext.getQInstance().getTables().containsKey(tableName))
+      {
+         throw (new QException("Unrecognized table name: " + tableName));
+      }
+
+      ////////////////////////////////////////////////////////////////////////////////////////////////
+      // get the automation provider name to use - either as the only-one-in-instance, or via param //
+      ////////////////////////////////////////////////////////////////////////////////////////////////
+      String automationProviderName = runBackendStepInput.getValueString("automationProviderName");
+      if(!StringUtils.hasContent(automationProviderName))
+      {
+         Map<String, QAutomationProviderMetaData> automationProviders = CollectionUtils.nonNullMap(qInstance.getAutomationProviders());
+         if(automationProviders.size() == 1)
+         {
+            automationProviderName = automationProviders.keySet().iterator().next();
+         }
+         else
+         {
+            throw (new QException("Missing required input value: automationProviderName (and there is not exactly 1 in the active instance)"));
+         }
+      }
+
+      /////////////////////////////////////////////
+      // run automations for the requested table //
+      /////////////////////////////////////////////
+      List<PollingAutomationPerTableRunner.TableActionsInterface> tableActions = PollingAutomationPerTableRunner.getTableActions(qInstance, automationProviderName);
+      for(PollingAutomationPerTableRunner.TableActionsInterface tableAction : tableActions)
+      {
+         if(tableName.equals(tableAction.tableName()))
+         {
+            PollingAutomationPerTableRunner pollingAutomationPerTableRunner = new PollingAutomationPerTableRunner(qInstance, automationProviderName, () -> QContext.getQSession(), tableAction);
+            pollingAutomationPerTableRunner.processTableInsertOrUpdate(qInstance.getTable(tableAction.tableName()), tableAction.status());
+         }
+      }
+
+      runBackendStepOutput.addValue("ok", "true");
+   }
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/columnstats/ColumnStatsStep.java

@@ -25,6 +25,8 @@ package com.kingsrook.qqq.backend.core.processes.implementations.columnstats;
 import java.io.Serializable;
 import java.math.BigDecimal;
 import java.math.RoundingMode;
+import java.time.Instant;
+import java.time.ZoneId;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.Comparator;
@@ -173,11 +175,10 @@ public class ColumnStatsStep implements BackendStep
          Aggregate aggregate = new Aggregate(table.getPrimaryKeyField(), AggregateOperator.COUNT).withFieldType(QFieldType.DECIMAL);
          GroupBy   groupBy   = new GroupBy(field.getType(), fieldName);
 
-         // todo - something here about "by-date, not time"
+         // todo - something here about an input param to specify how you want dates & date-times grouped
          if(field.getType().equals(QFieldType.DATE_TIME))
          {
-            // groupBy = new GroupBy(field.getType(), fieldName, "DATE(%s)");
-            String sqlExpression = DateTimeGroupBy.HOUR.getSqlExpression();
+            String sqlExpression = DateTimeGroupBy.HOUR.getSqlExpression(ZoneId.systemDefault());
             groupBy = new GroupBy(QFieldType.STRING, fieldName, sqlExpression);
          }
 
@@ -230,6 +231,12 @@ public class ColumnStatsStep implements BackendStep
          for(AggregateResult result : aggregateOutput.getResults())
          {
             Serializable value = result.getGroupByValue(groupBy);
+
+            if(field.getType().equals(QFieldType.DATE_TIME) && value != null)
+            {
+               value = Instant.parse(value + ":00:00Z");
+            }
+
             Integer      count = ValueUtils.getValueAsInteger(result.getAggregateValue(aggregate));
             valueCounts.add(new QRecord().withValue(fieldName, value).withValue("count", count));
          }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/savedviews/DeleteSavedViewProcess.java

@@ -1,6 +1,6 @@
 /*
  * QQQ - Low-code Application Framework for Engineers.
- * Copyright (C) 2021-2023.  Kingsrook, LLC
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
  * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
  * contact@kingsrook.com
  * https://github.com/Kingsrook/
@@ -19,7 +19,7 @@
  * along with this program.  If not, see <https://www.gnu.org/licenses/>.
  */
 
-package com.kingsrook.qqq.backend.core.processes.implementations.savedfilters;
+package com.kingsrook.qqq.backend.core.processes.implementations.savedviews;
 
 
 import java.util.List;
@@ -34,15 +34,15 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput;
 import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
 import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
-import com.kingsrook.qqq.backend.core.model.savedfilters.SavedFilter;
+import com.kingsrook.qqq.backend.core.model.savedviews.SavedView;
 
 
 /*******************************************************************************
- ** Process used by the delete filter dialog
+ ** Process used by the delete view dialog
  *******************************************************************************/
-public class DeleteSavedFilterProcess implements BackendStep
+public class DeleteSavedViewProcess implements BackendStep
 {
-   private static final QLogger LOG = QLogger.getLogger(DeleteSavedFilterProcess.class);
+   private static final QLogger LOG = QLogger.getLogger(DeleteSavedViewProcess.class);
 
 
 
@@ -52,10 +52,10 @@ public class DeleteSavedFilterProcess implements BackendStep
    public static QProcessMetaData getProcessMetaData()
    {
       return (new QProcessMetaData()
-         .withName("deleteSavedFilter")
+         .withName("deleteSavedView")
          .withStepList(List.of(
             new QBackendStepMetaData()
-               .withCode(new QCodeReference(DeleteSavedFilterProcess.class))
+               .withCode(new QCodeReference(DeleteSavedViewProcess.class))
                .withName("delete")
          )));
    }
@@ -72,16 +72,16 @@ public class DeleteSavedFilterProcess implements BackendStep
 
       try
       {
-         Integer savedFilterId = runBackendStepInput.getValueInteger("id");
+         Integer savedViewId = runBackendStepInput.getValueInteger("id");
 
          DeleteInput input = new DeleteInput();
-         input.setTableName(SavedFilter.TABLE_NAME);
-         input.setPrimaryKeys(List.of(savedFilterId));
+         input.setTableName(SavedView.TABLE_NAME);
+         input.setPrimaryKeys(List.of(savedViewId));
          new DeleteAction().execute(input);
       }
       catch(Exception e)
       {
-         LOG.warn("Error deleting saved filter", e);
+         LOG.warn("Error deleting saved view", e);
          throw (e);
       }
    }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/savedviews/QuerySavedViewProcess.java

@@ -1,6 +1,6 @@
 /*
  * QQQ - Low-code Application Framework for Engineers.
- * Copyright (C) 2021-2023.  Kingsrook, LLC
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
  * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
  * contact@kingsrook.com
  * https://github.com/Kingsrook/
@@ -19,7 +19,7 @@
  * along with this program.  If not, see <https://www.gnu.org/licenses/>.
  */
 
-package com.kingsrook.qqq.backend.core.processes.implementations.savedfilters;
+package com.kingsrook.qqq.backend.core.processes.implementations.savedviews;
 
 
 import java.io.Serializable;
@@ -43,15 +43,15 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput;
 import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
 import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
-import com.kingsrook.qqq.backend.core.model.savedfilters.SavedFilter;
+import com.kingsrook.qqq.backend.core.model.savedviews.SavedView;
 
 
 /*******************************************************************************
- ** Process used by the saved filter dialogs
+ ** Process used by the saved view dialogs
  *******************************************************************************/
-public class QuerySavedFilterProcess implements BackendStep
+public class QuerySavedViewProcess implements BackendStep
 {
-   private static final QLogger LOG = QLogger.getLogger(QuerySavedFilterProcess.class);
+   private static final QLogger LOG = QLogger.getLogger(QuerySavedViewProcess.class);
 
 
 
@@ -61,10 +61,10 @@ public class QuerySavedFilterProcess implements BackendStep
    public static QProcessMetaData getProcessMetaData()
    {
       return (new QProcessMetaData()
-         .withName("querySavedFilter")
+         .withName("querySavedView")
          .withStepList(List.of(
             new QBackendStepMetaData()
-               .withCode(new QCodeReference(QuerySavedFilterProcess.class))
+               .withCode(new QCodeReference(QuerySavedViewProcess.class))
                .withName("query")
          )));
    }
@@ -81,36 +81,36 @@ public class QuerySavedFilterProcess implements BackendStep
 
       try
       {
-         Integer savedFilterId = runBackendStepInput.getValueInteger("id");
-         if(savedFilterId != null)
+         Integer savedViewId = runBackendStepInput.getValueInteger("id");
+         if(savedViewId != null)
          {
             GetInput input = new GetInput();
-            input.setTableName(SavedFilter.TABLE_NAME);
-            input.setPrimaryKey(savedFilterId);
+            input.setTableName(SavedView.TABLE_NAME);
+            input.setPrimaryKey(savedViewId);
 
             GetOutput output = new GetAction().execute(input);
             runBackendStepOutput.addRecord(output.getRecord());
-            runBackendStepOutput.addValue("savedFilter", output.getRecord());
-            runBackendStepOutput.addValue("savedFilterList", (Serializable) List.of(output.getRecord()));
+            runBackendStepOutput.addValue("savedView", output.getRecord());
+            runBackendStepOutput.addValue("savedViewList", (Serializable) List.of(output.getRecord()));
          }
          else
          {
             String tableName = runBackendStepInput.getValueString("tableName");
 
             QueryInput input = new QueryInput();
-            input.setTableName(SavedFilter.TABLE_NAME);
+            input.setTableName(SavedView.TABLE_NAME);
             input.setFilter(new QQueryFilter()
                .withCriteria(new QFilterCriteria("tableName", QCriteriaOperator.EQUALS, tableName))
                .withOrderBy(new QFilterOrderBy("label")));
 
             QueryOutput output = new QueryAction().execute(input);
             runBackendStepOutput.setRecords(output.getRecords());
-            runBackendStepOutput.addValue("savedFilterList", (Serializable) output.getRecords());
+            runBackendStepOutput.addValue("savedViewList", (Serializable) output.getRecords());
          }
       }
       catch(Exception e)
       {
-         LOG.warn("Error deleting saved filter", e);
+         LOG.warn("Error querying for saved views", e);
          throw (e);
       }
    }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/savedviews/StoreSavedViewProcess.java

@@ -1,6 +1,6 @@
 /*
  * QQQ - Low-code Application Framework for Engineers.
- * Copyright (C) 2021-2023.  Kingsrook, LLC
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
  * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
  * contact@kingsrook.com
  * https://github.com/Kingsrook/
@@ -19,37 +19,45 @@
  * along with this program.  If not, see <https://www.gnu.org/licenses/>.
  */
 
-package com.kingsrook.qqq.backend.core.processes.implementations.savedfilters;
+package com.kingsrook.qqq.backend.core.processes.implementations.savedviews;
 
 
 import java.io.Serializable;
-import java.util.ArrayList;
 import java.util.List;
 import com.kingsrook.qqq.backend.core.actions.ActionHelper;
 import com.kingsrook.qqq.backend.core.actions.processes.BackendStep;
 import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
+import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
 import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction;
+import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.exceptions.QUserFacingException;
 import com.kingsrook.qqq.backend.core.logging.QLogger;
 import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
 import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertOutput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateOutput;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
 import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
 import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
-import com.kingsrook.qqq.backend.core.model.savedfilters.SavedFilter;
+import com.kingsrook.qqq.backend.core.model.savedviews.SavedView;
+import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
 
 
 /*******************************************************************************
- ** Process used by the saved filter dialog
+ ** Process used by the saved view dialog
  *******************************************************************************/
-public class StoreSavedFilterProcess implements BackendStep
+public class StoreSavedViewProcess implements BackendStep
 {
-   private static final QLogger LOG = QLogger.getLogger(StoreSavedFilterProcess.class);
+   private static final QLogger LOG = QLogger.getLogger(StoreSavedViewProcess.class);
 
 
 
@@ -59,10 +67,10 @@ public class StoreSavedFilterProcess implements BackendStep
    public static QProcessMetaData getProcessMetaData()
    {
       return (new QProcessMetaData()
-         .withName("storeSavedFilter")
+         .withName("storeSavedView")
          .withStepList(List.of(
             new QBackendStepMetaData()
-               .withCode(new QCodeReference(StoreSavedFilterProcess.class))
+               .withCode(new QCodeReference(StoreSavedViewProcess.class))
                .withName("store")
          )));
    }
@@ -79,39 +87,73 @@ public class StoreSavedFilterProcess implements BackendStep
 
       try
       {
+         String userId    = QContext.getQSession().getUser().getIdReference();
+         String tableName = runBackendStepInput.getValueString("tableName");
+         String label     = runBackendStepInput.getValueString("label");
+
          QRecord qRecord = new QRecord()
             .withValue("id", runBackendStepInput.getValueInteger("id"))
-            .withValue("label", runBackendStepInput.getValueString("label"))
-            .withValue("tableName", runBackendStepInput.getValueString("tableName"))
-            .withValue("filterJson", runBackendStepInput.getValueString("filterJson"))
-            .withValue("userId", runBackendStepInput.getSession().getUser().getIdReference());
+            .withValue("viewJson", runBackendStepInput.getValueString("viewJson"))
+            .withValue("label", label)
+            .withValue("tableName", tableName)
+            .withValue("userId", userId);
 
-         List<QRecord> savedFilterList = new ArrayList<>();
+         List<QRecord> savedViewList;
          if(qRecord.getValueInteger("id") == null)
          {
+            checkForDuplicates(userId, tableName, label, null);
+
             InsertInput input = new InsertInput();
-            input.setTableName(SavedFilter.TABLE_NAME);
+            input.setTableName(SavedView.TABLE_NAME);
             input.setRecords(List.of(qRecord));
 
             InsertOutput output = new InsertAction().execute(input);
-            savedFilterList = output.getRecords();
+            savedViewList = output.getRecords();
          }
          else
          {
+            checkForDuplicates(userId, tableName, label, qRecord.getValueInteger("id"));
+
             UpdateInput input = new UpdateInput();
-            input.setTableName(SavedFilter.TABLE_NAME);
+            input.setTableName(SavedView.TABLE_NAME);
             input.setRecords(List.of(qRecord));
 
             UpdateOutput output = new UpdateAction().execute(input);
-            savedFilterList = output.getRecords();
+            savedViewList = output.getRecords();
          }
 
-         runBackendStepOutput.addValue("savedFilterList", (Serializable) savedFilterList);
+         runBackendStepOutput.addValue("savedViewList", (Serializable) savedViewList);
       }
       catch(Exception e)
       {
-         LOG.warn("Error storing data saved filter", e);
+         LOG.warn("Error storing saved view", e);
          throw (e);
       }
    }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private static void checkForDuplicates(String userId, String tableName, String label, Integer id) throws QException
+   {
+      QueryInput queryInput = new QueryInput();
+      queryInput.setTableName(SavedView.TABLE_NAME);
+      queryInput.setFilter(new QQueryFilter(
+         new QFilterCriteria("userId", QCriteriaOperator.EQUALS, userId),
+         new QFilterCriteria("tableName", QCriteriaOperator.EQUALS, tableName),
+         new QFilterCriteria("label", QCriteriaOperator.EQUALS, label)));
+
+      if(id != null)
+      {
+         queryInput.getFilter().addCriteria(new QFilterCriteria("id", QCriteriaOperator.NOT_EQUALS, id));
+      }
+
+      QueryOutput queryOutput = new QueryAction().execute(queryInput);
+      if(CollectionUtils.nullSafeHasContents(queryOutput.getRecords()))
+      {
+         throw (new QUserFacingException("You already have a saved view on this table with this name."));
+      }
+   }
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/scripts/RunRecordScriptLoadStep.java

@@ -26,7 +26,6 @@ import java.io.Serializable;
 import java.util.ArrayList;
 import java.util.List;
 import com.kingsrook.qqq.backend.core.actions.audits.AuditAction;
-import com.kingsrook.qqq.backend.core.actions.audits.DMLAuditAction;
 import com.kingsrook.qqq.backend.core.actions.scripts.RunAdHocRecordScriptAction;
 import com.kingsrook.qqq.backend.core.actions.scripts.logging.StoreScriptLogAndScriptLogLineExecutionLogger;
 import com.kingsrook.qqq.backend.core.actions.tables.GetAction;
@@ -133,11 +132,6 @@ public class RunRecordScriptLoadStep extends AbstractLoadStep implements Process
          throw (new QException("Could not find script by id: " + scriptId));
       }
 
-      //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-      // set an "audit context" - so any DML executed during the script will include the note of what script was running. //
-      //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-      runBackendStepInput.addValue(DMLAuditAction.AUDIT_CONTEXT_FIELD_NAME, "via Script \"" + script.getValue("name") + "\"");
-
       String tableName = script.getValueString("tableName");
 
       RunAdHocRecordScriptInput input = new RunAdHocRecordScriptInput();

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/ExceptionUtils.java

@@ -164,4 +164,23 @@ public class ExceptionUtils
 
       return (StringUtils.join("; ", messages));
    }
+
+
+
+   /*******************************************************************************
+    ** Get the messages from the top & bottoms (root) of an exception.
+    **
+    ** If there's no root, just return the top (e.g., parameter)'s message.
+    ** If they are both found, put ": " between them.
+    *******************************************************************************/
+   public static String getTopAndBottomMessages(Exception e)
+   {
+      String    rs            = e.getMessage();
+      Throwable rootException = getRootException(e);
+      if(rootException != e)
+      {
+         rs += ": " + rootException.getMessage();
+      }
+      return (rs);
+   }
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/QQueryFilterDeduper.java

@@ -0,0 +1,363 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.utils;
+
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Set;
+import com.kingsrook.qqq.backend.core.logging.QLogger;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
+import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
+import static com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator.EQUALS;
+import static com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator.IN;
+import static com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator.NOT_EQUALS;
+import static com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator.NOT_IN;
+
+
+/*******************************************************************************
+ ** Class to help deduplicate redundant criteria in filters.
+ **
+ ** Original use-case is for making more clean url links out of filters.
+ **
+ ** Does not (at this time) look into sub-filters at all, or support any "OR"
+ ** filters other than the most basic (a=1 OR a=1).
+ **
+ ** Also, other than for completely redundant criteria (e.g., a>1 and a>1) only
+ * works on a limited subset of criteria operators (EQUALS, NOT_EQUALS, IN, and NOT_IN)
+ *******************************************************************************/
+public class QQueryFilterDeduper
+{
+   private static final QLogger LOG = QLogger.getLogger(QQueryFilterDeduper.class);
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public static QQueryFilter dedupeFilter(QQueryFilter filter)
+   {
+      if(filter == null)
+      {
+         return (null);
+      }
+
+      try
+      {
+         /////////////////////////////////////////////////////////////////
+         // track (just for logging) if we failed or if we did any good //
+         /////////////////////////////////////////////////////////////////
+         List<String> log        = new ArrayList<>();
+         boolean      fail       = false;
+         boolean      didAnyGood = false;
+
+         //////////////////////////////////////////////////////////////////////////////////////////
+         // always create a clone to be returned.  this is especially useful because,            //
+         // the clone's lists will be ArrayLists, which are mutable - since some of the deduping //
+         // involves manipulating value lists.                                                   //
+         //////////////////////////////////////////////////////////////////////////////////////////
+         QQueryFilter rs = filter.clone();
+
+         ////////////////////////////////////////////////////////////////////////////////////
+         // general strategy is:                                                           //
+         // iterate over criteria, possibly removing the one the iterator is pointing at,  //
+         // if we are able to somehow merge it into other criteria we've already seen.     //
+         // the others-we've-seen will be tracked in the criteriaByFieldName listing hash. //
+         ////////////////////////////////////////////////////////////////////////////////////
+         ListingHash<String, QFilterCriteria> criteriaByFieldName = new ListingHash<>();
+         Iterator<QFilterCriteria>            iterator            = rs.getCriteria().iterator();
+         while(iterator.hasNext())
+         {
+            QFilterCriteria criteria = iterator.next();
+
+            /////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+            // first thing to check is, have we seen any other criteria for this field - if so - try to do some de-duping. //
+            // note that, any time we do a remove, we'll need to do a continue - to avoid adding the now-removed criteria  //
+            // to the listing hash                                                                                         //
+            /////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+            if(criteriaByFieldName.containsKey(criteria.getFieldName()))
+            {
+               List<QFilterCriteria> others = criteriaByFieldName.get(criteria.getFieldName());
+               QFilterCriteria       other  = others.get(0);
+
+               if(others.size() == 1 && other.equals(criteria))
+               {
+                  /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+                  // if we've only see 1 other criteria for this field so far, and this one is an exact match, then remove this one. //
+                  /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+                  log.add(String.format("Remove duplicate criteria [%s]", criteria));
+                  iterator.remove();
+                  didAnyGood = true;
+                  continue;
+               }
+               else
+               {
+                  /////////////////////////////////////////////////////////////////////////////////////////
+                  // else - if there's still just 1 other, and it's an AND query - then apply some basic //
+                  // logic-merging operations, based on the pair of criteria operators                   //
+                  /////////////////////////////////////////////////////////////////////////////////////////
+                  if(others.size() == 1 && QQueryFilter.BooleanOperator.AND.equals(filter.getBooleanOperator()))
+                  {
+                     if((NOT_EQUALS.equals(other.getOperator()) || NOT_IN.equals(other.getOperator())) && EQUALS.equals(criteria.getOperator()))
+                     {
+                        ///////////////////////////////////////////////////////////////////////////
+                        // if we previously saw a not-equals or not-in, and now we see an equals //
+                        // and the value from the EQUALS isn't in the not-in list                //
+                        // then replace the not-equals with the equals                           //
+                        // then just discard this equals                                         //
+                        ///////////////////////////////////////////////////////////////////////////
+                        if(other.getValues().contains(criteria.getValues().get(0)))
+                        {
+                           log.add("Contradicting NOT_EQUALS/NOT_IN and EQUALS");
+                           fail = true;
+                        }
+                        else
+                        {
+                           other.setOperator(criteria.getOperator());
+                           other.setValues(criteria.getValues());
+                           iterator.remove();
+                           didAnyGood = true;
+                           log.add("Replace a not-equals or not-in superseded by an equals");
+                           continue;
+                        }
+                     }
+                     else if(EQUALS.equals(other.getOperator()) && (NOT_EQUALS.equals(criteria.getOperator()) || NOT_IN.equals(criteria.getOperator())))
+                     {
+                        /////////////////////////////////////////////////////////////////////////////
+                        // if we previously saw an equals, and now we see a not-equals or a not-in //
+                        // and the value from the EQUALS isn't in the not-in list                  //
+                        // then just discard this not-equals                                       //
+                        /////////////////////////////////////////////////////////////////////////////
+                        if(criteria.getValues().contains(other.getValues().get(0)))
+                        {
+                           log.add("Contradicting NOT_EQUALS/NOT_IN and EQUALS");
+                           fail = true;
+                        }
+                        else
+                        {
+                           iterator.remove();
+                           didAnyGood = true;
+                           log.add("Remove a redundant not-equals");
+                           continue;
+                        }
+                     }
+                     else if(NOT_EQUALS.equals(other.getOperator()) && IN.equals(criteria.getOperator()))
+                     {
+                        /////////////////////////////////////////////////////////////////////////////////////////////////////
+                        // if we previously saw a not-equals, and now we see an IN                                         //
+                        // then replace the not-equals with the IN (making sure the not-equals value isn't in the in-list) //
+                        // then just discard this equals                                                                   //
+                        /////////////////////////////////////////////////////////////////////////////////////////////////////
+                        Serializable       notEqualsValue = other.getValues().get(0);
+                        List<Serializable> inValues       = new ArrayList<>(criteria.getValues());
+                        inValues.remove(notEqualsValue);
+                        if(inValues.isEmpty())
+                        {
+                           ///////////////////////////////////////////////////////////////////////////////////
+                           // if the only in-value was the not-equal value, then... i don't know, don't try //
+                           ///////////////////////////////////////////////////////////////////////////////////
+                           log.add("Contradicting IN and NOT_EQUAL");
+                           fail = true;
+                        }
+                        else
+                        {
+                           //////////////////////////////////////////////////////////////////
+                           // else, we can proceed by replacing the not-equals with the in //
+                           //////////////////////////////////////////////////////////////////
+                           other.setOperator(criteria.getOperator());
+                           other.setValues(criteria.getValues());
+                           iterator.remove();
+                           didAnyGood = true;
+                           log.add("Replace superseded not-equals (removing its value from in-list)");
+                           continue;
+                        }
+                     }
+                     else if(IN.equals(other.getOperator()) && NOT_EQUALS.equals(criteria.getOperator()))
+                     {
+                        //////////////////////////////////////////////////////////////////
+                        // if we previously saw an in, and now we see a not-equals      //
+                        // discard the not-equals (removing its value from the in-list) //
+                        // then just discard this not-equals                            //
+                        //////////////////////////////////////////////////////////////////
+                        Serializable       notEqualsValue = criteria.getValues().get(0);
+                        List<Serializable> inValues       = new ArrayList<>(other.getValues());
+                        inValues.remove(notEqualsValue);
+                        if(inValues.isEmpty())
+                        {
+                           ///////////////////////////////////////////////////////////////////////////////////
+                           // if the only in-value was the not-equal value, then... i don't know, don't try //
+                           ///////////////////////////////////////////////////////////////////////////////////
+                           log.add("Contradicting IN and NOT_EQUAL");
+                           fail = true;
+                        }
+                        else
+                        {
+                           //////////////////////////////////////////////////////////////////
+                           // else, we can proceed by replacing the not-equals with the in //
+                           //////////////////////////////////////////////////////////////////
+                           iterator.remove();
+                           didAnyGood = true;
+                           log.add("Remove redundant not-equals (removing its value from in-list)");
+                           continue;
+                        }
+                     }
+                     else if(NOT_EQUALS.equals(other.getOperator()) && NOT_IN.equals(criteria.getOperator()))
+                     {
+                        /////////////////////////////////////////////////////////////////////////////////////////
+                        // if we previously saw a not-equals, and now we see a not-in                          //
+                        // we can change the not-equals to the not-in, and make sure it's value is in the list //
+                        // then just discard this not-in                                                       //
+                        /////////////////////////////////////////////////////////////////////////////////////////
+                        Serializable originalNotEqualsValue = other.getValues().get(0);
+                        other.setOperator(criteria.getOperator());
+                        other.setValues(criteria.getValues());
+                        if(!other.getValues().contains(originalNotEqualsValue))
+                        {
+                           other.getValues().add(originalNotEqualsValue);
+                        }
+                        iterator.remove();
+                        didAnyGood = true;
+                        log.add("Replace superseded not-equals with not-in");
+                        continue;
+                     }
+                     else if(NOT_IN.equals(other.getOperator()) && NOT_EQUALS.equals(criteria.getOperator()))
+                     {
+                        ////////////////////////////////////////////////////////////////////////////////////////
+                        // if we previously saw a not-in, and now we see a not-equals                         //
+                        // we can discard this not-equals, and just make sure its value is in the not-in list //
+                        ////////////////////////////////////////////////////////////////////////////////////////
+                        Serializable originalNotEqualsValue = criteria.getValues().get(0);
+                        if(!other.getValues().contains(originalNotEqualsValue))
+                        {
+                           other.getValues().add(originalNotEqualsValue);
+                        }
+                        iterator.remove();
+                        didAnyGood = true;
+                        log.add("Remove not-equals, absorbing into not-in");
+                        continue;
+                     }
+                     else if(NOT_IN.equals(other.getOperator()) && NOT_IN.equals(criteria.getOperator()))
+                     {
+                        ////////////////////////////////////////////////////////////////
+                        // for multiple not-ins, just merge their values (as a union) //
+                        ////////////////////////////////////////////////////////////////
+                        for(Serializable value : criteria.getValues())
+                        {
+                           if(!other.getValues().contains(value))
+                           {
+                              other.getValues().add(value);
+                           }
+                        }
+                        iterator.remove();
+                        didAnyGood = true;
+                        log.add("Merging not-ins");
+                        continue;
+                     }
+                     else if(IN.equals(other.getOperator()) && IN.equals(criteria.getOperator()))
+                     {
+                        ////////////////////////////////////////////////////////////////////////
+                        // for multiple not-ins, just merge their values (as an intersection) //
+                        ////////////////////////////////////////////////////////////////////////
+                        Set<Serializable> otherValues    = new HashSet<>(other.getValues());
+                        Set<Serializable> criteriaValues = new HashSet<>(criteria.getValues());
+                        otherValues.retainAll(criteriaValues);
+                        if(otherValues.isEmpty())
+                        {
+                           log.add("Contradicting IN lists (no values)");
+                           fail = true;
+                        }
+                        else
+                        {
+                           other.setValues(new ArrayList<>(otherValues));
+                           iterator.remove();
+                           didAnyGood = true;
+                           log.add("Merging not-ins");
+                           continue;
+                        }
+                     }
+                     else if(NOT_EQUALS.equals(other.getOperator()) && NOT_EQUALS.equals(criteria.getOperator()))
+                     {
+                        /////////////////////////////////////////////////////////////////////////////////////
+                        // if we have 2 not-equals, we can merge them in a not-in                          //
+                        // we can assume their values are different, else they'd have been equals up above //
+                        /////////////////////////////////////////////////////////////////////////////////////
+                        other.setOperator(NOT_IN);
+                        other.setValues(new ArrayList<>(List.of(other.getValues().get(0), criteria.getValues().get(0))));
+                        iterator.remove();
+                        didAnyGood = true;
+                        log.add("Merge two not-equals as not-in");
+                        continue;
+                     }
+                     else
+                     {
+                        log.add("Fail because unhandled operator pair");
+                        fail = true;
+                     }
+                  }
+                  else
+                  {
+                     log.add("Fail because > 1 other or operator: OR");
+                     fail = true;
+                  }
+               }
+            }
+
+            ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+            // if we reach here (e.g., no continue), then assuming we didn't remove the criteria, add it to the listing hash. //
+            ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+            criteriaByFieldName.add(criteria.getFieldName(), criteria);
+         }
+
+         ///////////////////////////
+         // log based on booleans //
+         ///////////////////////////
+         if(fail && didAnyGood)
+         {
+            LOG.info("Partially unsuccessful dedupe of filter", logPair("original", filter), logPair("deduped", rs), logPair("log", log));
+         }
+         else if(fail)
+         {
+            LOG.info("Unsuccessful dedupe of filter", logPair("filter", filter), logPair("log", log));
+         }
+         else if(didAnyGood)
+         {
+            LOG.debug("Successful dedupe of filter", logPair("original", filter), logPair("deduped", rs), logPair("log", log));
+         }
+         else
+         {
+            LOG.debug("No duplicates in filter, so nothing to dedupe", logPair("original", filter));
+         }
+
+         return rs;
+      }
+      catch(Exception e)
+      {
+         LOG.warn("Error de-duping filter", e, logPair("filter", filter));
+         return (filter.clone());
+      }
+   }
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/lambdas/UnsafeLambda.java

@@ -0,0 +1,37 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.utils.lambdas;
+
+
+/*******************************************************************************
+ **
+ *******************************************************************************/
+@FunctionalInterface
+public interface UnsafeLambda
+{
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   void run() throws Exception;
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/memoization/Memoization.java

@@ -30,6 +30,7 @@ import java.util.LinkedHashMap;
 import java.util.Map;
 import java.util.Optional;
 import com.kingsrook.qqq.backend.core.logging.QLogger;
+import com.kingsrook.qqq.backend.core.utils.lambdas.UnsafeFunction;
 
 
 /*******************************************************************************
@@ -58,8 +59,16 @@ public class Memoization<K, V>
 
 
    /*******************************************************************************
+    ** Get the memoized Value for a given input Key.
     **
+    ** But note, this looks the same to the caller, whether the key just wasn't in
+    ** the internal map (e.g., had never been looked up), or if it was previously looked
+    ** up, and that returned null.  In either case, the optional will be empty.
+    **
+    ** See getMemoizedResult for where we can tell the difference (and we would
+    ** generally want to call that.
     *******************************************************************************/
+   @Deprecated
    public Optional<V> getResult(K key)
    {
       MemoizedResult<V> result = map.get(key);
@@ -77,7 +86,57 @@ public class Memoization<K, V>
 
 
    /*******************************************************************************
+    ** Get the memoized Value for a given input Key - computing it if it wasn't previously
+    ** memoized (or expired).
     **
+    ** In here, if the optional is empty, it means the value is null (whether that
+    ** came form memoization, or from the lookupFunction, you don't care - the answer
+    ** is null).
+    *******************************************************************************/
+   public Optional<V> getResult(K key, UnsafeFunction<K, V, ?> lookupFunction)
+   {
+      MemoizedResult<V> result = map.get(key);
+      if(result != null)
+      {
+         if(result.getTime().isAfter(Instant.now().minus(timeout)))
+         {
+            //////////////////////////////////////////////////////////////////////////////
+            // ok, we have a memoized value, and it's not expired, so we can return it. //
+            // of course, it might be a memoized null, so we use .ofNullable.           //
+            //////////////////////////////////////////////////////////////////////////////
+            return (Optional.ofNullable(result.getResult()));
+         }
+      }
+
+      /////////////////////////////////////////////////////////////////////////////////////////////
+      // ok - either we never memoized this key, or it's expired, so, apply the lookup function, //
+      // store the result, and then return the value (in an Optional.ofNullable)                 //
+      /////////////////////////////////////////////////////////////////////////////////////////////
+      try
+      {
+         V value = lookupFunction.apply(key);
+         storeResult(key, value);
+         return (Optional.ofNullable(value));
+      }
+      catch(Exception e)
+      {
+         LOG.warn("Uncaught Exception while executing a Memoization lookupFunction (to avoid this log, add a catch in the lookupFunction)", e);
+         storeResult(key, null);
+         return (Optional.empty());
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** Get a memoized result, optionally containing a Value, for a given input Key.
+    **
+    ** In this method (contrasted with getResult), if the returned Optional is empty,
+    ** it means that we haven't ever looked up or memoized the key (or it's expired).
+    **
+    ** If the returned Optional is not empty, then it means we've memoized something
+    ** (and it's not expired) - so if the Value from the MemoizedResult is null,
+    ** then null is the proper memoized value.
     *******************************************************************************/
    public Optional<MemoizedResult<V>> getMemoizedResult(K key)
    {
@@ -86,7 +145,7 @@ public class Memoization<K, V>
       {
          if(result.getTime().isAfter(Instant.now().minus(timeout)))
          {
-            return (Optional.ofNullable(result));
+            return (Optional.of(result));
          }
       }
 
@@ -181,4 +240,47 @@ public class Memoization<K, V>
    {
       return map;
    }
+
+
+
+   /*******************************************************************************
+    ** Getter for timeout
+    *******************************************************************************/
+   public Duration getTimeout()
+   {
+      return (this.timeout);
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for timeout
+    *******************************************************************************/
+   public Memoization<K, V> withTimeout(Duration timeout)
+   {
+      this.timeout = timeout;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for maxSize
+    *******************************************************************************/
+   public Integer getMaxSize()
+   {
+      return (this.maxSize);
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for maxSize
+    *******************************************************************************/
+   public Memoization<K, V> withMaxSize(Integer maxSize)
+   {
+      this.maxSize = maxSize;
+      return (this);
+   }
+
 }

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/automation/polling/PollingAutomationPerTableRunnerAutomtationUpdatingSelfAvoidInfiniteLoopTest.java

@@ -0,0 +1,220 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.actions.automation.polling;
+
+
+import java.math.BigDecimal;
+import java.util.List;
+import java.util.Objects;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.ScheduledExecutorService;
+import com.kingsrook.qqq.backend.core.BaseTest;
+import com.kingsrook.qqq.backend.core.actions.automation.AutomationStatus;
+import com.kingsrook.qqq.backend.core.actions.automation.RecordAutomationHandler;
+import com.kingsrook.qqq.backend.core.actions.tables.GetAction;
+import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
+import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction;
+import com.kingsrook.qqq.backend.core.context.CapturedContext;
+import com.kingsrook.qqq.backend.core.context.QContext;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
+import com.kingsrook.qqq.backend.core.model.automation.RecordAutomationInput;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.TableAutomationAction;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.TriggerEvent;
+import com.kingsrook.qqq.backend.core.utils.TestUtils;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import static com.kingsrook.qqq.backend.core.actions.automation.polling.PollingAutomationPerTableRunnerTest.runAllTableActions;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNotEquals;
+
+
+/*******************************************************************************
+ ** Test for the case where:
+ ** - inserting into a main table and a child table, and the child table has a
+ ** post-insert customizer, which mo
+ *******************************************************************************/
+public class PollingAutomationPerTableRunnerAutomtationUpdatingSelfAvoidInfiniteLoopTest extends BaseTest
+{
+   private static boolean didFailInThread = false;
+
+   static
+   {
+      ///////////////////////////////////////////////////////////////////////////////////////////////////
+      // we can set this property to revert to the behavior that existed before this test was written. //
+      ///////////////////////////////////////////////////////////////////////////////////////////////////
+      // System.setProperty("qqq.recordAutomationStatusUpdater.skipPreUpdateFetch", "true");
+   }
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @BeforeEach
+   void beforeEach()
+   {
+      didFailInThread = false;
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void test() throws QException
+   {
+      ////////////////////////////////////
+      // add automations to order table //
+      ////////////////////////////////////
+      QContext.getQInstance().getTable(TestUtils.TABLE_NAME_ORDER)
+         .withField(TestUtils.standardQqqAutomationStatusField())
+         .withAutomationDetails(TestUtils.defineStandardAutomationDetails()
+            .withAction(new TableAutomationAction()
+               .withName("orderPostInsertAction")
+               .withTriggerEvent(TriggerEvent.POST_INSERT)
+               .withCodeReference(new QCodeReference(OrderPostInsertAndUpdateAction.class)))
+            .withAction(new TableAutomationAction()
+               .withName("orderPostUpdateAction")
+               .withTriggerEvent(TriggerEvent.POST_UPDATE)
+               .withCodeReference(new QCodeReference(OrderPostInsertAndUpdateAction.class))));
+
+      QContext.getQSession().withSecurityKeyValue(TestUtils.SECURITY_KEY_TYPE_STORE_ALL_ACCESS, true);
+
+      InsertInput insertInput = new InsertInput();
+      insertInput.setTableName(TestUtils.TABLE_NAME_ORDER);
+      insertInput.setRecords(List.of(new QRecord().withValue("orderNo", "10101").withValue("total", new BigDecimal(1))));
+      new InsertAction().execute(insertInput);
+
+      //////////////////////////////////////////////////////
+      // make sure the order is in pending-inserts status //
+      //////////////////////////////////////////////////////
+      {
+         QRecord order = new GetAction().executeForRecord(new GetInput(TestUtils.TABLE_NAME_ORDER).withPrimaryKey(1));
+         assertEquals(AutomationStatus.PENDING_INSERT_AUTOMATIONS.getId(), order.getValue(TestUtils.standardQqqAutomationStatusField().getName()));
+         assertEquals(new BigDecimal(1), order.getValueBigDecimal("total"));
+      }
+
+      ////////////////////////////////////////////////////////////////////////////////////////////////
+      // run automations - that should update the order via the automation - but leave status as OK //
+      ////////////////////////////////////////////////////////////////////////////////////////////////
+      runAllTableActions(QContext.getQInstance());
+      assertFalse(didFailInThread, "A failure condition happened in the automation sub-thread.  Check System.out for message.");
+
+      {
+         QRecord order = new GetAction().executeForRecord(new GetInput(TestUtils.TABLE_NAME_ORDER).withPrimaryKey(1));
+         assertEquals(AutomationStatus.OK.getId(), order.getValue(TestUtils.standardQqqAutomationStatusField().getName()));
+         assertEquals(new BigDecimal(2), order.getValueBigDecimal("total"));
+      }
+
+      //////////////////////////////////////////////////////////////////
+      // now update the order, verify status moves to pending-updates //
+      //////////////////////////////////////////////////////////////////
+      new UpdateAction().execute(new UpdateInput(TestUtils.TABLE_NAME_ORDER).withRecord(new QRecord()
+         .withValue("id", 1)
+         .withValue("storeId", "x")));
+
+      {
+         QRecord order = new GetAction().executeForRecord(new GetInput(TestUtils.TABLE_NAME_ORDER).withPrimaryKey(1));
+         assertEquals(AutomationStatus.PENDING_UPDATE_AUTOMATIONS.getId(), order.getValue(TestUtils.standardQqqAutomationStatusField().getName()));
+         assertEquals(new BigDecimal(2), order.getValueBigDecimal("total"));
+         assertEquals("x", order.getValueString("storeId"));
+      }
+
+      ////////////////////////////////////////////////////////////////////////////////////////////////
+      // run automations - that should update the order via the automation - but leave status as OK //
+      ////////////////////////////////////////////////////////////////////////////////////////////////
+      runAllTableActions(QContext.getQInstance());
+      assertFalse(didFailInThread, "A failure condition happened in the automation sub-thread.  Check System.out for message.");
+
+      {
+         QRecord order = new GetAction().executeForRecord(new GetInput(TestUtils.TABLE_NAME_ORDER).withPrimaryKey(1));
+         assertEquals(AutomationStatus.OK.getId(), order.getValue(TestUtils.standardQqqAutomationStatusField().getName()));
+         assertEquals(new BigDecimal(3), order.getValueBigDecimal("total"));
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public static class OrderPostInsertAndUpdateAction extends RecordAutomationHandler
+   {
+
+      /*******************************************************************************
+       **
+       *******************************************************************************/
+      @Override
+      public void execute(RecordAutomationInput recordAutomationInput) throws QException
+      {
+         ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+         // launch a new thread, to make sure we avoid the "stack contains automations" check in RecordAutomationStatusUpdater //
+         ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+         CapturedContext capturedContext = QContext.capture();
+         for(QRecord record : recordAutomationInput.getRecordList())
+         {
+            ScheduledExecutorService service = Executors.newSingleThreadScheduledExecutor();
+            Future<?> submit = service.submit(() ->
+            {
+               QContext.init(capturedContext);
+               try
+               {
+                  new UpdateAction().execute(new UpdateInput(TestUtils.TABLE_NAME_ORDER).withRecord(new QRecord()
+                     .withValue("id", record.getValue("id"))
+                     .withValue("total", record.getValueBigDecimal("total").add(new BigDecimal(1)))
+                  ));
+
+                  ///////////////////////////////////////////////////////////////////
+                  // make sure that update action didn't change the order's status //
+                  ///////////////////////////////////////////////////////////////////
+                  QRecord order = new GetAction().executeForRecord(new GetInput(TestUtils.TABLE_NAME_ORDER).withPrimaryKey(1));
+                  if(Objects.equals(AutomationStatus.PENDING_UPDATE_AUTOMATIONS.getId(), order.getValue(TestUtils.standardQqqAutomationStatusField().getName())))
+                  {
+                     System.out.println("Failing test - expected status to not be [PENDING_UPDATE_AUTOMATIONS], but it was.");
+                     didFailInThread = true;
+                  }
+                  assertNotEquals(AutomationStatus.PENDING_UPDATE_AUTOMATIONS.getId(), order.getValue(TestUtils.standardQqqAutomationStatusField().getName()));
+               }
+               catch(QException e)
+               {
+                  e.printStackTrace();
+               }
+               finally
+               {
+                  QContext.clear();
+               }
+            });
+
+            while(!submit.isDone())
+            {
+            }
+         }
+      }
+   }
+
+}

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/automation/polling/PollingAutomationPerTableRunnerChildPostInsertCustomizerTest.java

@@ -0,0 +1,224 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.actions.automation.polling;
+
+
+import java.io.Serializable;
+import java.math.BigDecimal;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.UUID;
+import java.util.stream.Collectors;
+import com.kingsrook.qqq.backend.core.BaseTest;
+import com.kingsrook.qqq.backend.core.actions.automation.AutomationStatus;
+import com.kingsrook.qqq.backend.core.actions.automation.RecordAutomationHandler;
+import com.kingsrook.qqq.backend.core.actions.customizers.AbstractPostInsertCustomizer;
+import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizers;
+import com.kingsrook.qqq.backend.core.actions.tables.AggregateAction;
+import com.kingsrook.qqq.backend.core.actions.tables.GetAction;
+import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
+import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction;
+import com.kingsrook.qqq.backend.core.context.QContext;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.Aggregate;
+import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateOperator;
+import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateOutput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateResult;
+import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.GroupBy;
+import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertOutput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
+import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
+import com.kingsrook.qqq.backend.core.model.automation.RecordAutomationInput;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.TableAutomationAction;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.TriggerEvent;
+import com.kingsrook.qqq.backend.core.utils.TestUtils;
+import com.kingsrook.qqq.backend.core.utils.ValueUtils;
+import org.junit.jupiter.api.Test;
+import static com.kingsrook.qqq.backend.core.actions.automation.polling.PollingAutomationPerTableRunnerTest.runAllTableActions;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+
+/*******************************************************************************
+ ** Test for the case where:
+ ** - inserting into a main table and a child table, and the child table has a
+ ** post-insert customizer, which mo
+ *******************************************************************************/
+public class PollingAutomationPerTableRunnerChildPostInsertCustomizerTest extends BaseTest
+{
+
+   static
+   {
+      ///////////////////////////////////////////////////////////////////////////////////////////////////
+      // we can set this property to revert to the behavior that existed before this test was written. //
+      ///////////////////////////////////////////////////////////////////////////////////////////////////
+      // System.setProperty("qqq.recordAutomationStatusUpdater.skipPreUpdateFetch", "true");
+   }
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void test() throws QException
+   {
+      ////////////////////////////////////
+      // add automations to order table //
+      ////////////////////////////////////
+      QContext.getQInstance().getTable(TestUtils.TABLE_NAME_ORDER)
+         .withField(TestUtils.standardQqqAutomationStatusField())
+         .withAutomationDetails(TestUtils.defineStandardAutomationDetails()
+            .withAction(new TableAutomationAction()
+               .withName("orderPostInsertAction")
+               .withTriggerEvent(TriggerEvent.POST_INSERT)
+               .withCodeReference(new QCodeReference(OrderPostInsertAction.class))
+            ));
+
+      ///////////////////////////////////////////////////////////////////////////
+      // add a post-insert customizer to line-ite table (child of order table) //
+      ///////////////////////////////////////////////////////////////////////////
+      QContext.getQInstance().getTable(TestUtils.TABLE_NAME_LINE_ITEM)
+         .withCustomizer(TableCustomizers.POST_INSERT_RECORD, new QCodeReference(LineItemPostInsertCustomizer.class));
+
+      QContext.getQSession().withSecurityKeyValue(TestUtils.SECURITY_KEY_TYPE_STORE_ALL_ACCESS, true);
+
+      InsertInput insertInput = new InsertInput();
+      insertInput.setTableName(TestUtils.TABLE_NAME_ORDER);
+      insertInput.setRecords(List.of(new QRecord()
+         .withValue("orderNo", "10101")
+         .withAssociatedRecord("orderLine", new QRecord()
+            .withValue("sku", "ABC")
+            .withValue("quantity", 1))));
+      InsertOutput insertOutput = new InsertAction().execute(insertInput);
+
+      ///////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      // make sure the order is in pending-inserts status (at one time, a bug meant that it wouldn't have been...) //
+      ///////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      {
+         QRecord order = new GetAction().executeForRecord(new GetInput(TestUtils.TABLE_NAME_ORDER).withPrimaryKey(1));
+         assertEquals(AutomationStatus.PENDING_INSERT_AUTOMATIONS.getId(), order.getValue(TestUtils.standardQqqAutomationStatusField().getName()));
+         assertEquals(new BigDecimal(1), order.getValueBigDecimal("total"));
+      }
+
+      ///////////////////////////////////////////////////////////////////////////////////////////////
+      // run automations - that should... insert a second line item, but should leave the order in //
+      // automation-status = OK, to avoid perpetual re-running                                     //
+      // the line-item post-inserter should run a second time, making the order's total = 2        //
+      ///////////////////////////////////////////////////////////////////////////////////////////////
+      runAllTableActions(QContext.getQInstance());
+
+      {
+         QRecord order = new GetAction().executeForRecord(new GetInput(TestUtils.TABLE_NAME_ORDER).withPrimaryKey(1));
+         assertEquals(AutomationStatus.OK.getId(), order.getValue(TestUtils.standardQqqAutomationStatusField().getName()));
+         assertEquals(new BigDecimal(2), order.getValueBigDecimal("total"));
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public static class OrderPostInsertAction extends RecordAutomationHandler
+   {
+
+      /*******************************************************************************
+       **
+       *******************************************************************************/
+      @Override
+      public void execute(RecordAutomationInput recordAutomationInput) throws QException
+      {
+         ///////////////////////////////////////
+         // add a new line item to the orders //
+         ///////////////////////////////////////
+         List<QRecord> lineItemsToInsert = new ArrayList<>();
+         for(QRecord record : recordAutomationInput.getRecordList())
+         {
+            lineItemsToInsert.add(new QRecord()
+               .withValue("orderId", record.getValue("id"))
+               .withValue("sku", UUID.randomUUID())
+               .withValue("quantity", 1)
+            );
+         }
+         new InsertAction().execute(new InsertInput(TestUtils.TABLE_NAME_LINE_ITEM).withRecords(lineItemsToInsert));
+
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public static class LineItemPostInsertCustomizer extends AbstractPostInsertCustomizer
+   {
+      @Override
+      public List<QRecord> apply(List<QRecord> records) throws QException
+      {
+         //////////////////////////////////
+         // count line items by order id //
+         //////////////////////////////////
+         Set<Serializable> orderIds = records.stream().map(r -> r.getValue("orderId")).collect(Collectors.toSet());
+
+         GroupBy   groupByOrderId = new GroupBy(QFieldType.STRING, "orderId");
+         Aggregate countId        = new Aggregate("id", AggregateOperator.COUNT);
+
+         AggregateInput aggregateInput = new AggregateInput();
+         aggregateInput.setTableName(TestUtils.TABLE_NAME_LINE_ITEM);
+         aggregateInput.setFilter(new QQueryFilter(new QFilterCriteria("orderId", QCriteriaOperator.IN, orderIds)));
+         aggregateInput.withGroupBy(groupByOrderId);
+         aggregateInput.withAggregate(countId);
+         AggregateOutput       aggregateOutput = new AggregateAction().execute(aggregateInput);
+         Map<Integer, Integer> countByOrderId  = new HashMap<>();
+         for(AggregateResult result : aggregateOutput.getResults())
+         {
+            countByOrderId.put(ValueUtils.getValueAsInteger(result.getGroupByValue(groupByOrderId)), ValueUtils.getValueAsInteger(result.getAggregateValue(countId)));
+         }
+
+         ///////////////////////////////////
+         // update the order total fields //
+         // s/b in bulk, but, meh         //
+         ///////////////////////////////////
+         for(Integer orderId : countByOrderId.keySet())
+         {
+            UpdateInput updateInput = new UpdateInput();
+            updateInput.setTableName(TestUtils.TABLE_NAME_ORDER);
+            updateInput.setRecords(List.of(new QRecord()
+               .withValue("id", orderId)
+               .withValue("total", new BigDecimal(countByOrderId.get(orderId)))));
+            new UpdateAction().execute(updateInput);
+         }
+
+         return (records);
+      }
+   }
+
+}

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/automation/polling/PollingAutomationPerTableRunnerTest.java

@@ -46,6 +46,7 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
 import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.DynamicDefaultValueBehavior;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
 import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
@@ -56,14 +57,11 @@ import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.QTableAut
 import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.TableAutomationAction;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.TriggerEvent;
 import com.kingsrook.qqq.backend.core.model.session.QSession;
-import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryRecordStore;
 import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.ExtractViaQueryStep;
 import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.LoadViaInsertStep;
 import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess;
 import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcessTest;
 import com.kingsrook.qqq.backend.core.utils.TestUtils;
-import org.junit.jupiter.api.AfterEach;
-import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
 import static org.assertj.core.api.Assertions.assertThat;
 import static org.assertj.core.api.Assertions.assertThatThrownBy;
@@ -77,18 +75,6 @@ import static org.junit.jupiter.api.Assertions.assertFalse;
 class PollingAutomationPerTableRunnerTest extends BaseTest
 {
 
-   /*******************************************************************************
-    **
-    *******************************************************************************/
-   @BeforeEach
-   @AfterEach
-   void beforeAndAfterEach()
-   {
-      MemoryRecordStore.getInstance().reset();
-   }
-
-
-
    /*******************************************************************************
     ** Test a cycle that does an insert, some automations, then and an update, and more automations.
     *******************************************************************************/
@@ -200,7 +186,7 @@ class PollingAutomationPerTableRunnerTest extends BaseTest
    /*******************************************************************************
     **
     *******************************************************************************/
-   private void runAllTableActions(QInstance qInstance) throws QException
+   static void runAllTableActions(QInstance qInstance) throws QException
    {
       List<PollingAutomationPerTableRunner.TableActionsInterface> tableActions = PollingAutomationPerTableRunner.getTableActions(qInstance, TestUtils.POLLING_AUTOMATION);
       for(PollingAutomationPerTableRunner.TableActionsInterface tableAction : tableActions)
@@ -210,7 +196,7 @@ class PollingAutomationPerTableRunnerTest extends BaseTest
          /////////////////////////////////////////////////////////////////////////////////////////////////////
          // note - don't call run - it is meant to be called async - e.g., it sets & clears thread context. //
          /////////////////////////////////////////////////////////////////////////////////////////////////////
-         pollingAutomationPerTableRunner.processTableInsertOrUpdate(qInstance.getTable(tableAction.tableName()), QContext.getQSession(), tableAction.status());
+         pollingAutomationPerTableRunner.processTableInsertOrUpdate(qInstance.getTable(tableAction.tableName()), tableAction.status());
       }
    }
 
@@ -512,7 +498,7 @@ class PollingAutomationPerTableRunnerTest extends BaseTest
             /////////////////////////////////////////////////////////////////////////////////////////////////////
             // note - don't call run - it is meant to be called async - e.g., it sets & clears thread context. //
             /////////////////////////////////////////////////////////////////////////////////////////////////////
-            pollingAutomationPerTableRunner.processTableInsertOrUpdate(qInstance.getTable(tableAction.tableName()), QContext.getQSession(), tableAction.status());
+            pollingAutomationPerTableRunner.processTableInsertOrUpdate(qInstance.getTable(tableAction.tableName()), tableAction.status());
          }
       }).hasMessage(PollingAutomationPerTableRunnerThatShouldSimulateServerShutdownMidRun.EXCEPTION_MESSAGE);
 
@@ -593,4 +579,70 @@ class PollingAutomationPerTableRunnerTest extends BaseTest
       new PollingAutomationPerTableRunner.ShardedTableActions(null, null, null, null, null).noopToFakeTestCoverage();
    }
 
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testAddOrderByToQueryFilter()
+   {
+      //////////////////////////////////////////////////////////////////////////
+      // make a table we'll test with.  just put a primary-key id on it first //
+      //////////////////////////////////////////////////////////////////////////
+      QTableMetaData table = new QTableMetaData()
+         .withPrimaryKeyField("id")
+         .withField(new QFieldMetaData("id", QFieldType.INTEGER));
+      {
+         QQueryFilter filter = new QQueryFilter();
+         PollingAutomationPerTableRunner.addOrderByToQueryFilter(table, AutomationStatus.PENDING_INSERT_AUTOMATIONS, filter);
+         assertEquals("id", filter.getOrderBys().get(0).getFieldName());
+      }
+
+      {
+         QQueryFilter filter = new QQueryFilter();
+         PollingAutomationPerTableRunner.addOrderByToQueryFilter(table, AutomationStatus.PENDING_UPDATE_AUTOMATIONS, filter);
+         assertEquals("id", filter.getOrderBys().get(0).getFieldName());
+      }
+
+      ////////////////////////////////////////////////////////////////////////////////
+      // add createDate & modifyDate fields, but not with dynamic-default-behaviors //
+      // so should still sort by id                                                 //
+      ////////////////////////////////////////////////////////////////////////////////
+      QFieldMetaData createDate = new QFieldMetaData("createDate", QFieldType.DATE_TIME);
+      QFieldMetaData modifyDate = new QFieldMetaData("modifyDate", QFieldType.DATE_TIME);
+      table.addField(createDate);
+      table.addField(modifyDate);
+      {
+         QQueryFilter filter = new QQueryFilter();
+         PollingAutomationPerTableRunner.addOrderByToQueryFilter(table, AutomationStatus.PENDING_INSERT_AUTOMATIONS, filter);
+         assertEquals("id", filter.getOrderBys().get(0).getFieldName());
+      }
+
+      {
+         QQueryFilter filter = new QQueryFilter();
+         PollingAutomationPerTableRunner.addOrderByToQueryFilter(table, AutomationStatus.PENDING_UPDATE_AUTOMATIONS, filter);
+         assertEquals("id", filter.getOrderBys().get(0).getFieldName());
+      }
+
+      /////////////////////////////////////////////////////////////////////////////////////
+      // add dynamic default value behaviors, confirm create/modify date fields are used //
+      /////////////////////////////////////////////////////////////////////////////////////
+      createDate.withBehavior(DynamicDefaultValueBehavior.CREATE_DATE);
+      modifyDate.withBehavior(DynamicDefaultValueBehavior.MODIFY_DATE);
+
+      {
+         QQueryFilter filter = new QQueryFilter();
+         PollingAutomationPerTableRunner.addOrderByToQueryFilter(table, AutomationStatus.PENDING_INSERT_AUTOMATIONS, filter);
+         assertEquals("createDate", filter.getOrderBys().get(0).getFieldName());
+      }
+
+      {
+         QQueryFilter filter = new QQueryFilter();
+         PollingAutomationPerTableRunner.addOrderByToQueryFilter(table, AutomationStatus.PENDING_UPDATE_AUTOMATIONS, filter);
+         assertEquals("modifyDate", filter.getOrderBys().get(0).getFieldName());
+      }
+
+   }
+
 }

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/dashboard/AbstractHTMLWidgetRendererTest.java

@@ -0,0 +1,58 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.actions.dashboard;
+
+
+import com.kingsrook.qqq.backend.core.BaseTest;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
+import com.kingsrook.qqq.backend.core.utils.TestUtils;
+import org.junit.jupiter.api.Test;
+import static org.assertj.core.api.Assertions.assertThat;
+
+
+/*******************************************************************************
+ ** Unit test for AbstractHTMLWidgetRenderer 
+ *******************************************************************************/
+class AbstractHTMLWidgetRendererTest extends BaseTest
+{
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void test() throws QException
+   {
+      String link = AbstractHTMLWidgetRenderer.getCountLink(null, TestUtils.TABLE_NAME_PERSON, new QQueryFilter()
+         .withCriteria(new QFilterCriteria("a", QCriteriaOperator.EQUALS, 1))
+         .withCriteria(new QFilterCriteria("a", QCriteriaOperator.EQUALS, 1)), 2
+      );
+
+      ////////////////////////////////////////////////////
+      // assert that filter de-duplication is occurring //
+      ////////////////////////////////////////////////////
+      assertThat(link).doesNotMatch(".*EQUALS.*EQUALS.*");
+   }
+
+}

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/tables/helpers/UpdateActionRecordSplitHelperTest.java

@@ -38,7 +38,6 @@ import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 import com.kingsrook.qqq.backend.core.model.statusmessages.SystemErrorStatusMessage;
 import com.kingsrook.qqq.backend.core.utils.ListingHash;
 import org.junit.jupiter.api.Test;
-import static org.assertj.core.api.Assertions.assertThat;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertFalse;
 import static org.junit.jupiter.api.Assertions.assertTrue;
@@ -64,6 +63,7 @@ class UpdateActionRecordSplitHelperTest extends BaseTest
          .withField(new QFieldMetaData("B", QFieldType.INTEGER))
          .withField(new QFieldMetaData("modifyDate", QFieldType.DATE_TIME)));
 
+      Instant now = Instant.now();
       UpdateInput updateInput = new UpdateInput(tableName)
          .withRecord(new QRecord().withValue("id", 1).withValue("A", 1))
          .withRecord(new QRecord().withValue("id", 2).withValue("A", 2))
@@ -71,6 +71,7 @@ class UpdateActionRecordSplitHelperTest extends BaseTest
          .withRecord(new QRecord().withValue("id", 4).withValue("B", 3))
          .withRecord(new QRecord().withValue("id", 5).withValue("B", 3))
          .withRecord(new QRecord().withValue("id", 6).withValue("A", 4).withValue("B", 5));
+      updateInput.getRecords().forEach(r -> r.setValue("modifyDate", now));
       UpdateActionRecordSplitHelper updateActionRecordSplitHelper = new UpdateActionRecordSplitHelper();
       updateActionRecordSplitHelper.init(updateInput);
       ListingHash<List<String>, QRecord> recordsByFieldBeingUpdated = updateActionRecordSplitHelper.getRecordsByFieldBeingUpdated();
@@ -78,12 +79,6 @@ class UpdateActionRecordSplitHelperTest extends BaseTest
       Function<Collection<QRecord>, Set<Integer>> extractIds = (records) ->
          records.stream().map(r -> r.getValueInteger("id")).collect(Collectors.toSet());
 
-      ////////////////////////////////////////
-      // validate that modify dates got set //
-      ////////////////////////////////////////
-      updateInput.getRecords().forEach(r ->
-         assertThat(r.getValue("modifyDate")).isInstanceOf(Instant.class));
-
       //////////////////////////////////////////////////////////////
       // validate the grouping of records by fields-being-updated //
       //////////////////////////////////////////////////////////////

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/values/ValueBehaviorApplierTest.java

@@ -39,7 +39,9 @@ import static org.junit.jupiter.api.Assertions.fail;
 
 
 /*******************************************************************************
- ** Unit test for ValueBehaviorApplier
+ ** Unit test for ValueBehaviorApplier - and also providing coverage for
+ ** ValueTooLongBehavior (the first implementation, which was previously in the
+ ** class under test).
  *******************************************************************************/
 class ValueBehaviorApplierTest extends BaseTest
 {
@@ -61,7 +63,7 @@ class ValueBehaviorApplierTest extends BaseTest
          new QRecord().withValue("id", 2).withValue("firstName", "John").withValue("lastName", "Last name too long").withValue("email", "john@smith.com"),
          new QRecord().withValue("id", 3).withValue("firstName", "First name too long").withValue("lastName", "Smith").withValue("email", "john.smith@emaildomainwayytolongtofit.com")
       );
-      ValueBehaviorApplier.applyFieldBehaviors(qInstance, table, recordList);
+      ValueBehaviorApplier.applyFieldBehaviors(ValueBehaviorApplier.Action.INSERT, qInstance, table, recordList);
 
       assertEquals("First name", getRecordById(recordList, 1).getValueString("firstName"));
       assertEquals("Last na...", getRecordById(recordList, 2).getValueString("lastName"));
@@ -93,7 +95,7 @@ class ValueBehaviorApplierTest extends BaseTest
          new QRecord().withValue("id", 1).withValue("firstName", "First name too long").withValue("lastName", null).withValue("email", "john@smith.com"),
          new QRecord().withValue("id", 2).withValue("firstName", "").withValue("lastName", "Last name too long").withValue("email", "john@smith.com")
       );
-      ValueBehaviorApplier.applyFieldBehaviors(qInstance, table, recordList);
+      ValueBehaviorApplier.applyFieldBehaviors(ValueBehaviorApplier.Action.INSERT, qInstance, table, recordList);
 
       assertEquals("First name too long", getRecordById(recordList, 1).getValueString("firstName"));
       assertNull(getRecordById(recordList, 1).getValueString("lastName"));

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricherTest.java

@@ -29,6 +29,7 @@ import java.util.Optional;
 import com.kingsrook.qqq.backend.core.BaseTest;
 import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.AdornmentType;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.DynamicDefaultValueBehavior;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.FieldAdornment;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
@@ -493,4 +494,39 @@ class QInstanceEnricherTest extends BaseTest
       return (tableMetaData);
    }
 
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testCreateDateAndModifyDateBehaviors()
+   {
+      QInstance qInstance = TestUtils.defineInstance();
+      qInstance.addTable(newTable("A", "id", "createDate", "modifyDate"));
+      QTableMetaData table = qInstance.getTable("A");
+
+      ////////////////////////////////////////////////
+      // make sure behavior wasn't there by default //
+      ////////////////////////////////////////////////
+      assertNull(table.getField("createDate").getBehaviorOnlyIfSet(DynamicDefaultValueBehavior.class));
+      assertNull(table.getField("modifyDate").getBehaviorOnlyIfSet(DynamicDefaultValueBehavior.class));
+
+      //////////////////////////////////////////////////////////////////
+      // make sure if config'ing off the adding of the behavior works //
+      //////////////////////////////////////////////////////////////////
+      new QInstanceEnricher(qInstance)
+         .withConfigAddDynamicDefaultValuesToFieldsNamedCreateDateAndModifyDate(false)
+         .enrich();
+      assertNull(table.getField("createDate").getBehaviorOnlyIfSet(DynamicDefaultValueBehavior.class));
+      assertNull(table.getField("modifyDate").getBehaviorOnlyIfSet(DynamicDefaultValueBehavior.class));
+
+      /////////////////////////////////////////////////////////////////////////////////////////////
+      // make sure default value for the config (e.g., in a new enricher) is to add the behavior //
+      /////////////////////////////////////////////////////////////////////////////////////////////
+      new QInstanceEnricher(qInstance).enrich();
+      assertEquals(DynamicDefaultValueBehavior.CREATE_DATE, table.getField("createDate").getBehaviorOnlyIfSet(DynamicDefaultValueBehavior.class));
+      assertEquals(DynamicDefaultValueBehavior.MODIFY_DATE, table.getField("modifyDate").getBehaviorOnlyIfSet(DynamicDefaultValueBehavior.class));
+   }
+
 }

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/data/QRecordTest.java

@@ -26,6 +26,7 @@ import java.math.BigDecimal;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.LinkedHashMap;
+import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
 import com.kingsrook.qqq.backend.core.BaseTest;
@@ -175,7 +176,7 @@ class QRecordTest extends BaseTest
     **
     *******************************************************************************/
    @Test
-   void testListAsValue()
+   void testArrayListAsValue()
    {
       ArrayList<Integer> originalArrayList        = new ArrayList<>(List.of(1, 2, 3));
       QRecord            recordWithArrayListValue = new QRecord().withValue("myList", originalArrayList);
@@ -196,6 +197,31 @@ class QRecordTest extends BaseTest
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testLinkedListAsValue()
+   {
+      LinkedList<Integer> originalLinkedList        = new LinkedList<>(List.of(1, 2, 3));
+      QRecord             recordWithLinkedListValue = new QRecord().withValue("myList", originalLinkedList);
+      QRecord             cloneWithLinkedListValue  = new QRecord(recordWithLinkedListValue);
+
+      ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      // the clone list and original list should be equals (have contents that are equals), but not be the same (reference) //
+      ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      assertEquals(List.of(1, 2, 3), cloneWithLinkedListValue.getValue("myList"));
+      assertNotSame(originalLinkedList, cloneWithLinkedListValue.getValue("myList"));
+
+      //////////////////////////////////////////////////////////////////////////////////////////////////////
+      // make sure a change to the original list doesn't change the cloned list (as it was cloned deeply) //
+      //////////////////////////////////////////////////////////////////////////////////////////////////////
+      originalLinkedList.add(4);
+      assertNotEquals(originalLinkedList, cloneWithLinkedListValue.getValue("myList"));
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/metadata/fields/DynamicDefaultValueBehaviorTest.java

@@ -0,0 +1,139 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.model.metadata.fields;
+
+
+import java.time.LocalDate;
+import java.util.List;
+import com.kingsrook.qqq.backend.core.BaseTest;
+import com.kingsrook.qqq.backend.core.actions.values.ValueBehaviorApplier;
+import com.kingsrook.qqq.backend.core.context.QContext;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+import com.kingsrook.qqq.backend.core.utils.TestUtils;
+import org.junit.jupiter.api.Test;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertNull;
+
+
+/*******************************************************************************
+ ** Unit test for DynamicDefaultValueBehavior 
+ *******************************************************************************/
+class DynamicDefaultValueBehaviorTest extends BaseTest
+{
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testCreateDateHappyPath()
+   {
+      QInstance      qInstance = QContext.getQInstance();
+      QTableMetaData table     = qInstance.getTable(TestUtils.TABLE_NAME_PERSON_MEMORY);
+
+      QRecord record = new QRecord().withValue("id", 1);
+      ValueBehaviorApplier.applyFieldBehaviors(ValueBehaviorApplier.Action.INSERT, qInstance, table, List.of(record));
+
+      assertNotNull(record.getValue("createDate"));
+      assertNotNull(record.getValue("modifyDate"));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testModifyDateHappyPath()
+   {
+      QInstance      qInstance = QContext.getQInstance();
+      QTableMetaData table     = qInstance.getTable(TestUtils.TABLE_NAME_PERSON_MEMORY);
+
+      QRecord record = new QRecord().withValue("id", 1);
+      ValueBehaviorApplier.applyFieldBehaviors(ValueBehaviorApplier.Action.UPDATE, qInstance, table, List.of(record));
+
+      assertNull(record.getValue("createDate"));
+      assertNotNull(record.getValue("modifyDate"));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testNone()
+   {
+      QInstance      qInstance = QContext.getQInstance();
+      QTableMetaData table     = qInstance.getTable(TestUtils.TABLE_NAME_PERSON_MEMORY);
+      table.getField("createDate").withBehavior(DynamicDefaultValueBehavior.NONE);
+      table.getField("modifyDate").withBehavior(DynamicDefaultValueBehavior.NONE);
+
+      QRecord record = new QRecord().withValue("id", 1);
+
+      ValueBehaviorApplier.applyFieldBehaviors(ValueBehaviorApplier.Action.INSERT, qInstance, table, List.of(record));
+      assertNull(record.getValue("createDate"));
+      assertNull(record.getValue("modifyDate"));
+
+      ValueBehaviorApplier.applyFieldBehaviors(ValueBehaviorApplier.Action.UPDATE, qInstance, table, List.of(record));
+      assertNull(record.getValue("createDate"));
+      assertNull(record.getValue("modifyDate"));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testDateInsteadOfDateTimeField()
+   {
+      QInstance      qInstance = QContext.getQInstance();
+      QTableMetaData table     = qInstance.getTable(TestUtils.TABLE_NAME_PERSON_MEMORY);
+      table.getField("createDate").withType(QFieldType.DATE);
+
+      QRecord record = new QRecord().withValue("id", 1);
+      ValueBehaviorApplier.applyFieldBehaviors(ValueBehaviorApplier.Action.INSERT, qInstance, table, List.of(record));
+      assertNotNull(record.getValue("createDate"));
+      assertThat(record.getValue("createDate")).isInstanceOf(LocalDate.class);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testNonDateField()
+   {
+      QInstance      qInstance = QContext.getQInstance();
+      QTableMetaData table     = qInstance.getTable(TestUtils.TABLE_NAME_PERSON_MEMORY);
+      table.getField("firstName").withBehavior(DynamicDefaultValueBehavior.CREATE_DATE);
+
+      QRecord record = new QRecord().withValue("id", 1);
+      ValueBehaviorApplier.applyFieldBehaviors(ValueBehaviorApplier.Action.INSERT, qInstance, table, List.of(record));
+      assertNull(record.getValue("firstName"));
+   }
+
+}

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/metadata/fields/QFieldMetaDataTest.java

@@ -0,0 +1,71 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.model.metadata.fields;
+
+
+import com.kingsrook.qqq.backend.core.BaseTest;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
+import org.junit.jupiter.api.Test;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+
+/*******************************************************************************
+ ** Unit test for QFieldMetaData 
+ *******************************************************************************/
+class QFieldMetaDataTest extends BaseTest
+{
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testFieldBehaviors()
+   {
+      /////////////////////////////////////////
+      // create field - assert default state //
+      /////////////////////////////////////////
+      QFieldMetaData field = new QFieldMetaData("createDate", QFieldType.DATE_TIME);
+      assertTrue(CollectionUtils.nullSafeIsEmpty(field.getBehaviors()));
+      assertNull(field.getBehaviorOnlyIfSet(DynamicDefaultValueBehavior.class));
+      assertEquals(DynamicDefaultValueBehavior.NONE, field.getBehaviorOrDefault(new QInstance(), DynamicDefaultValueBehavior.class));
+
+      //////////////////////////////////////
+      // add NONE behavior - assert state //
+      //////////////////////////////////////
+      field.withBehavior(DynamicDefaultValueBehavior.NONE);
+      assertEquals(1, field.getBehaviors().size());
+      assertEquals(DynamicDefaultValueBehavior.NONE, field.getBehaviorOnlyIfSet(DynamicDefaultValueBehavior.class));
+      assertEquals(DynamicDefaultValueBehavior.NONE, field.getBehaviorOrDefault(new QInstance(), DynamicDefaultValueBehavior.class));
+
+      /////////////////////////////////////////////////////////
+      // replace behavior - assert it got rid of the old one //
+      /////////////////////////////////////////////////////////
+      field.withBehavior(DynamicDefaultValueBehavior.CREATE_DATE);
+      assertEquals(1, field.getBehaviors().size());
+      assertEquals(DynamicDefaultValueBehavior.CREATE_DATE, field.getBehaviorOnlyIfSet(DynamicDefaultValueBehavior.class));
+      assertEquals(DynamicDefaultValueBehavior.CREATE_DATE, field.getBehaviorOrDefault(new QInstance(), DynamicDefaultValueBehavior.class));
+   }
+
+}

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/automation/HealBadRecordAutomationStatusesProcessStepTest.java

@@ -0,0 +1,204 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.processes.implementations.automation;
+
+
+import java.time.Instant;
+import java.time.temporal.ChronoUnit;
+import java.util.List;
+import java.util.Map;
+import com.kingsrook.qqq.backend.core.BaseTest;
+import com.kingsrook.qqq.backend.core.actions.automation.AutomationStatus;
+import com.kingsrook.qqq.backend.core.actions.automation.RecordAutomationStatusUpdater;
+import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
+import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
+import com.kingsrook.qqq.backend.core.context.QContext;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
+import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.DynamicDefaultValueBehavior;
+import com.kingsrook.qqq.backend.core.utils.TestUtils;
+import org.junit.jupiter.api.Test;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+
+/*******************************************************************************
+ ** Unit test for HealBadRecordAutomationStatusesProcessStep 
+ *******************************************************************************/
+class HealBadRecordAutomationStatusesProcessStepTest extends BaseTest
+{
+   private static String tableName = TestUtils.TABLE_NAME_PERSON_MEMORY;
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testTwoFailedUpdates() throws QException
+   {
+      new InsertAction().execute(new InsertInput(tableName).withRecords(List.of(new QRecord(), new QRecord())));
+      List<QRecord> records = queryAllRecords();
+      RecordAutomationStatusUpdater.setAutomationStatusInRecordsAndUpdate(QContext.getQInstance().getTable(tableName), records, AutomationStatus.FAILED_UPDATE_AUTOMATIONS, null);
+
+      assertThat(queryAllRecords()).allMatch(r -> AutomationStatus.FAILED_UPDATE_AUTOMATIONS.getId().equals(getAutomationStatus(r)));
+
+      RunBackendStepOutput output = runProcessStep();
+
+      assertEquals(2, output.getValueInteger("totalRecordsUpdated"));
+      assertThat(queryAllRecords()).allMatch(r -> AutomationStatus.PENDING_UPDATE_AUTOMATIONS.getId().equals(getAutomationStatus(r)));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testOneFailedUpdateOneFailedInsert() throws QException
+   {
+      new InsertAction().execute(new InsertInput(tableName).withRecords(List.of(new QRecord(), new QRecord())));
+      List<QRecord> records = queryAllRecords();
+      RecordAutomationStatusUpdater.setAutomationStatusInRecordsAndUpdate(QContext.getQInstance().getTable(tableName), records.subList(0, 1), AutomationStatus.FAILED_UPDATE_AUTOMATIONS, null);
+      RecordAutomationStatusUpdater.setAutomationStatusInRecordsAndUpdate(QContext.getQInstance().getTable(tableName), records.subList(1, 2), AutomationStatus.FAILED_INSERT_AUTOMATIONS, null);
+
+      assertThat(queryAllRecords())
+         .anyMatch(r -> AutomationStatus.FAILED_UPDATE_AUTOMATIONS.getId().equals(getAutomationStatus(r)))
+         .anyMatch(r -> AutomationStatus.FAILED_INSERT_AUTOMATIONS.getId().equals(getAutomationStatus(r)));
+
+      RunBackendStepOutput output = runProcessStep();
+
+      assertEquals(2, output.getValueInteger("totalRecordsUpdated"));
+      assertThat(queryAllRecords())
+         .anyMatch(r -> AutomationStatus.PENDING_UPDATE_AUTOMATIONS.getId().equals(getAutomationStatus(r)))
+         .anyMatch(r -> AutomationStatus.PENDING_INSERT_AUTOMATIONS.getId().equals(getAutomationStatus(r)));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testOldRunning() throws QException
+   {
+      /////////////////////////////////////////////////
+      // temporarily remove the modify-date behavior //
+      /////////////////////////////////////////////////
+      QContext.getQInstance().getTable(tableName).getField("modifyDate").withBehavior(DynamicDefaultValueBehavior.NONE);
+
+      //////////////////////////////////////////////////////////////////////////
+      // insert 2 records, one with an old modifyDate, one with 6 minutes ago //
+      //////////////////////////////////////////////////////////////////////////
+      new InsertAction().execute(new InsertInput(tableName).withRecords(List.of(
+         new QRecord().withValue("firstName", "Darin").withValue("modifyDate", Instant.parse("2023-01-01T12:00:00Z")),
+         new QRecord().withValue("firstName", "Tim").withValue("modifyDate", Instant.now().minus(6, ChronoUnit.MINUTES))
+      )));
+      List<QRecord> records = queryAllRecords();
+
+      ///////////////////////////////////////////////////////
+      // put those records both in status: running-updates //
+      ///////////////////////////////////////////////////////
+      RecordAutomationStatusUpdater.setAutomationStatusInRecordsAndUpdate(QContext.getQInstance().getTable(tableName), records, AutomationStatus.RUNNING_UPDATE_AUTOMATIONS, null);
+
+      assertThat(queryAllRecords())
+         .allMatch(r -> AutomationStatus.RUNNING_UPDATE_AUTOMATIONS.getId().equals(getAutomationStatus(r)));
+
+      /////////////////////////////////////
+      // restore the modifyDate behavior //
+      /////////////////////////////////////
+      QContext.getQInstance().getTable(tableName).getField("modifyDate").withBehavior(DynamicDefaultValueBehavior.MODIFY_DATE);
+
+      /////////////////////////
+      // run code under test //
+      /////////////////////////
+      RunBackendStepOutput output = runProcessStep();
+
+      /////////////////////////////////////////////////////////////////////////////////////////////
+      // assert we updated 1 (the old one) to pending-updates, the other left as running-updates //
+      /////////////////////////////////////////////////////////////////////////////////////////////
+      assertEquals(1, output.getValueInteger("totalRecordsUpdated"));
+      assertThat(queryAllRecords())
+         .anyMatch(r -> AutomationStatus.PENDING_UPDATE_AUTOMATIONS.getId().equals(getAutomationStatus(r)))
+         .anyMatch(r -> AutomationStatus.RUNNING_UPDATE_AUTOMATIONS.getId().equals(getAutomationStatus(r)));
+
+      /////////////////////////////////
+      // re-run, with 3-minute limit //
+      /////////////////////////////////
+      output = runProcessStep(new RunBackendStepInput().withValues(Map.of("minutesOldLimit", 3)));
+
+      /////////////////////////////////////////////////////////////////
+      // assert that one updated too, and all are now pending-update //
+      /////////////////////////////////////////////////////////////////
+      assertEquals(1, output.getValueInteger("totalRecordsUpdated"));
+      assertThat(queryAllRecords())
+         .allMatch(r -> AutomationStatus.PENDING_UPDATE_AUTOMATIONS.getId().equals(getAutomationStatus(r)));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private static Integer getAutomationStatus(QRecord r)
+   {
+      return r.getValueInteger(TestUtils.standardQqqAutomationStatusField().getName());
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private static List<QRecord> queryAllRecords() throws QException
+   {
+      return new QueryAction().execute(new QueryInput(tableName)).getRecords();
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private static RunBackendStepOutput runProcessStep() throws QException
+   {
+      RunBackendStepInput input = new RunBackendStepInput();
+      return runProcessStep(input);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private static RunBackendStepOutput runProcessStep(RunBackendStepInput input) throws QException
+   {
+      RunBackendStepOutput output = new RunBackendStepOutput();
+      new HealBadRecordAutomationStatusesProcessStep().run(input, output);
+      return output;
+   }
+
+}

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/automation/RunTableAutomationsProcessStepTest.java

@@ -0,0 +1,97 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.processes.implementations.automation;
+
+
+import com.kingsrook.qqq.backend.core.BaseTest;
+import com.kingsrook.qqq.backend.core.actions.automation.AutomationStatus;
+import com.kingsrook.qqq.backend.core.actions.tables.GetAction;
+import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
+import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.utils.TestUtils;
+import com.kingsrook.qqq.backend.core.utils.lambdas.UnsafeSupplier;
+import org.junit.jupiter.api.Test;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+
+/*******************************************************************************
+ ** Unit test for RunTableAutomationsProcessStep 
+ *******************************************************************************/
+class RunTableAutomationsProcessStepTest extends BaseTest
+{
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void test() throws Exception
+   {
+      UnsafeSupplier<Integer, ?> getAutomationStatus = () -> new GetAction().executeForRecord(new GetInput(TestUtils.TABLE_NAME_PERSON_MEMORY).withPrimaryKey(1)).getValueInteger("qqqAutomationStatus");
+
+      new InsertAction().execute(new InsertInput(TestUtils.TABLE_NAME_PERSON_MEMORY).withRecord(new QRecord()));
+      assertEquals(AutomationStatus.PENDING_INSERT_AUTOMATIONS.getId(), getAutomationStatus.get());
+
+      RunBackendStepInput input = new RunBackendStepInput();
+      input.addValue("tableName", TestUtils.TABLE_NAME_PERSON_MEMORY);
+      RunBackendStepOutput output = new RunBackendStepOutput();
+      new RunTableAutomationsProcessStep().run(input, output);
+      assertEquals("true", output.getValue("ok"));
+
+      assertEquals(AutomationStatus.OK.getId(), getAutomationStatus.get());
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testThrowsWithoutTableName() throws QException
+   {
+      RunBackendStepInput  input  = new RunBackendStepInput();
+      RunBackendStepOutput output = new RunBackendStepOutput();
+      assertThatThrownBy(() -> new RunTableAutomationsProcessStep().run(input, output))
+         .hasMessageContaining("Missing required input value: tableName");
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testThrowsWithInvalidTableName() throws QException
+   {
+      RunBackendStepInput  input  = new RunBackendStepInput();
+      RunBackendStepOutput output = new RunBackendStepOutput();
+      input.addValue("tableName", "asdf");
+      assertThatThrownBy(() -> new RunTableAutomationsProcessStep().run(input, output))
+         .hasMessageContaining("Unrecognized table name: asdf");
+   }
+
+}

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/columnstats/ColumnStatsStepTest.java

@@ -24,6 +24,7 @@ package com.kingsrook.qqq.backend.core.processes.implementations.columnstats;
 
 import java.io.Serializable;
 import java.math.BigDecimal;
+import java.time.Instant;
 import java.util.List;
 import java.util.Map;
 import com.kingsrook.qqq.backend.core.BaseTest;
@@ -91,4 +92,50 @@ class ColumnStatsStepTest extends BaseTest
          .hasFieldOrPropertyWithValue("percent", new BigDecimal("16.67"));
    }
 
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testDateTimesRollupByHour() throws QException
+   {
+      InsertInput insertInput = new InsertInput();
+      insertInput.setTableName(TestUtils.TABLE_NAME_PERSON_MEMORY);
+      insertInput.setRecords(List.of(
+         new QRecord().withValue("timestamp", Instant.parse("2024-01-31T09:59:01Z")),
+         new QRecord().withValue("timestamp", Instant.parse("2024-01-31T09:59:59Z")),
+         new QRecord().withValue("timestamp", Instant.parse("2024-01-31T10:00:00Z")),
+         new QRecord().withValue("timestamp", Instant.parse("2024-01-31T10:01:01Z")),
+         new QRecord().withValue("timestamp", Instant.parse("2024-01-31T10:59:59Z")),
+         new QRecord().withValue("timestamp", null)
+      ));
+      new InsertAction().execute(insertInput);
+
+      RunBackendStepInput input = new RunBackendStepInput();
+      input.addValue("tableName", TestUtils.TABLE_NAME_PERSON_MEMORY);
+      input.addValue("fieldName", "timestamp");
+      input.addValue("orderBy", "count.desc");
+
+      RunBackendStepOutput output = new RunBackendStepOutput();
+      new ColumnStatsStep().run(input, output);
+
+      Map<String, Serializable> values = output.getValues();
+
+      @SuppressWarnings("unchecked")
+      List<QRecord> valueCounts = (List<QRecord>) values.get("valueCounts");
+
+      assertThat(valueCounts.get(0).getValues())
+         .hasFieldOrPropertyWithValue("timestamp", Instant.parse("2024-01-31T10:00:00Z"))
+         .hasFieldOrPropertyWithValue("count", 3);
+
+      assertThat(valueCounts.get(1).getValues())
+         .hasFieldOrPropertyWithValue("timestamp", Instant.parse("2024-01-31T09:00:00Z"))
+         .hasFieldOrPropertyWithValue("count", 2);
+
+      assertThat(valueCounts.get(2).getValues())
+         .hasFieldOrPropertyWithValue("timestamp", null)
+         .hasFieldOrPropertyWithValue("count", 1);
+   }
+
 }

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/savedfilters/SavedFilterProcessTests.java

@@ -1,143 +0,0 @@
-/*
- * QQQ - Low-code Application Framework for Engineers.
- * Copyright (C) 2021-2023.  Kingsrook, LLC
- * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
- * contact@kingsrook.com
- * https://github.com/Kingsrook/
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program.  If not, see <https://www.gnu.org/licenses/>.
- */
-
-package com.kingsrook.qqq.backend.core.processes.implementations.savedfilters;
-
-
-import java.util.List;
-import com.kingsrook.qqq.backend.core.BaseTest;
-import com.kingsrook.qqq.backend.core.actions.processes.RunProcessAction;
-import com.kingsrook.qqq.backend.core.context.QContext;
-import com.kingsrook.qqq.backend.core.exceptions.QException;
-import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput;
-import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessOutput;
-import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
-import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
-import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
-import com.kingsrook.qqq.backend.core.model.data.QRecord;
-import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
-import com.kingsrook.qqq.backend.core.model.savedfilters.SavedFiltersMetaDataProvider;
-import com.kingsrook.qqq.backend.core.utils.JsonUtils;
-import com.kingsrook.qqq.backend.core.utils.TestUtils;
-import org.junit.jupiter.api.Test;
-import static org.junit.jupiter.api.Assertions.assertEquals;
-import static org.junit.jupiter.api.Assertions.assertNotNull;
-
-
-/*******************************************************************************
- ** Unit test for all saved filter processes
- *******************************************************************************/
-class SavedFilterProcessTests extends BaseTest
-{
-
-   /*******************************************************************************
-    **
-    *******************************************************************************/
-   @Test
-   void test() throws QException
-   {
-      QInstance qInstance = QContext.getQInstance();
-      new SavedFiltersMetaDataProvider().defineAll(qInstance, TestUtils.MEMORY_BACKEND_NAME, null);
-      String tableName = TestUtils.TABLE_NAME_PERSON_MEMORY;
-
-      {
-         ///////////////////////////////////////////
-         // query - should be no filters to start //
-         ///////////////////////////////////////////
-         RunProcessInput runProcessInput = new RunProcessInput();
-         runProcessInput.setProcessName(QuerySavedFilterProcess.getProcessMetaData().getName());
-         runProcessInput.addValue("tableName", tableName);
-         RunProcessOutput runProcessOutput = new RunProcessAction().execute(runProcessInput);
-         assertEquals(0, ((List<?>) runProcessOutput.getValues().get("savedFilterList")).size());
-      }
-
-      Integer savedFilterId;
-      {
-         ////////////////////////
-         // store a new filter //
-         ////////////////////////
-         RunProcessInput runProcessInput = new RunProcessInput();
-         runProcessInput.setProcessName(StoreSavedFilterProcess.getProcessMetaData().getName());
-         runProcessInput.addValue("label", "My Filter");
-         runProcessInput.addValue("tableName", tableName);
-         runProcessInput.addValue("filterJson", JsonUtils.toJson(new QQueryFilter(new QFilterCriteria("id", QCriteriaOperator.EQUALS, 47))));
-         RunProcessOutput runProcessOutput = new RunProcessAction().execute(runProcessInput);
-         List<QRecord>    savedFilterList  = (List<QRecord>) runProcessOutput.getValues().get("savedFilterList");
-         assertEquals(1, savedFilterList.size());
-         savedFilterId = savedFilterList.get(0).getValueInteger("id");
-         assertNotNull(savedFilterId);
-      }
-
-      {
-         ////////////////////////////////////
-         // query - should find our filter //
-         ////////////////////////////////////
-         RunProcessInput runProcessInput = new RunProcessInput();
-         runProcessInput.setProcessName(QuerySavedFilterProcess.getProcessMetaData().getName());
-         runProcessInput.addValue("tableName", tableName);
-         RunProcessOutput runProcessOutput = new RunProcessAction().execute(runProcessInput);
-         List<QRecord>    savedFilterList  = (List<QRecord>) runProcessOutput.getValues().get("savedFilterList");
-         assertEquals(1, savedFilterList.size());
-         assertEquals(1, savedFilterList.get(0).getValueInteger("id"));
-         assertEquals("My Filter", savedFilterList.get(0).getValueString("label"));
-      }
-
-      {
-         ///////////////////////
-         // update our filter //
-         ///////////////////////
-         RunProcessInput runProcessInput = new RunProcessInput();
-         runProcessInput.setProcessName(StoreSavedFilterProcess.getProcessMetaData().getName());
-         runProcessInput.addValue("id", savedFilterId);
-         runProcessInput.addValue("label", "My Updated Filter");
-         runProcessInput.addValue("tableName", tableName);
-         runProcessInput.addValue("filterJson", JsonUtils.toJson(new QQueryFilter(new QFilterCriteria("id", QCriteriaOperator.EQUALS, 47))));
-         RunProcessOutput runProcessOutput = new RunProcessAction().execute(runProcessInput);
-         List<QRecord>    savedFilterList  = (List<QRecord>) runProcessOutput.getValues().get("savedFilterList");
-         assertEquals(1, savedFilterList.size());
-         assertEquals(1, savedFilterList.get(0).getValueInteger("id"));
-         assertEquals("My Updated Filter", savedFilterList.get(0).getValueString("label"));
-      }
-
-      {
-         ///////////////////////
-         // delete our filter //
-         ///////////////////////
-         RunProcessInput runProcessInput = new RunProcessInput();
-         runProcessInput.setProcessName(DeleteSavedFilterProcess.getProcessMetaData().getName());
-         runProcessInput.addValue("id", savedFilterId);
-         RunProcessOutput runProcessOutput = new RunProcessAction().execute(runProcessInput);
-      }
-
-      {
-         ////////////////////////////////////////
-         // query - should be no filters again //
-         ////////////////////////////////////////
-         RunProcessInput runProcessInput = new RunProcessInput();
-         runProcessInput.setProcessName(QuerySavedFilterProcess.getProcessMetaData().getName());
-         runProcessInput.addValue("tableName", tableName);
-         RunProcessOutput runProcessOutput = new RunProcessAction().execute(runProcessInput);
-         assertEquals(0, ((List<?>) runProcessOutput.getValues().get("savedFilterList")).size());
-      }
-
-   }
-
-}

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/savedviews/SavedViewProcessTests.java

@@ -0,0 +1,189 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.processes.implementations.savedviews;
+
+
+import java.util.List;
+import com.kingsrook.qqq.backend.core.BaseTest;
+import com.kingsrook.qqq.backend.core.actions.processes.RunProcessAction;
+import com.kingsrook.qqq.backend.core.context.QContext;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.exceptions.QUserFacingException;
+import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput;
+import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessOutput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.savedviews.SavedViewsMetaDataProvider;
+import com.kingsrook.qqq.backend.core.utils.JsonUtils;
+import com.kingsrook.qqq.backend.core.utils.TestUtils;
+import org.junit.jupiter.api.Test;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+
+
+/*******************************************************************************
+ ** Unit test for all saved view processes
+ *******************************************************************************/
+class SavedViewProcessTests extends BaseTest
+{
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void test() throws QException
+   {
+      QInstance qInstance = QContext.getQInstance();
+      new SavedViewsMetaDataProvider().defineAll(qInstance, TestUtils.MEMORY_BACKEND_NAME, null);
+      String tableName = TestUtils.TABLE_NAME_PERSON_MEMORY;
+
+      {
+         /////////////////////////////////////////
+         // query - should be no views to start //
+         /////////////////////////////////////////
+         RunProcessInput runProcessInput = new RunProcessInput();
+         runProcessInput.setProcessName(QuerySavedViewProcess.getProcessMetaData().getName());
+         runProcessInput.addValue("tableName", tableName);
+         RunProcessOutput runProcessOutput = new RunProcessAction().execute(runProcessInput);
+         assertEquals(0, ((List<?>) runProcessOutput.getValues().get("savedViewList")).size());
+      }
+
+      Integer savedViewId;
+      {
+         //////////////////////
+         // store a new view //
+         //////////////////////
+         RunProcessInput runProcessInput = new RunProcessInput();
+         runProcessInput.setProcessName(StoreSavedViewProcess.getProcessMetaData().getName());
+         runProcessInput.addValue("label", "My View");
+         runProcessInput.addValue("tableName", tableName);
+         runProcessInput.addValue("viewJson", JsonUtils.toJson(new QQueryFilter(new QFilterCriteria("id", QCriteriaOperator.EQUALS, 47))));
+         RunProcessOutput runProcessOutput = new RunProcessAction().execute(runProcessInput);
+         List<QRecord> savedViewList = (List<QRecord>) runProcessOutput.getValues().get("savedViewList");
+         assertEquals(1, savedViewList.size());
+         savedViewId = savedViewList.get(0).getValueInteger("id");
+         assertNotNull(savedViewId);
+
+         //////////////////////////////////////////////////////////////////
+         // try to store it again - should throw a "duplicate" exception //
+         //////////////////////////////////////////////////////////////////
+         assertThatThrownBy(() -> new RunProcessAction().execute(runProcessInput))
+            .isInstanceOf(QUserFacingException.class)
+            .hasMessageContaining("already have a saved view");
+      }
+
+      {
+         ///////////////////////////////////
+         // query - should find our views //
+         ///////////////////////////////////
+         RunProcessInput runProcessInput = new RunProcessInput();
+         runProcessInput.setProcessName(QuerySavedViewProcess.getProcessMetaData().getName());
+         runProcessInput.addValue("tableName", tableName);
+         RunProcessOutput runProcessOutput = new RunProcessAction().execute(runProcessInput);
+         List<QRecord> savedViewList = (List<QRecord>) runProcessOutput.getValues().get("savedViewList");
+         assertEquals(1, savedViewList.size());
+         assertEquals(1, savedViewList.get(0).getValueInteger("id"));
+         assertEquals("My View", savedViewList.get(0).getValueString("label"));
+      }
+
+      {
+         /////////////////////
+         // update our view //
+         /////////////////////
+         RunProcessInput runProcessInput = new RunProcessInput();
+         runProcessInput.setProcessName(StoreSavedViewProcess.getProcessMetaData().getName());
+         runProcessInput.addValue("id", savedViewId);
+         runProcessInput.addValue("label", "My Updated View");
+         runProcessInput.addValue("tableName", tableName);
+         runProcessInput.addValue("viewJson", JsonUtils.toJson(new QQueryFilter(new QFilterCriteria("id", QCriteriaOperator.EQUALS, 47))));
+         RunProcessOutput runProcessOutput = new RunProcessAction().execute(runProcessInput);
+         List<QRecord> savedViewList = (List<QRecord>) runProcessOutput.getValues().get("savedViewList");
+         assertEquals(1, savedViewList.size());
+         assertEquals(1, savedViewList.get(0).getValueInteger("id"));
+         assertEquals("My Updated View", savedViewList.get(0).getValueString("label"));
+      }
+
+      Integer anotherSavedViewId;
+      {
+         /////////////////////////////////////////////////////////////////////////////////////////////
+         // store a second one w/ different name (will be used below in update-dupe-check use-case) //
+         /////////////////////////////////////////////////////////////////////////////////////////////
+         RunProcessInput runProcessInput = new RunProcessInput();
+         runProcessInput.setProcessName(StoreSavedViewProcess.getProcessMetaData().getName());
+         runProcessInput.addValue("label", "My Second View");
+         runProcessInput.addValue("tableName", tableName);
+         runProcessInput.addValue("viewJson", JsonUtils.toJson(new QQueryFilter(new QFilterCriteria("id", QCriteriaOperator.EQUALS, 47))));
+         RunProcessOutput runProcessOutput = new RunProcessAction().execute(runProcessInput);
+         List<QRecord>    savedViewList    = (List<QRecord>) runProcessOutput.getValues().get("savedViewList");
+         anotherSavedViewId = savedViewList.get(0).getValueInteger("id");
+      }
+
+      {
+         /////////////////////////////////////////////////
+         // try to rename the second to match the first //
+         /////////////////////////////////////////////////
+         RunProcessInput runProcessInput = new RunProcessInput();
+         runProcessInput.setProcessName(StoreSavedViewProcess.getProcessMetaData().getName());
+         runProcessInput.addValue("id", anotherSavedViewId);
+         runProcessInput.addValue("label", "My Updated View");
+         runProcessInput.addValue("tableName", tableName);
+         runProcessInput.addValue("viewJson", JsonUtils.toJson(new QQueryFilter(new QFilterCriteria("id", QCriteriaOperator.EQUALS, 47))));
+         
+         //////////////////////////////////////////
+         // should throw a "duplicate" exception //
+         //////////////////////////////////////////
+         assertThatThrownBy(() -> new RunProcessAction().execute(runProcessInput))
+            .isInstanceOf(QUserFacingException.class)
+            .hasMessageContaining("already have a saved view");
+      }
+
+      {
+         //////////////////////
+         // delete our views //
+         //////////////////////
+         RunProcessInput runProcessInput = new RunProcessInput();
+         runProcessInput.setProcessName(DeleteSavedViewProcess.getProcessMetaData().getName());
+         runProcessInput.addValue("id", savedViewId);
+         RunProcessOutput runProcessOutput = new RunProcessAction().execute(runProcessInput);
+
+         runProcessInput.addValue("id", anotherSavedViewId);
+         runProcessOutput = new RunProcessAction().execute(runProcessInput);
+      }
+
+      {
+         //////////////////////////////////////
+         // query - should be no views again //
+         //////////////////////////////////////
+         RunProcessInput runProcessInput = new RunProcessInput();
+         runProcessInput.setProcessName(QuerySavedViewProcess.getProcessMetaData().getName());
+         runProcessInput.addValue("tableName", tableName);
+         RunProcessOutput runProcessOutput = new RunProcessAction().execute(runProcessInput);
+         assertEquals(0, ((List<?>) runProcessOutput.getValues().get("savedViewList")).size());
+      }
+
+   }
+
+}

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/ExceptionUtilsTest.java

@@ -116,6 +116,19 @@ class ExceptionUtilsTest extends BaseTest
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testGetTopAndBottomMessages()
+   {
+      assertEquals("foo", ExceptionUtils.getTopAndBottomMessages(new Exception("foo")));
+      assertEquals("foo: bar", ExceptionUtils.getTopAndBottomMessages(new Exception("foo", new Exception("bar"))));
+      assertEquals("foo: baz", ExceptionUtils.getTopAndBottomMessages(new Exception("foo", new Exception("bar", new Exception("baz")))));
+   }
+
+
+
    /*******************************************************************************
     ** Test exception class - lets you set the cause, easier to create a loop.
     *******************************************************************************/

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/QQueryFilterDeduperTest.java

@@ -0,0 +1,355 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.utils;
+
+
+import com.kingsrook.qqq.backend.core.BaseTest;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
+import org.junit.jupiter.api.Test;
+import static com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator.EQUALS;
+import static com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator.GREATER_THAN;
+import static com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator.IN;
+import static com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator.NOT_EQUALS;
+import static com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator.NOT_IN;
+import static com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter.BooleanOperator.OR;
+import static com.kingsrook.qqq.backend.core.utils.QQueryFilterDeduper.dedupeFilter;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotSame;
+import static org.junit.jupiter.api.Assertions.assertNull;
+
+
+/*******************************************************************************
+ ** Unit test for QQueryFilterDeduper 
+ *******************************************************************************/
+class QQueryFilterDeduperTest extends BaseTest
+{
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testDegenerateCases()
+   {
+      assertNull(dedupeFilter(null));
+
+      QQueryFilter empty = new QQueryFilter();
+      assertEquals(empty, dedupeFilter(empty));
+      assertNotSame(empty, dedupeFilter(empty)); // method always clones, so, just assert that.
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testSimpleFiltersWithNoChanges()
+   {
+      QQueryFilter oneCriteria = new QQueryFilter()
+         .withCriteria(new QFilterCriteria("a", EQUALS, 1));
+      assertEquals(oneCriteria, dedupeFilter(oneCriteria));
+      assertNotSame(oneCriteria, dedupeFilter(oneCriteria));
+
+      QQueryFilter twoCriteriaDifferentFields = new QQueryFilter()
+         .withCriteria(new QFilterCriteria("a", EQUALS, 1))
+         .withCriteria(new QFilterCriteria("b", GREATER_THAN, 2));
+      assertEquals(twoCriteriaDifferentFields, dedupeFilter(twoCriteriaDifferentFields));
+      assertNotSame(twoCriteriaDifferentFields, dedupeFilter(twoCriteriaDifferentFields));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testOrs()
+   {
+      ///////////////////////////////////////////////////////
+      // we've only written the simplest cases with ORs... //
+      ///////////////////////////////////////////////////////
+      assertEquals(new QQueryFilter().withBooleanOperator(OR).withCriteria(new QFilterCriteria("a", EQUALS, 1)), dedupeFilter(new QQueryFilter()
+         .withBooleanOperator(OR)
+         .withCriteria(new QFilterCriteria("a", EQUALS, 1))
+         .withCriteria(new QFilterCriteria("a", EQUALS, 1))
+         .withCriteria(new QFilterCriteria("a", EQUALS, 1))
+      ));
+
+      //////////////////////////////////////////////////////////////////////
+      // just not built at this time - obviously, could become an IN list //
+      //////////////////////////////////////////////////////////////////////
+      QQueryFilter notSupportedOrTwoEquals = new QQueryFilter()
+         .withBooleanOperator(OR)
+         .withCriteria(new QFilterCriteria("f", EQUALS, 1))
+         .withCriteria(new QFilterCriteria("f", EQUALS, 2));
+      assertEquals(notSupportedOrTwoEquals, dedupeFilter(notSupportedOrTwoEquals));
+
+      ///////////////////////////////////////////////////////////////////////////////////
+      // I think the logic would be, that the EQUALS 1 would be removed (is redundant) //
+      ///////////////////////////////////////////////////////////////////////////////////
+      QQueryFilter notSupportedOrEqualsNotEquals = new QQueryFilter()
+         .withBooleanOperator(OR)
+         .withCriteria(new QFilterCriteria("f", EQUALS, 1))
+         .withCriteria(new QFilterCriteria("f", NOT_EQUALS, 2));
+      assertEquals(notSupportedOrEqualsNotEquals, dedupeFilter(notSupportedOrEqualsNotEquals));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testMoreOperators()
+   {
+      //////////////////////////////////////////////////////////////////////
+      // only simplest case (of criteria being .equals()) is supported... //
+      //////////////////////////////////////////////////////////////////////
+      assertEquals(new QQueryFilter().withCriteria(new QFilterCriteria("a", GREATER_THAN, 1)), dedupeFilter(new QQueryFilter()
+         .withCriteria(new QFilterCriteria("a", GREATER_THAN, 1))
+         .withCriteria(new QFilterCriteria("a", GREATER_THAN, 1))
+      ));
+
+      ///////////////////////////////////////////////////////////////////////////////////
+      // in theory, we could do more, but we just haven't yet (e.g, this could be > 5) //
+      ///////////////////////////////////////////////////////////////////////////////////
+      QQueryFilter tooComplex = new QQueryFilter()
+         .withCriteria(new QFilterCriteria("f", GREATER_THAN, 1))
+         .withCriteria(new QFilterCriteria("f", GREATER_THAN, 5));
+      assertEquals(tooComplex, dedupeFilter(tooComplex));
+
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testAllEquals()
+   {
+      assertEquals(new QQueryFilter().withCriteria(new QFilterCriteria("a", EQUALS, 1)), dedupeFilter(new QQueryFilter()
+         .withCriteria(new QFilterCriteria("a", EQUALS, 1))
+         .withCriteria(new QFilterCriteria("a", EQUALS, 1))
+      ));
+
+      assertEquals(new QQueryFilter().withCriteria(new QFilterCriteria("a", EQUALS, 1)), dedupeFilter(new QQueryFilter()
+         .withCriteria(new QFilterCriteria("a", EQUALS, 1))
+         .withCriteria(new QFilterCriteria("a", EQUALS, 1))
+         .withCriteria(new QFilterCriteria("a", EQUALS, 1))
+      ));
+
+      assertEquals(new QQueryFilter()
+            .withCriteria(new QFilterCriteria("a", EQUALS, 1))
+            .withCriteria(new QFilterCriteria("b", EQUALS, 2))
+            .withCriteria(new QFilterCriteria("c", EQUALS, 3)),
+         dedupeFilter(new QQueryFilter()
+            .withCriteria(new QFilterCriteria("a", EQUALS, 1))
+            .withCriteria(new QFilterCriteria("b", EQUALS, 2))
+            .withCriteria(new QFilterCriteria("a", EQUALS, 1))
+            .withCriteria(new QFilterCriteria("b", EQUALS, 2))
+            .withCriteria(new QFilterCriteria("b", EQUALS, 2))
+            .withCriteria(new QFilterCriteria("a", EQUALS, 1))
+            .withCriteria(new QFilterCriteria("c", EQUALS, 3))
+            .withCriteria(new QFilterCriteria("c", EQUALS, 3))
+            .withCriteria(new QFilterCriteria("c", EQUALS, 3))
+         ));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testEqualsAndNotEqualsAndNotIn()
+   {
+      assertEquals(new QQueryFilter().withCriteria(new QFilterCriteria("f", EQUALS, 1)), dedupeFilter(new QQueryFilter()
+         .withCriteria(new QFilterCriteria("f", EQUALS, 1))
+         .withCriteria(new QFilterCriteria("f", NOT_EQUALS, 2))
+      ));
+
+      assertEquals(new QQueryFilter().withCriteria(new QFilterCriteria("f", EQUALS, 1)), dedupeFilter(new QQueryFilter()
+         .withCriteria(new QFilterCriteria("f", EQUALS, 1))
+         .withCriteria(new QFilterCriteria("f", NOT_EQUALS, 2))
+         .withCriteria(new QFilterCriteria("f", NOT_EQUALS, 3))
+      ));
+
+      assertEquals(new QQueryFilter().withCriteria(new QFilterCriteria("f", EQUALS, 1)), dedupeFilter(new QQueryFilter()
+         .withCriteria(new QFilterCriteria("f", NOT_EQUALS, 2))
+         .withCriteria(new QFilterCriteria("f", EQUALS, 1))
+         .withCriteria(new QFilterCriteria("f", NOT_EQUALS, 3))
+      ));
+
+      assertEquals(new QQueryFilter().withCriteria(new QFilterCriteria("f", EQUALS, 1)), dedupeFilter(new QQueryFilter()
+         .withCriteria(new QFilterCriteria("f", NOT_EQUALS, 2))
+         .withCriteria(new QFilterCriteria("f", NOT_EQUALS, 3))
+         .withCriteria(new QFilterCriteria("f", EQUALS, 1))
+      ));
+
+      assertEquals(new QQueryFilter().withCriteria(new QFilterCriteria("f", EQUALS, 1)), dedupeFilter(new QQueryFilter()
+         .withCriteria(new QFilterCriteria("f", EQUALS, 1))
+         .withCriteria(new QFilterCriteria("f", NOT_IN, 2, 3))
+      ));
+
+      assertEquals(new QQueryFilter().withCriteria(new QFilterCriteria("f", EQUALS, 1)), dedupeFilter(new QQueryFilter()
+         .withCriteria(new QFilterCriteria("f", EQUALS, 1))
+         .withCriteria(new QFilterCriteria("f", NOT_IN, 2, 3))
+         .withCriteria(new QFilterCriteria("f", NOT_EQUALS, 4))
+      ));
+
+      assertEquals(new QQueryFilter().withCriteria(new QFilterCriteria("f", EQUALS, 1)), dedupeFilter(new QQueryFilter()
+         .withCriteria(new QFilterCriteria("f", NOT_IN, 2, 3))
+         .withCriteria(new QFilterCriteria("f", NOT_EQUALS, 4))
+         .withCriteria(new QFilterCriteria("f", EQUALS, 1))
+      ));
+
+      assertEquals(new QQueryFilter().withCriteria(new QFilterCriteria("f", EQUALS, 1)), dedupeFilter(new QQueryFilter()
+         .withCriteria(new QFilterCriteria("f", NOT_IN, 2, 3))
+         .withCriteria(new QFilterCriteria("f", EQUALS, 1))
+         .withCriteria(new QFilterCriteria("f", NOT_EQUALS, 4))
+      ));
+
+      ////////////////////////////////////////////////////////////
+      // this is a contradiction, so we choose not to dedupe it //
+      ////////////////////////////////////////////////////////////
+      QQueryFilter contradiction1 = new QQueryFilter()
+         .withCriteria(new QFilterCriteria("f", EQUALS, 1))
+         .withCriteria(new QFilterCriteria("f", NOT_EQUALS, 1));
+      assertEquals(contradiction1, dedupeFilter(contradiction1));
+
+      QQueryFilter contradiction2 = new QQueryFilter()
+         .withCriteria(new QFilterCriteria("f", EQUALS, 1))
+         .withCriteria(new QFilterCriteria("f", NOT_IN, 0, 1));
+      assertEquals(contradiction2, dedupeFilter(contradiction2));
+
+      ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      // this case can collapse the two not-equals, but then fails to merge the equals with them, because they are a contradiction! //
+      ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      assertEquals(new QQueryFilter()
+            .withCriteria(new QFilterCriteria("f", NOT_IN, 2, 3))
+            .withCriteria(new QFilterCriteria("f", EQUALS, 2)),
+         dedupeFilter(new QQueryFilter()
+            .withCriteria(new QFilterCriteria("f", NOT_EQUALS, 2))
+            .withCriteria(new QFilterCriteria("f", NOT_EQUALS, 3))
+            .withCriteria(new QFilterCriteria("f", EQUALS, 2))
+         ));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testNotEqualsAndNotIn()
+   {
+      assertEquals(new QQueryFilter().withCriteria(new QFilterCriteria("f", NOT_IN, 1, 2, 3)), dedupeFilter(new QQueryFilter()
+         .withCriteria(new QFilterCriteria("f", NOT_EQUALS, 1))
+         .withCriteria(new QFilterCriteria("f", NOT_EQUALS, 2))
+         .withCriteria(new QFilterCriteria("f", NOT_EQUALS, 3))
+      ));
+
+      //////////////////////////////////////////////////////////////////////////////////////////
+      // ideally, maybe, this would have the values ordered 1,2,3, but, is equivalent enough //
+      //////////////////////////////////////////////////////////////////////////////////////////
+      assertEquals(new QQueryFilter().withCriteria(new QFilterCriteria("f", NOT_IN, 2, 3, 1)), dedupeFilter(new QQueryFilter()
+         .withCriteria(new QFilterCriteria("f", NOT_EQUALS, 1))
+         .withCriteria(new QFilterCriteria("f", NOT_IN, 2, 3))
+      ));
+
+      assertEquals(new QQueryFilter().withCriteria(new QFilterCriteria("f", NOT_IN, 2, 3, 1)), dedupeFilter(new QQueryFilter()
+         .withCriteria(new QFilterCriteria("f", NOT_IN, 2, 3))
+         .withCriteria(new QFilterCriteria("f", NOT_EQUALS, 1))
+      ));
+
+      assertEquals(new QQueryFilter().withCriteria(new QFilterCriteria("f", NOT_IN, 1, 2, 3)), dedupeFilter(new QQueryFilter()
+         .withCriteria(new QFilterCriteria("f", NOT_IN, 1, 2))
+         .withCriteria(new QFilterCriteria("f", NOT_IN, 2, 3))
+      ));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testInAndNotEquals()
+   {
+      assertEquals(new QQueryFilter().withCriteria(new QFilterCriteria("f", IN, 2, 3)), dedupeFilter(new QQueryFilter()
+         .withCriteria(new QFilterCriteria("f", NOT_EQUALS, 1))
+         .withCriteria(new QFilterCriteria("f", IN, 2, 3))
+      ));
+
+      assertEquals(new QQueryFilter().withCriteria(new QFilterCriteria("f", IN, 2, 3)), dedupeFilter(new QQueryFilter()
+         .withCriteria(new QFilterCriteria("f", IN, 2, 3))
+         .withCriteria(new QFilterCriteria("f", NOT_EQUALS, 1))
+      ));
+
+      QQueryFilter contradiction1 = new QQueryFilter()
+         .withCriteria(new QFilterCriteria("f", NOT_EQUALS, 1))
+         .withCriteria(new QFilterCriteria("f", IN, 1));
+      assertEquals(contradiction1, dedupeFilter(contradiction1));
+
+      QQueryFilter contradiction2 = new QQueryFilter()
+         .withCriteria(new QFilterCriteria("f", IN, 1))
+         .withCriteria(new QFilterCriteria("f", NOT_EQUALS, 1));
+      assertEquals(contradiction2, dedupeFilter(contradiction2));
+
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testMultipleInLists()
+   {
+      assertEquals(new QQueryFilter().withCriteria(new QFilterCriteria("f", IN, 2)), dedupeFilter(new QQueryFilter()
+         .withCriteria(new QFilterCriteria("f", IN, 1, 2))
+         .withCriteria(new QFilterCriteria("f", IN, 2, 3))
+      ));
+
+      assertEquals(new QQueryFilter().withCriteria(new QFilterCriteria("f", IN, 3, 4)), dedupeFilter(new QQueryFilter()
+         .withCriteria(new QFilterCriteria("f", IN, 1, 2, 3, 4))
+         .withCriteria(new QFilterCriteria("f", IN, 3, 4, 5, 6))
+      ));
+
+      assertEquals(new QQueryFilter().withCriteria(new QFilterCriteria("f", IN, 3)), dedupeFilter(new QQueryFilter()
+         .withCriteria(new QFilterCriteria("f", IN, 1, 2, 3, 4))
+         .withCriteria(new QFilterCriteria("f", IN, 3, 4, 5, 6))
+         .withCriteria(new QFilterCriteria("f", IN, 1, 3, 5, 7))
+      ));
+
+      ///////////////////////////////////////////////////////////////////
+      // contradicting in-lists - we give up and refuse to simplify it //
+      ///////////////////////////////////////////////////////////////////
+      QQueryFilter contradiction = new QQueryFilter()
+         .withCriteria(new QFilterCriteria("f", IN, 1, 2))
+         .withCriteria(new QFilterCriteria("f", IN, 3, 4));
+      assertEquals(contradiction, dedupeFilter(contradiction));
+   }
+
+}

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/StringUtilsTest.java

@@ -78,6 +78,20 @@ class StringUtilsTest extends BaseTest
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void test_safeAppend()
+   {
+      assertEquals("Foo", StringUtils.safeAppend("Foo", null));
+      assertEquals("Foo", StringUtils.safeAppend(null, "Foo"));
+      assertEquals("FooBar", StringUtils.safeAppend("Foo", "Bar"));
+      assertEquals("", StringUtils.safeAppend(null, null));
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java

@@ -720,7 +720,7 @@ public class TestUtils
    /*******************************************************************************
     **
     *******************************************************************************/
-   private static QTableAutomationDetails defineStandardAutomationDetails()
+   public static QTableAutomationDetails defineStandardAutomationDetails()
    {
       return (new QTableAutomationDetails()
          .withProviderName(POLLING_AUTOMATION)

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/memoization/MemoizationTest.java

@@ -32,11 +32,14 @@ import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
 import java.util.concurrent.Future;
 import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
 import com.kingsrook.qqq.backend.core.BaseTest;
 import com.kingsrook.qqq.backend.core.utils.SleepUtils;
+import com.kingsrook.qqq.backend.core.utils.lambdas.UnsafeFunction;
 import org.junit.jupiter.api.Disabled;
 import org.junit.jupiter.api.Test;
 import static org.assertj.core.api.Assertions.assertThat;
+import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertNotNull;
 import static org.junit.jupiter.api.Assertions.assertNull;
 import static org.junit.jupiter.api.Assertions.assertTrue;
@@ -122,6 +125,58 @@ class MemoizationTest extends BaseTest
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testLookupFunction()
+   {
+      AtomicInteger lookupFunctionCallCounter = new AtomicInteger(0);
+
+      Memoization<String, Integer> memoization = new Memoization<>();
+
+      UnsafeFunction<String, Integer, Exception> lookupFunction = numberString ->
+      {
+         lookupFunctionCallCounter.getAndIncrement();
+
+         if(numberString.equals("null"))
+         {
+            return (null);
+         }
+
+         return Integer.parseInt(numberString);
+      };
+
+      //////////////////////////////////////////////////////////////////////////////////////////
+      // get "1" twice - should return 1 each time, and call the lookup function exactly once //
+      //////////////////////////////////////////////////////////////////////////////////////////
+      assertThat(memoization.getResult("1", lookupFunction)).isPresent().contains(1);
+      assertEquals(1, lookupFunctionCallCounter.get());
+
+      assertThat(memoization.getResult("1", lookupFunction)).isPresent().contains(1);
+      assertEquals(1, lookupFunctionCallCounter.get());
+
+      /////////////////////////////////////////////////////////////////////////////////////////////////////////
+      // now get "null" twice - should return null each time, and call the lookup function exactly once more //
+      /////////////////////////////////////////////////////////////////////////////////////////////////////////
+      assertThat(memoization.getResult("null", lookupFunction)).isEmpty();
+      assertEquals(2, lookupFunctionCallCounter.get());
+
+      assertThat(memoization.getResult("null", lookupFunction)).isEmpty();
+      assertEquals(2, lookupFunctionCallCounter.get());
+
+      ////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      // now make a call that throws twice - again, should return null each time, and only do one more loookup call //
+      ////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      assertThat(memoization.getResult(null, lookupFunction)).isEmpty();
+      assertEquals(3, lookupFunctionCallCounter.get());
+
+      assertThat(memoization.getResult(null, lookupFunction)).isEmpty();
+      assertEquals(3, lookupFunctionCallCounter.get());
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/

qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/actions/BaseAPIActionUtil.java

@@ -96,6 +96,7 @@ import org.apache.http.impl.client.HttpClientBuilder;
 import org.apache.http.impl.client.HttpClients;
 import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
 import org.apache.http.util.EntityUtils;
+import org.apache.logging.log4j.Level;
 import org.json.JSONArray;
 import org.json.JSONObject;
 import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
@@ -1081,7 +1082,7 @@ public class BaseAPIActionUtil
                /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
                // trim response body (just to keep logs smaller, or, in case someone consuming logs doesn't want such long lines) //
                /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-               LOG.info("Received successful response with code [" + qResponse.getStatusCode() + "] and content [" + StringUtils.safeTruncate(qResponse.getContent(), getMaxResponseMessageLengthForLog(), "...") + "].");
+               LOG.log(getAPIResponseLogLevel(), "Received successful response with code [" + qResponse.getStatusCode() + "] and content [" + StringUtils.safeTruncate(qResponse.getContent(), getMaxResponseMessageLengthForLog(), "...") + "].");
                return (qResponse);
             }
          }
@@ -1507,4 +1508,14 @@ public class BaseAPIActionUtil
       // nothing to do at this layer, meant to be overridden by subclasses //
       ///////////////////////////////////////////////////////////////////////
    }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   protected Level getAPIResponseLogLevel() throws QException
+   {
+      return (Level.DEBUG);
+   }
 }

qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/importer/FilesystemImporterStep.java

@@ -355,7 +355,7 @@ public class FilesystemImporterStep implements BackendStep
          + "-" + sourceFileName.replaceAll(".*" + File.separator, "");
       path = AbstractBaseFilesystemAction.stripDuplicatedSlashes(path);
 
-      LOG.info("Archiving file", logPair("path", path));
+      LOG.info("Archiving file", logPair("path", path), logPair("archiveBackendName", archiveBackend.getName()), logPair("archiveTableName", archiveTable.getName()));
       archiveActionBase.writeFile(archiveBackend, path, bytes);
 
       return (path);

qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/AbstractS3Action.java

@@ -42,6 +42,7 @@ import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.AbstractF
 import com.kingsrook.qqq.backend.module.filesystem.exceptions.FilesystemException;
 import com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata.S3BackendMetaData;
 import com.kingsrook.qqq.backend.module.filesystem.s3.utils.S3Utils;
+import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
 
 
 /*******************************************************************************
@@ -162,9 +163,18 @@ public class AbstractS3Action extends AbstractBaseFilesystemAction<S3ObjectSumma
    @Override
    public void writeFile(QBackendMetaData backendMetaData, String path, byte[] contents) throws IOException
    {
-      path = stripLeadingSlash(stripDuplicatedSlashes(path));
       String bucketName = ((S3BackendMetaData) backendMetaData).getBucketName();
-      getS3Utils().writeFile(bucketName, path, contents);
+
+      try
+      {
+         path = stripLeadingSlash(stripDuplicatedSlashes(path));
+         getS3Utils().writeFile(bucketName, path, contents);
+      }
+      catch(Exception e)
+      {
+         LOG.warn("Error writing file", e, logPair("path", path), logPair("bucketName", bucketName));
+         throw (new IOException("Error writing file", e));
+      }
    }
 
 

qqq-middleware-api/src/main/java/com/kingsrook/qqq/api/actions/ApiImplementation.java

@@ -992,6 +992,8 @@ public class ApiImplementation
       runProcessInput.setProcessUUID(processUUID);
       // todo i don't think runProcessInput.setAsyncJobCallback();
 
+      PermissionsHelper.checkProcessPermissionThrowing(runProcessInput, processName);
+
       //////////////////////
       // map input values //
       //////////////////////

qqq-middleware-api/src/test/java/com/kingsrook/qqq/api/javalin/QJavalinApiHandlerPermissionsTest.java

@@ -0,0 +1,138 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2023.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.api.javalin;
+
+
+import com.kingsrook.qqq.api.BaseTest;
+import com.kingsrook.qqq.api.TestUtils;
+import com.kingsrook.qqq.api.actions.ApiImplementation;
+import com.kingsrook.qqq.backend.core.exceptions.QInstanceValidationException;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.permissions.PermissionLevel;
+import com.kingsrook.qqq.backend.core.model.metadata.permissions.QPermissionRules;
+import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+import com.kingsrook.qqq.backend.javalin.QJavalinImplementation;
+import io.javalin.apibuilder.EndpointGroup;
+import kong.unirest.Unirest;
+import org.eclipse.jetty.http.HttpStatus;
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import static com.kingsrook.qqq.api.javalin.QJavalinApiHandlerTest.assertErrorResponse;
+
+
+/*******************************************************************************
+ ** Unit test permissions within QJavalinApiHandler
+ *******************************************************************************/
+class QJavalinApiHandlerPermissionsTest extends BaseTest
+{
+   private static final   int    PORT     = 6263;
+   protected static final String BASE_URL = "http://localhost:" + PORT;
+
+   private static final String VERSION = "2023.Q1";
+
+   protected static QJavalinImplementation qJavalinImplementation;
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @BeforeAll
+   static void beforeAll() throws QInstanceValidationException
+   {
+      QInstance qInstance = TestUtils.defineInstance();
+
+      ///////////////////////////////////////////////////
+      // turn on permissions on all tables & processes //
+      ///////////////////////////////////////////////////
+      for(QTableMetaData table : qInstance.getTables().values())
+      {
+         table.withPermissionRules(new QPermissionRules().withLevel(PermissionLevel.READ_INSERT_EDIT_DELETE_PERMISSIONS));
+      }
+
+      for(QProcessMetaData process : qInstance.getProcesses().values())
+      {
+         process.withPermissionRules(new QPermissionRules().withLevel(PermissionLevel.HAS_ACCESS_PERMISSION));
+      }
+
+      qJavalinImplementation = new QJavalinImplementation(qInstance);
+      qJavalinImplementation.startJavalinServer(PORT);
+      EndpointGroup routes = new QJavalinApiHandler(qInstance).getRoutes();
+      qJavalinImplementation.getJavalinService().routes(routes);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @BeforeEach
+   @AfterEach
+   void beforeAndAfterEach()
+   {
+      ApiImplementation.clearCaches();
+   }
+
+
+
+   /*******************************************************************************
+    ** Before the class (all) runs, start a javalin server.
+    **
+    *******************************************************************************/
+   @AfterAll
+   static void afterAll()
+   {
+      qJavalinImplementation.stopJavalinServer();
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void test403s()
+   {
+      ////////////////////////////
+      // tables - single & bulk //
+      ////////////////////////////
+      assertErrorResponse(HttpStatus.FORBIDDEN_403, "You do not have permission", Unirest.get(BASE_URL + "/api/" + VERSION + "/order/query").asString());
+      assertErrorResponse(HttpStatus.FORBIDDEN_403, "You do not have permission", Unirest.get(BASE_URL + "/api/" + VERSION + "/order/1").asString());
+      assertErrorResponse(HttpStatus.FORBIDDEN_403, "You do not have permission", Unirest.post(BASE_URL + "/api/" + VERSION + "/person").asString());
+      assertErrorResponse(HttpStatus.FORBIDDEN_403, "You do not have permission", Unirest.post(BASE_URL + "/api/" + VERSION + "/person/bulk").asString());
+      assertErrorResponse(HttpStatus.FORBIDDEN_403, "You do not have permission", Unirest.delete(BASE_URL + "/api/" + VERSION + "/person/1").asString());
+      assertErrorResponse(HttpStatus.FORBIDDEN_403, "You do not have permission", Unirest.delete(BASE_URL + "/api/" + VERSION + "/person/bulk").asString());
+      assertErrorResponse(HttpStatus.FORBIDDEN_403, "You do not have permission", Unirest.patch(BASE_URL + "/api/" + VERSION + "/person/1").asString());
+      assertErrorResponse(HttpStatus.FORBIDDEN_403, "You do not have permission", Unirest.patch(BASE_URL + "/api/" + VERSION + "/person/bulk").asString());
+
+      ///////////////
+      // processes //
+      ///////////////
+      assertErrorResponse(HttpStatus.FORBIDDEN_403, "You do not have permission", Unirest.get(BASE_URL + "/api/" + VERSION + "/person/getPersonInfo").asString());
+      assertErrorResponse(HttpStatus.FORBIDDEN_403, "You do not have permission", Unirest.post(BASE_URL + "/api/" + VERSION + "/person/transformPeople").asString());
+   }
+
+}

qqq-middleware-api/src/test/java/com/kingsrook/qqq/api/javalin/QJavalinApiHandlerTest.java

@@ -1548,7 +1548,7 @@ class QJavalinApiHandlerTest extends BaseTest
    /*******************************************************************************
     **
     *******************************************************************************/
-   private void assertErrorResponse(Integer expectedStatusCode, String expectedErrorMessage, HttpResponse<String> response)
+   static void assertErrorResponse(Integer expectedStatusCode, String expectedErrorMessage, HttpResponse<String> response)
    {
       if(expectedStatusCode != null)
       {

qqq-middleware-javalin/src/test/java/com/kingsrook/qqq/backend/javalin/TestUtils.java

@@ -66,7 +66,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportView;
 import com.kingsrook.qqq.backend.core.model.metadata.reporting.ReportType;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.AssociatedScript;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
-import com.kingsrook.qqq.backend.core.model.savedfilters.SavedFiltersMetaDataProvider;
+import com.kingsrook.qqq.backend.core.model.savedviews.SavedViewsMetaDataProvider;
 import com.kingsrook.qqq.backend.core.model.scripts.ScriptsMetaDataProvider;
 import com.kingsrook.qqq.backend.core.processes.implementations.mock.MockBackendStep;
 import com.kingsrook.qqq.backend.module.rdbms.jdbc.ConnectionManager;
@@ -157,7 +157,7 @@ public class TestUtils
       qInstance.addBackend(defineMemoryBackend());
       try
       {
-         new SavedFiltersMetaDataProvider().defineAll(qInstance, defineMemoryBackend().getName(), null);
+         new SavedViewsMetaDataProvider().defineAll(qInstance, defineMemoryBackend().getName(), null);
          new ScriptsMetaDataProvider().defineAll(qInstance, defineMemoryBackend().getName(), null);
       }
       catch(Exception e)