updates to allow update input to omit updating the modify date behavior

CE-889 - improvements for streaming etl pipes:

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/async/AsyncRecordPipeLoop.java

@@ -83,6 +83,15 @@ public class AsyncRecordPipeLoop
       long    jobStartTime          = System.currentTimeMillis();
       boolean everCalledConsumer    = false;
 
+      ////////////////////////////////////////////////////////////////////////////
+      // in case the pipe capacity has been made very small (useful in tests!), //
+      // then make the minRecordsToConsume match it.                            //
+      ////////////////////////////////////////////////////////////////////////////
+      if(recordPipe.getCapacity() < minRecordsToConsume)
+      {
+         minRecordsToConsume = recordPipe.getCapacity();
+      }
+
       while(jobState.equals(AsyncJobState.RUNNING))
       {
          if(recordPipe.countAvailableRecords() < minRecordsToConsume)

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/GenerateReportAction.java

@@ -273,7 +273,7 @@ public class GenerateReportAction
       RunBackendStepOutput  transformStepOutput = null;
       if(tableView != null && tableView.getRecordTransformStep() != null)
       {
-         transformStep = QCodeLoader.getBackendStep(AbstractTransformStep.class, tableView.getRecordTransformStep());
+         transformStep = QCodeLoader.getAdHoc(AbstractTransformStep.class, tableView.getRecordTransformStep());
 
          transformStepInput = new RunBackendStepInput();
          transformStepInput.setValues(reportInput.getInputValues());

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/RecordPipe.java

@@ -44,7 +44,8 @@ public class RecordPipe
    private static final long BLOCKING_SLEEP_MILLIS = 100;
    private static final long MAX_SLEEP_LOOP_MILLIS = 300_000; // 5 minutes
 
-   private ArrayBlockingQueue<QRecord> queue = new ArrayBlockingQueue<>(1_000);
+   private int capacity = 1_000;
+   private ArrayBlockingQueue<QRecord> queue = new ArrayBlockingQueue<>(capacity);
 
    private boolean isTerminated = false;
 
@@ -72,6 +73,7 @@ public class RecordPipe
     *******************************************************************************/
    public RecordPipe(Integer overrideCapacity)
    {
+      this.capacity = overrideCapacity;
       queue = new ArrayBlockingQueue<>(overrideCapacity);
    }
 
@@ -213,4 +215,14 @@ public class RecordPipe
       this.postRecordActions = postRecordActions;
    }
 
+
+
+   /*******************************************************************************
+    ** Getter for capacity
+    **
+    *******************************************************************************/
+   public int getCapacity()
+   {
+      return capacity;
+   }
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/InsertAction.java

@@ -242,7 +242,7 @@ public class InsertAction extends AbstractQActionFunction<InsertInput, InsertOut
 
       setDefaultValuesInRecords(table, insertInput.getRecords());
 
-      ValueBehaviorApplier.applyFieldBehaviors(ValueBehaviorApplier.Action.INSERT, insertInput.getInstance(), table, insertInput.getRecords());
+      ValueBehaviorApplier.applyFieldBehaviors(ValueBehaviorApplier.Action.INSERT, insertInput.getInstance(), table, insertInput.getRecords(), null);
 
       runPreInsertCustomizerIfItIsTime(insertInput, preInsertCustomizer, AbstractPreInsertCustomizer.WhenToRun.BEFORE_UNIQUE_KEY_CHECKS);
       setErrorsIfUniqueKeyErrors(insertInput, table);
@@ -455,7 +455,7 @@ public class InsertAction extends AbstractQActionFunction<InsertInput, InsertOut
    private QBackendModuleInterface getBackendModuleInterface(QBackendMetaData backend) throws QException
    {
       QBackendModuleDispatcher qBackendModuleDispatcher = new QBackendModuleDispatcher();
-      QBackendModuleInterface qModule = qBackendModuleDispatcher.getQBackendModule(backend);
+      QBackendModuleInterface  qModule                  = qBackendModuleDispatcher.getQBackendModule(backend);
       return (qModule);
    }
 

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/ReplaceAction.java

@@ -136,19 +136,22 @@ public class ReplaceAction extends AbstractQActionFunction<ReplaceInput, Replace
          UpdateOutput updateOutput = new UpdateAction().execute(updateInput);
          output.setUpdateOutput(updateOutput);
 
-         QQueryFilter deleteFilter = new QQueryFilter(new QFilterCriteria(primaryKeyField, QCriteriaOperator.NOT_IN, primaryKeysToKeep));
+         if(input.getPerformDeletes())
-         if(input.getFilter() != null)
          {
-            deleteFilter.addSubFilter(input.getFilter());
+            QQueryFilter deleteFilter = new QQueryFilter(new QFilterCriteria(primaryKeyField, QCriteriaOperator.NOT_IN, primaryKeysToKeep));
-         }
+            if(input.getFilter() != null)
+            {
+               deleteFilter.addSubFilter(input.getFilter());
+            }
 
-         DeleteInput deleteInput = new DeleteInput();
+            DeleteInput deleteInput = new DeleteInput();
-         deleteInput.setTableName(table.getName());
+            deleteInput.setTableName(table.getName());
-         deleteInput.setQueryFilter(deleteFilter);
+            deleteInput.setQueryFilter(deleteFilter);
-         deleteInput.setTransaction(transaction);
+            deleteInput.setTransaction(transaction);
-         deleteInput.setOmitDmlAudit(input.getOmitDmlAudit());
+            deleteInput.setOmitDmlAudit(input.getOmitDmlAudit());
-         DeleteOutput deleteOutput = new DeleteAction().execute(deleteInput);
+            DeleteOutput deleteOutput = new DeleteAction().execute(deleteInput);
-         output.setDeleteOutput(deleteOutput);
+            output.setDeleteOutput(deleteOutput);
+         }
 
          if(weOwnTheTransaction)
          {

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/UpdateAction.java

@@ -57,6 +57,8 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateOutput;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.DynamicDefaultValueBehavior;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.FieldBehavior;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
 import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinOn;
@@ -72,6 +74,7 @@ import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleDispatcher;
 import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleInterface;
 import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
 import com.kingsrook.qqq.backend.core.utils.ValueUtils;
+import org.apache.commons.lang.BooleanUtils;
 import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
 
 
@@ -244,7 +247,13 @@ public class UpdateAction
       /////////////////////////////
       // run standard validators //
       /////////////////////////////
-      ValueBehaviorApplier.applyFieldBehaviors(ValueBehaviorApplier.Action.UPDATE, updateInput.getInstance(), table, updateInput.getRecords());
+      Set<FieldBehavior<?>> behaviorsToOmit = null;
+      if(BooleanUtils.isTrue(updateInput.getOmitModifyDateUpdate()))
+      {
+         behaviorsToOmit = Set.of(DynamicDefaultValueBehavior.MODIFY_DATE);
+      }
+
+      ValueBehaviorApplier.applyFieldBehaviors(ValueBehaviorApplier.Action.UPDATE, updateInput.getInstance(), table, updateInput.getRecords(), behaviorsToOmit);
       validatePrimaryKeysAreGiven(updateInput);
 
       if(oldRecordList.isPresent())

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/ValueBehaviorApplier.java

@@ -23,6 +23,7 @@ package com.kingsrook.qqq.backend.core.actions.values;
 
 
 import java.util.List;
+import java.util.Set;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
 import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.FieldBehavior;
@@ -32,7 +33,7 @@ import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
 
 
 /*******************************************************************************
- ** Utility class to apply value behaviors to records.  
+ ** Utility class to apply value behaviors to records.
  *******************************************************************************/
 public class ValueBehaviorApplier
 {
@@ -51,7 +52,7 @@ public class ValueBehaviorApplier
    /*******************************************************************************
     **
     *******************************************************************************/
-   public static void applyFieldBehaviors(Action action, QInstance instance, QTableMetaData table, List<QRecord> recordList)
+   public static void applyFieldBehaviors(Action action, QInstance instance, QTableMetaData table, List<QRecord> recordList, Set<FieldBehavior<?>> behaviorsToOmit)
    {
       if(CollectionUtils.nullSafeIsEmpty(recordList))
       {
@@ -62,7 +63,7 @@ public class ValueBehaviorApplier
       {
          for(FieldBehavior<?> fieldBehavior : CollectionUtils.nonNullCollection(field.getBehaviors()))
          {
-            fieldBehavior.apply(action, recordList, instance, table, field);
+            fieldBehavior.apply(action, recordList, instance, table, field, behaviorsToOmit);
          }
       }
    }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/replace/ReplaceInput.java

@@ -39,6 +39,7 @@ public class ReplaceInput extends AbstractTableActionInput
    private UniqueKey           key;
    private List<QRecord>       records;
    private QQueryFilter        filter;
+   private boolean             performDeletes = true;
 
    private boolean omitDmlAudit = false;
 
@@ -207,4 +208,35 @@ public class ReplaceInput extends AbstractTableActionInput
       return (this);
    }
 
+
+
+   /*******************************************************************************
+    ** Getter for performDeletes
+    *******************************************************************************/
+   public boolean getPerformDeletes()
+   {
+      return (this.performDeletes);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for performDeletes
+    *******************************************************************************/
+   public void setPerformDeletes(boolean performDeletes)
+   {
+      this.performDeletes = performDeletes;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for performDeletes
+    *******************************************************************************/
+   public ReplaceInput withPerformDeletes(boolean performDeletes)
+   {
+      this.performDeletes = performDeletes;
+      return (this);
+   }
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/update/UpdateInput.java

@@ -52,8 +52,9 @@ public class UpdateInput extends AbstractTableActionInput
    private Boolean areAllValuesBeingUpdatedTheSame = null;
 
    private boolean omitTriggeringAutomations = false;
-   private boolean omitDmlAudit = false;
+   private boolean omitDmlAudit              = false;
-   private String  auditContext = null;
+   private boolean omitModifyDateUpdate      = false;
+   private String  auditContext              = null;
 
 
 
@@ -353,4 +354,35 @@ public class UpdateInput extends AbstractTableActionInput
       return (this);
    }
 
+
+
+   /*******************************************************************************
+    ** Getter for omitModifyDateUpdate
+    *******************************************************************************/
+   public boolean getOmitModifyDateUpdate()
+   {
+      return (this.omitModifyDateUpdate);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for omitModifyDateUpdate
+    *******************************************************************************/
+   public void setOmitModifyDateUpdate(boolean omitModifyDateUpdate)
+   {
+      this.omitModifyDateUpdate = omitModifyDateUpdate;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for omitModifyDateUpdate
+    *******************************************************************************/
+   public UpdateInput withOmitModifyDateUpdate(boolean omitModifyDateUpdate)
+   {
+      this.omitModifyDateUpdate = omitModifyDateUpdate;
+      return (this);
+   }
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/dashboard/widgets/QWidgetData.java

@@ -34,6 +34,7 @@ import java.util.Map;
 public abstract class QWidgetData
 {
    private String       label;
+   private String       sublabel;
    private String       footerHTML;
    private List<String> dropdownNameList;
    private List<String> dropdownLabelList;
@@ -51,6 +52,7 @@ public abstract class QWidgetData
    private List<List<Serializable>> csvData;
 
 
+
    /*******************************************************************************
     ** Getter for type
     *******************************************************************************/
@@ -356,4 +358,35 @@ public abstract class QWidgetData
       return (this);
    }
 
+
+
+   /*******************************************************************************
+    ** Getter for sublabel
+    *******************************************************************************/
+   public String getSublabel()
+   {
+      return (this.sublabel);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for sublabel
+    *******************************************************************************/
+   public void setSublabel(String sublabel)
+   {
+      this.sublabel = sublabel;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for sublabel
+    *******************************************************************************/
+   public QWidgetData withSublabel(String sublabel)
+   {
+      this.sublabel = sublabel;
+      return (this);
+   }
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/DynamicDefaultValueBehavior.java

@@ -26,6 +26,7 @@ import java.io.Serializable;
 import java.time.Instant;
 import java.time.LocalDate;
 import java.util.List;
+import java.util.Set;
 import com.kingsrook.qqq.backend.core.actions.values.ValueBehaviorApplier;
 import com.kingsrook.qqq.backend.core.logging.QLogger;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
@@ -65,12 +66,16 @@ public enum DynamicDefaultValueBehavior implements FieldBehavior<DynamicDefaultV
     **
     *******************************************************************************/
    @Override
-   public void apply(ValueBehaviorApplier.Action action, List<QRecord> recordList, QInstance instance, QTableMetaData table, QFieldMetaData field)
+   public void apply(ValueBehaviorApplier.Action action, List<QRecord> recordList, QInstance instance, QTableMetaData table, QFieldMetaData field, Set<FieldBehavior<?>> behaviorsToOmit)
    {
       if(this.equals(NONE))
       {
          return;
       }
+      if(behaviorsToOmit != null && behaviorsToOmit.contains(this))
+      {
+         return;
+      }
 
       switch(this)
       {

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/FieldBehavior.java

@@ -23,6 +23,7 @@ package com.kingsrook.qqq.backend.core.model.metadata.fields;
 
 
 import java.util.List;
+import java.util.Set;
 import com.kingsrook.qqq.backend.core.actions.values.ValueBehaviorApplier;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
 import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
@@ -49,7 +50,7 @@ public interface FieldBehavior<T extends FieldBehavior<T>>
    /*******************************************************************************
     ** Apply this behavior to a list of records
     *******************************************************************************/
-   void apply(ValueBehaviorApplier.Action action, List<QRecord> recordList, QInstance instance, QTableMetaData table, QFieldMetaData field);
+   void apply(ValueBehaviorApplier.Action action, List<QRecord> recordList, QInstance instance, QTableMetaData table, QFieldMetaData field, Set<FieldBehavior<?>> behaviorsToOmit);
 
    /*******************************************************************************
     ** control if multiple behaviors of this type should be allowed together on a field.

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/ValueTooLongBehavior.java

@@ -23,6 +23,7 @@ package com.kingsrook.qqq.backend.core.model.metadata.fields;
 
 
 import java.util.List;
+import java.util.Set;
 import com.kingsrook.qqq.backend.core.actions.values.ValueBehaviorApplier;
 import com.kingsrook.qqq.backend.core.logging.QLogger;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
@@ -65,12 +66,16 @@ public enum ValueTooLongBehavior implements FieldBehavior<ValueTooLongBehavior>
     **
     *******************************************************************************/
    @Override
-   public void apply(ValueBehaviorApplier.Action action, List<QRecord> recordList, QInstance instance, QTableMetaData table, QFieldMetaData field)
+   public void apply(ValueBehaviorApplier.Action action, List<QRecord> recordList, QInstance instance, QTableMetaData table, QFieldMetaData field, Set<FieldBehavior<?>> behaviorsToOmit)
    {
       if(this.equals(PASS_THROUGH))
       {
          return;
       }
+      if(behaviorsToOmit != null && behaviorsToOmit.contains(this))
+      {
+         return;
+      }
 
       String fieldName = field.getName();
       if(!QFieldType.STRING.equals(field.getType()))

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/automation/HealBadRecordAutomationStatusesProcessStep.java

@@ -28,6 +28,7 @@ import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
+import java.util.function.Function;
 import com.kingsrook.qqq.backend.core.actions.automation.AutomationStatus;
 import com.kingsrook.qqq.backend.core.actions.processes.BackendStep;
 import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
@@ -101,6 +102,31 @@ public class HealBadRecordAutomationStatusesProcessStep implements BackendStep,
    @Override
    public QProcessMetaData produce(QInstance qInstance) throws QException
    {
+      Function<String, QFrontendStepMetaData> makeReviewOrResultStep = (String name) -> new QFrontendStepMetaData()
+         .withName(name)
+         .withComponent(new NoCodeWidgetFrontendComponentMetaData()
+            .withOutput(new WidgetHtmlLine()
+               .withCondition(new QFilterCriteria("warningCount", QCriteriaOperator.GREATER_THAN, 0))
+               .withWrapper(HtmlWrapper.divWithStyles(HtmlWrapper.STYLE_YELLOW))
+               .withVelocityTemplate("<b>Warning:</b>"))
+            .withOutput(new WidgetHtmlLine()
+               .withCondition(new QFilterCriteria("warningCount", QCriteriaOperator.GREATER_THAN, 0))
+               .withWrapper(HtmlWrapper.divWithStyles(HtmlWrapper.STYLE_INDENT_1))
+               .withWrapper(HtmlWrapper.divWithStyles(HtmlWrapper.STYLE_YELLOW))
+               .withVelocityTemplate("""
+                  <ul>
+                     #foreach($string in $warnings)
+                        <li>$string</li>
+                     #end
+                  </ul>
+                  """)))
+         .withComponent(new QFrontendComponentMetaData().withType(QComponentType.VIEW_FORM))
+         .withViewField(new QFieldMetaData("review".equals(name) ? "totalRecordsToUpdate" : "totalRecordsUpdated", QFieldType.INTEGER) /* todo - didn't display commas... .withDisplayFormat(DisplayFormat.COMMAS) */)
+         .withComponent(new QFrontendComponentMetaData().withType(QComponentType.RECORD_LIST))
+         .withRecordListField(new QFieldMetaData("tableName", QFieldType.STRING))
+         .withRecordListField(new QFieldMetaData("badStatus", QFieldType.STRING))
+         .withRecordListField(new QFieldMetaData("count", QFieldType.INTEGER).withDisplayFormat(DisplayFormat.COMMAS) /* todo - didn't display commas... */);
+
       QProcessMetaData processMetaData = new QProcessMetaData()
          .withName(NAME)
          .withStepList(List.of(
@@ -109,37 +135,14 @@ public class HealBadRecordAutomationStatusesProcessStep implements BackendStep,
                .withComponent(new QFrontendComponentMetaData().withType(QComponentType.EDIT_FORM))
                .withFormField(new QFieldMetaData("tableName", QFieldType.STRING).withPossibleValueSourceName(TablesPossibleValueSourceMetaDataProvider.NAME))
                .withFormField(new QFieldMetaData("minutesOldLimit", QFieldType.INTEGER).withDefaultValue(60)),
+            new QBackendStepMetaData()
+               .withName("preview")
+               .withCode(new QCodeReference(getClass())),
+            makeReviewOrResultStep.apply("review"),
             new QBackendStepMetaData()
                .withName("run")
                .withCode(new QCodeReference(getClass())),
-            new QFrontendStepMetaData()
+            makeReviewOrResultStep.apply("result")
-               .withName("output")
-
-               .withComponent(new NoCodeWidgetFrontendComponentMetaData()
-                  .withOutput(new WidgetHtmlLine()
-                     .withCondition(new QFilterCriteria("warningCount", QCriteriaOperator.GREATER_THAN, 0))
-                     .withWrapper(HtmlWrapper.divWithStyles(HtmlWrapper.STYLE_YELLOW))
-                     .withVelocityTemplate("<b>Warning:</b>"))
-                  .withOutput(new WidgetHtmlLine()
-                     .withCondition(new QFilterCriteria("warningCount", QCriteriaOperator.GREATER_THAN, 0))
-                     .withWrapper(HtmlWrapper.divWithStyles(HtmlWrapper.STYLE_INDENT_1))
-                     .withWrapper(HtmlWrapper.divWithStyles(HtmlWrapper.STYLE_YELLOW))
-                     .withVelocityTemplate("""
-                        <ul>
-                           #foreach($string in $warnings)
-                              <li>$string</li>
-                           #end
-                        </ul>
-                        """)))
-
-               .withComponent(new QFrontendComponentMetaData().withType(QComponentType.VIEW_FORM))
-               .withViewField(new QFieldMetaData("totalRecordsUpdated", QFieldType.INTEGER) /* todo - didn't display commas... .withDisplayFormat(DisplayFormat.COMMAS) */)
-
-               .withComponent(new QFrontendComponentMetaData().withType(QComponentType.RECORD_LIST))
-               .withRecordListField(new QFieldMetaData("tableName", QFieldType.STRING))
-               .withRecordListField(new QFieldMetaData("badStatus", QFieldType.STRING))
-               .withRecordListField(new QFieldMetaData("count", QFieldType.INTEGER).withDisplayFormat(DisplayFormat.COMMAS) /* todo - didn't display commas... */)
-
          ));
 
       return (processMetaData);
@@ -154,6 +157,7 @@ public class HealBadRecordAutomationStatusesProcessStep implements BackendStep,
    public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException
    {
       int recordsUpdated = 0;
+      boolean isReview = "preview".equals(runBackendStepInput.getStepName());
 
       ////////////////////////////////////////////////////////////////////////
       // if a table name is given, validate it, and run for just that table //
@@ -167,7 +171,7 @@ public class HealBadRecordAutomationStatusesProcessStep implements BackendStep,
             throw (new QException("Unrecognized table name: " + tableName));
          }
 
-         recordsUpdated += processTable(tableName, runBackendStepInput, runBackendStepOutput, warnings);
+         recordsUpdated += processTable(isReview, tableName, runBackendStepInput, runBackendStepOutput, warnings);
       }
       else
       {
@@ -176,11 +180,12 @@ public class HealBadRecordAutomationStatusesProcessStep implements BackendStep,
          //////////////////////////////////////////////////////////////////////////
          for(QTableMetaData table : QContext.getQInstance().getTables().values())
          {
-            recordsUpdated += processTable(table.getName(), runBackendStepInput, runBackendStepOutput, warnings);
+            recordsUpdated += processTable(isReview, table.getName(), runBackendStepInput, runBackendStepOutput, warnings);
          }
       }
 
       runBackendStepOutput.addValue("totalRecordsUpdated", recordsUpdated);
+      runBackendStepOutput.addValue("totalRecordsToUpdate", recordsUpdated);
       runBackendStepOutput.addValue("warnings", warnings);
       runBackendStepOutput.addValue("warningCount", warnings.size());
 
@@ -198,7 +203,7 @@ public class HealBadRecordAutomationStatusesProcessStep implements BackendStep,
    /*******************************************************************************
     **
     *******************************************************************************/
-   private int processTable(String tableName, RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput, List<String> warnings)
+   private int processTable(boolean isReview, String tableName, RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput, List<String> warnings)
    {
       try
       {
@@ -216,34 +221,42 @@ public class HealBadRecordAutomationStatusesProcessStep implements BackendStep,
             // find the modify-date field on the table //
             /////////////////////////////////////////////
             String modifyDateFieldName = null;
+            String createDateFieldName = null;
             for(QFieldMetaData field : table.getFields().values())
             {
                if(DynamicDefaultValueBehavior.MODIFY_DATE.equals(field.getBehaviorOnlyIfSet(DynamicDefaultValueBehavior.class)))
                {
                   modifyDateFieldName = field.getName();
-                  break;
+               }
+               if(DynamicDefaultValueBehavior.CREATE_DATE.equals(field.getBehaviorOnlyIfSet(DynamicDefaultValueBehavior.class)))
+               {
+                  createDateFieldName = field.getName();
                }
             }
 
-            if(modifyDateFieldName == null)
+            //////////////////////////////////////////////////////////////////////////////////////////////////
+            // set up a filter to query for records either FAILED, or RUNNING w/ create/modify date too old //
+            //////////////////////////////////////////////////////////////////////////////////////////////////
+            QQueryFilter filter = new QQueryFilter().withBooleanOperator(QQueryFilter.BooleanOperator.OR);
+            filter.addSubFilter(new QQueryFilter().withCriteria(new QFilterCriteria(automationStatusFieldName, QCriteriaOperator.IN, AutomationStatus.FAILED_INSERT_AUTOMATIONS.getId(), AutomationStatus.FAILED_UPDATE_AUTOMATIONS.getId())));
+
+            if(modifyDateFieldName != null)
             {
-               warnings.add("Could not find a Modify Date field on table: " + tableName);
+               filter.addSubFilter(new QQueryFilter()
-               LOG.info("Couldn't find a MODIFY_DATE field on table", logPair("tableName", tableName));
+                  .withCriteria(new QFilterCriteria(automationStatusFieldName, QCriteriaOperator.EQUALS, AutomationStatus.RUNNING_UPDATE_AUTOMATIONS.getId()))
-               return 0;
+                  .withCriteria(new QFilterCriteria(modifyDateFieldName, QCriteriaOperator.LESS_THAN, NowWithOffset.minus(minutesOldLimit, ChronoUnit.MINUTES))));
+            }
+
+            if(createDateFieldName != null)
+            {
+               filter.addSubFilter(new QQueryFilter()
+                  .withCriteria(new QFilterCriteria(automationStatusFieldName, QCriteriaOperator.EQUALS, AutomationStatus.RUNNING_INSERT_AUTOMATIONS.getId()))
+                  .withCriteria(new QFilterCriteria(createDateFieldName, QCriteriaOperator.LESS_THAN, NowWithOffset.minus(minutesOldLimit, ChronoUnit.MINUTES))));
             }
 
-            ////////////////////////////////////////////////////////////////////////
-            // query for records either FAILED, or RUNNING w/ modify date too old //
-            ////////////////////////////////////////////////////////////////////////
             QueryInput queryInput = new QueryInput();
             queryInput.setTableName(tableName);
-            queryInput.setFilter(new QQueryFilter().withBooleanOperator(QQueryFilter.BooleanOperator.OR)
+            queryInput.setFilter(filter);
-               .withSubFilter(new QQueryFilter()
-                  .withCriteria(new QFilterCriteria(automationStatusFieldName, QCriteriaOperator.IN, AutomationStatus.FAILED_INSERT_AUTOMATIONS.getId(), AutomationStatus.FAILED_UPDATE_AUTOMATIONS.getId())))
-               .withSubFilter(new QQueryFilter()
-                  .withCriteria(new QFilterCriteria(automationStatusFieldName, QCriteriaOperator.IN, AutomationStatus.RUNNING_INSERT_AUTOMATIONS.getId(), AutomationStatus.RUNNING_UPDATE_AUTOMATIONS.getId()))
-                  .withCriteria(new QFilterCriteria(modifyDateFieldName, QCriteriaOperator.LESS_THAN, NowWithOffset.minus(minutesOldLimit, ChronoUnit.MINUTES))))
-            );
             QueryOutput queryOutput = new QueryAction().execute(queryInput);
 
             //////////////////////////////////////////////////////////////////////////////////////////////////////////////////
@@ -269,7 +282,10 @@ public class HealBadRecordAutomationStatusesProcessStep implements BackendStep,
                }
             }
 
-            if(!recordsToUpdate.isEmpty())
+            //////////////////////////////////////////////////////////////////////////////////////
+            // if there are record to update (and this isn't the review step), then update them //
+            //////////////////////////////////////////////////////////////////////////////////////
+            if(!recordsToUpdate.isEmpty() && !isReview)
             {
                LOG.info("Healing bad record automation statuses", logPair("tableName", tableName), logPair("count", recordsToUpdate.size()));
                new UpdateAction().execute(new UpdateInput(tableName).withRecords(recordsToUpdate).withOmitTriggeringAutomations(true));
@@ -278,7 +294,7 @@ public class HealBadRecordAutomationStatusesProcessStep implements BackendStep,
             for(Map.Entry<String, Integer> entry : countByStatus.entrySet())
             {
                runBackendStepOutput.addRecord(new QRecord()
-                  .withValue("tableName", tableName)
+                  .withValue("tableName", QContext.getQInstance().getTable(tableName).getLabel())
                   .withValue("badStatus", entry.getKey())
                   .withValue("count", entry.getValue()));
             }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/AbstractLoadStep.java

@@ -24,7 +24,6 @@ package com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwit
 
 import java.util.Optional;
 import com.kingsrook.qqq.backend.core.actions.QBackendTransaction;
-import com.kingsrook.qqq.backend.core.actions.processes.BackendStep;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
 import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
@@ -38,11 +37,11 @@ import com.kingsrook.qqq.backend.core.model.session.QSession;
  ** should be written to the output object's Records, noting that when running
  ** as a streamed-ETL process, those input & output objects will be instances of
  ** the StreamedBackendStep{Input,Output} classes, that will be associated with
- ** a page of records flowing thorugh a pipe.
+ ** a page of records flowing through a pipe.
  **
  ** Also - use the transaction member variable!!!
  *******************************************************************************/
-public abstract class AbstractLoadStep implements BackendStep
+public abstract class AbstractLoadStep
 {
    private   Optional<QBackendTransaction> transaction = Optional.empty();
    protected QSession                      session;
@@ -51,6 +50,25 @@ public abstract class AbstractLoadStep implements BackendStep
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Deprecated
+   public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException
+   {
+      runOnePage(runBackendStepInput, runBackendStepOutput);
+   }
+
+
+   /*******************************************************************************
+    ** todo - make abstract when run is deleted.
+    *******************************************************************************/
+   public void runOnePage(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException
+   {
+
+   }
+
+
    /*******************************************************************************
     ** Allow subclasses to do an action before the run is complete - before any
     ** pages of records are passed in.

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/AbstractTransformStep.java

@@ -24,7 +24,6 @@ package com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwit
 
 import java.util.Optional;
 import com.kingsrook.qqq.backend.core.actions.QBackendTransaction;
-import com.kingsrook.qqq.backend.core.actions.processes.BackendStep;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
 import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
@@ -40,12 +39,33 @@ import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutp
  ** a page of records flowing through a pipe.
  **
  *******************************************************************************/
-public abstract class AbstractTransformStep implements BackendStep, ProcessSummaryProviderInterface
+public abstract class AbstractTransformStep implements ProcessSummaryProviderInterface
 {
    private Optional<QBackendTransaction> transaction = Optional.empty();
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Deprecated
+   public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException
+   {
+      runOnePage(runBackendStepInput, runBackendStepOutput);
+   }
+
+
+
+   /*******************************************************************************
+    ** todo - make abstract when run is deleted.
+    *******************************************************************************/
+   public void runOnePage(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException
+   {
+
+   }
+
+
+
    /*******************************************************************************
     ** Allow subclasses to do an action before the run is complete - before any
     ** pages of records are passed in.

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/BaseStreamedETLStep.java

@@ -63,7 +63,7 @@ public class BaseStreamedETLStep
    protected AbstractTransformStep getTransformStep(RunBackendStepInput runBackendStepInput)
    {
       QCodeReference codeReference = (QCodeReference) runBackendStepInput.getValue(StreamedETLWithFrontendProcess.FIELD_TRANSFORM_CODE);
-      return (QCodeLoader.getBackendStep(AbstractTransformStep.class, codeReference));
+      return (QCodeLoader.getAdHoc(AbstractTransformStep.class, codeReference));
    }
 
 
@@ -74,7 +74,7 @@ public class BaseStreamedETLStep
    protected AbstractLoadStep getLoadStep(RunBackendStepInput runBackendStepInput)
    {
       QCodeReference codeReference = (QCodeReference) runBackendStepInput.getValue(StreamedETLWithFrontendProcess.FIELD_LOAD_CODE);
-      return (QCodeLoader.getBackendStep(AbstractLoadStep.class, codeReference));
+      return (QCodeLoader.getAdHoc(AbstractLoadStep.class, codeReference));
    }
 
 

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/LoadViaUpdateStep.java

@@ -32,6 +32,7 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.InputSource;
 import com.kingsrook.qqq.backend.core.model.actions.tables.QInputSource;
 import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateOutput;
+import org.apache.commons.lang.BooleanUtils;
 
 
 /*******************************************************************************
@@ -40,7 +41,9 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateOutput;
  *******************************************************************************/
 public class LoadViaUpdateStep extends AbstractLoadStep
 {
-   public static final String FIELD_DESTINATION_TABLE = "destinationTable";
+   public static final String FIELD_DESTINATION_TABLE               = "destinationTable";
+   public static final String DO_NOT_UPDATE_MODIFY_DATE_FIELD_NAME  = "doNotUpdateModifyDateFieldName";
+   public static final String DO_NOT_TRIGGER_AUTOMATIONS_FIELD_NAME = "doNotTriggerAutomationsFieldName";
 
 
 
@@ -67,6 +70,15 @@ public class LoadViaUpdateStep extends AbstractLoadStep
       updateInput.setRecords(runBackendStepInput.getRecords());
       getTransaction().ifPresent(updateInput::setTransaction);
       updateInput.setAsyncJobCallback(runBackendStepInput.getAsyncJobCallback());
+
+      //////////////////////////////////////////////////////////////////////////////////////////
+      // look for flags in the input to either not update modify dates or not run automations //
+      //////////////////////////////////////////////////////////////////////////////////////////
+      boolean doNotUpdateModifyDate   = BooleanUtils.isTrue(runBackendStepInput.getValueBoolean(LoadViaUpdateStep.DO_NOT_UPDATE_MODIFY_DATE_FIELD_NAME));
+      boolean doNotTriggerAutomations = BooleanUtils.isTrue(runBackendStepInput.getValueBoolean(LoadViaUpdateStep.DO_NOT_TRIGGER_AUTOMATIONS_FIELD_NAME));
+      updateInput.setOmitModifyDateUpdate(doNotUpdateModifyDate);
+      updateInput.setOmitTriggeringAutomations(doNotTriggerAutomations);
+
       UpdateOutput updateOutput = new UpdateAction().execute(updateInput);
       runBackendStepOutput.getRecords().addAll(updateOutput.getRecords());
    }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLExecuteStep.java

@@ -83,23 +83,32 @@ public class StreamedETLExecuteStep extends BaseStreamedETLStep implements Backe
          // before it can put more records in.                                      //
          /////////////////////////////////////////////////////////////////////////////
          RecordPipe recordPipe;
-         Integer    overrideRecordPipeCapacity = loadStep.getOverrideRecordPipeCapacity(runBackendStepInput);
+         Integer    overrideRecordPipeCapacity = runBackendStepInput.getValueInteger("recordPipeCapacity");
          if(overrideRecordPipeCapacity != null)
          {
             recordPipe = new RecordPipe(overrideRecordPipeCapacity);
-            LOG.debug("per " + loadStep.getClass().getName() + ", we are overriding record pipe capacity to: " + overrideRecordPipeCapacity);
+            LOG.debug("per input value [recordPipeCapacity], we are overriding record pipe capacity to: " + overrideRecordPipeCapacity);
          }
          else
          {
-            overrideRecordPipeCapacity = transformStep.getOverrideRecordPipeCapacity(runBackendStepInput);
+            overrideRecordPipeCapacity = loadStep.getOverrideRecordPipeCapacity(runBackendStepInput);
             if(overrideRecordPipeCapacity != null)
             {
                recordPipe = new RecordPipe(overrideRecordPipeCapacity);
-               LOG.debug("per " + transformStep.getClass().getName() + ", we are overriding record pipe capacity to: " + overrideRecordPipeCapacity);
+               LOG.debug("per " + loadStep.getClass().getName() + ", we are overriding record pipe capacity to: " + overrideRecordPipeCapacity);
             }
             else
             {
-               recordPipe = new RecordPipe();
+               overrideRecordPipeCapacity = transformStep.getOverrideRecordPipeCapacity(runBackendStepInput);
+               if(overrideRecordPipeCapacity != null)
+               {
+                  recordPipe = new RecordPipe(overrideRecordPipeCapacity);
+                  LOG.debug("per " + transformStep.getClass().getName() + ", we are overriding record pipe capacity to: " + overrideRecordPipeCapacity);
+               }
+               else
+               {
+                  recordPipe = new RecordPipe();
+               }
             }
          }
 

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLValidateStep.java

@@ -81,13 +81,41 @@ public class StreamedETLValidateStep extends BaseStreamedETLStep implements Back
       // basically repeat the preview step, but with no limit //
       //////////////////////////////////////////////////////////
       runBackendStepInput.getAsyncJobCallback().updateStatus("Validating Records");
-      RecordPipe          recordPipe  = new RecordPipe();
+
       AbstractExtractStep extractStep = getExtractStep(runBackendStepInput);
+      AbstractTransformStep transformStep = getTransformStep(runBackendStepInput);
+
+      //////////////////////////////////////////////////////////////////////
+      // let the transform step override the capacity for the record pipe //
+      //////////////////////////////////////////////////////////////////////
+      RecordPipe recordPipe;
+      Integer    overrideRecordPipeCapacity = runBackendStepInput.getValueInteger("recordPipeCapacity");
+      if(overrideRecordPipeCapacity != null)
+      {
+         recordPipe = new RecordPipe(overrideRecordPipeCapacity);
+         LOG.debug("per input value [recordPipeCapacity], we are overriding record pipe capacity to: " + overrideRecordPipeCapacity);
+      }
+      else
+      {
+         overrideRecordPipeCapacity = transformStep.getOverrideRecordPipeCapacity(runBackendStepInput);
+         if(overrideRecordPipeCapacity != null)
+         {
+            recordPipe = new RecordPipe(overrideRecordPipeCapacity);
+            LOG.debug("per " + transformStep.getClass().getName() + ", we are overriding record pipe capacity to: " + overrideRecordPipeCapacity);
+         }
+         else
+         {
+            recordPipe = new RecordPipe();
+         }
+      }
+
+      /////////////////////////////
+      // set up the extract step //
+      /////////////////////////////
       extractStep.setLimit(null);
       extractStep.setRecordPipe(recordPipe);
       extractStep.preRun(runBackendStepInput, runBackendStepOutput);
 
-      AbstractTransformStep transformStep = getTransformStep(runBackendStepInput);
       transformStep.preRun(runBackendStepInput, runBackendStepOutput);
 
       List<QRecord> previewRecordList = new ArrayList<>();

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLWithFrontendProcess.java

@@ -151,6 +151,7 @@ public class StreamedETLWithFrontendProcess
             .withField(new QFieldMetaData(FIELD_DEFAULT_QUERY_FILTER, QFieldType.STRING).withDefaultValue(defaultFieldValues.get(FIELD_DEFAULT_QUERY_FILTER)))
             .withField(new QFieldMetaData(FIELD_EXTRACT_CODE, QFieldType.STRING).withDefaultValue(extractStepClass == null ? null : new QCodeReference(extractStepClass)))
             .withField(new QFieldMetaData(FIELD_TRANSFORM_CODE, QFieldType.STRING).withDefaultValue(transformStepClass == null ? null : new QCodeReference(transformStepClass)))
+            .withField(new QFieldMetaData(FIELD_TRANSFORM_CODE + "_expectedType", QFieldType.STRING).withDefaultValue(AbstractTransformStep.class.getName()))
             .withField(new QFieldMetaData(FIELD_PREVIEW_MESSAGE, QFieldType.STRING).withDefaultValue(defaultFieldValues.getOrDefault(FIELD_PREVIEW_MESSAGE, DEFAULT_PREVIEW_MESSAGE_FOR_INSERT)))
             .withField(new QFieldMetaData(FIELD_TRANSACTION_LEVEL, QFieldType.STRING).withDefaultValue(defaultFieldValues.getOrDefault(FIELD_TRANSACTION_LEVEL, TRANSACTION_LEVEL_PROCESS)))
          );
@@ -170,7 +171,8 @@ public class StreamedETLWithFrontendProcess
          .withName(STEP_NAME_EXECUTE)
          .withCode(new QCodeReference(StreamedETLExecuteStep.class))
          .withInputData(new QFunctionInputMetaData()
-            .withField(new QFieldMetaData(FIELD_LOAD_CODE, QFieldType.STRING).withDefaultValue(loadStepClass == null ? null : new QCodeReference(loadStepClass))))
+            .withField(new QFieldMetaData(FIELD_LOAD_CODE, QFieldType.STRING).withDefaultValue(loadStepClass == null ? null : new QCodeReference(loadStepClass)))
+            .withField(new QFieldMetaData(FIELD_LOAD_CODE + "_expectedType", QFieldType.STRING).withDefaultValue(AbstractLoadStep.class.getName())))
          .withOutputMetaData(new QFunctionOutputMetaData()
             .withField(new QFieldMetaData(FIELD_PROCESS_SUMMARY, QFieldType.STRING))
          );

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/tables/UpdateActionTest.java

@@ -23,6 +23,7 @@ package com.kingsrook.qqq.backend.core.actions.tables;
 
 
 import java.math.BigDecimal;
+import java.time.Instant;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Objects;
@@ -787,4 +788,59 @@ class UpdateActionTest extends BaseTest
       }
    }
 
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testUpdateDoNotUpdateModifyDate() throws QException
+   {
+      QContext.getQSession().withSecurityKeyValues(MapBuilder.of(TestUtils.SECURITY_KEY_TYPE_STORE_ALL_ACCESS, ListBuilder.of(true)));
+
+      ////////////////////////////////////////////////////
+      // create a test order and capture its modifyDate //
+      ////////////////////////////////////////////////////
+      InsertInput insertInput = new InsertInput();
+      insertInput.setTableName(TestUtils.TABLE_NAME_ORDER);
+      insertInput.setRecords(List.of(new QRecord()));
+      QRecord qRecord = new InsertAction().execute(insertInput).getRecords().get(0);
+
+      Instant initialModifyDate = qRecord.getValueInstant("modifyDate");
+      assertNotNull(initialModifyDate);
+
+      ///////////////////////////////////////////////////////////////////
+      // update the order, the modify date should be in the future now //
+      ///////////////////////////////////////////////////////////////////
+      {
+         UpdateInput updateInput = new UpdateInput();
+         updateInput.setTableName(TestUtils.TABLE_NAME_ORDER);
+         updateInput.setRecords(List.of(qRecord));
+         QRecord updatedRecord = new UpdateAction().execute(updateInput).getRecords().get(0);
+
+         Instant newModifyDate = updatedRecord.getValueInstant("modifyDate");
+         assertNotNull(initialModifyDate);
+         assertThat(initialModifyDate).isBefore(newModifyDate);
+
+         /////////////////////////////////////////////////////////////////////////////
+         // set the initial modify date to this modify date for the next test below //
+         /////////////////////////////////////////////////////////////////////////////
+         initialModifyDate = newModifyDate;
+      }
+
+      ////////////////////////////////////////////////////////////////////////////////////////////////
+      // now do an update setting flag to not update the modify date, then compare, should be equal //
+      ////////////////////////////////////////////////////////////////////////////////////////////////
+      {
+         UpdateInput updateInput = new UpdateInput();
+         updateInput.setTableName(TestUtils.TABLE_NAME_ORDER);
+         updateInput.setRecords(List.of(qRecord));
+         updateInput.setOmitModifyDateUpdate(true);
+         QRecord updatedRecord = new UpdateAction().execute(updateInput).getRecords().get(0);
+
+         Instant newModifyDate = updatedRecord.getValueInstant("modifyDate");
+         assertNotNull(initialModifyDate);
+         assertThat(initialModifyDate).isEqualTo(newModifyDate);
+      }
+   }
 }

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/values/ValueBehaviorApplierTest.java

@@ -24,10 +24,12 @@ package com.kingsrook.qqq.backend.core.actions.values;
 
 import java.util.List;
 import java.util.Optional;
+import java.util.Set;
 import com.kingsrook.qqq.backend.core.BaseTest;
 import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
 import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.FieldBehavior;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.ValueTooLongBehavior;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 import com.kingsrook.qqq.backend.core.utils.TestUtils;
@@ -35,6 +37,7 @@ import org.junit.jupiter.api.Test;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertFalse;
 import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
 import static org.junit.jupiter.api.Assertions.fail;
 
 
@@ -63,7 +66,7 @@ class ValueBehaviorApplierTest extends BaseTest
          new QRecord().withValue("id", 2).withValue("firstName", "John").withValue("lastName", "Last name too long").withValue("email", "john@smith.com"),
          new QRecord().withValue("id", 3).withValue("firstName", "First name too long").withValue("lastName", "Smith").withValue("email", "john.smith@emaildomainwayytolongtofit.com")
       );
-      ValueBehaviorApplier.applyFieldBehaviors(ValueBehaviorApplier.Action.INSERT, qInstance, table, recordList);
+      ValueBehaviorApplier.applyFieldBehaviors(ValueBehaviorApplier.Action.INSERT, qInstance, table, recordList, null);
 
       assertEquals("First name", getRecordById(recordList, 1).getValueString("firstName"));
       assertEquals("Last na...", getRecordById(recordList, 2).getValueString("lastName"));
@@ -73,6 +76,38 @@ class ValueBehaviorApplierTest extends BaseTest
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testOmitBehaviors()
+   {
+      QInstance      qInstance = QContext.getQInstance();
+      QTableMetaData table     = qInstance.getTable(TestUtils.TABLE_NAME_PERSON_MEMORY);
+      table.getField("firstName").withMaxLength(10).withBehavior(ValueTooLongBehavior.TRUNCATE);
+      table.getField("lastName").withMaxLength(10).withBehavior(ValueTooLongBehavior.TRUNCATE_ELLIPSIS);
+      table.getField("email").withMaxLength(20).withBehavior(ValueTooLongBehavior.ERROR);
+
+      List<QRecord> recordList = List.of(
+         new QRecord().withValue("id", 1).withValue("firstName", "First name too long").withValue("lastName", "Smith").withValue("email", "john@smith.com"),
+         new QRecord().withValue("id", 2).withValue("firstName", "John").withValue("lastName", "Last name too long").withValue("email", "john@smith.com"),
+         new QRecord().withValue("id", 3).withValue("firstName", "First name too long").withValue("lastName", "Smith").withValue("email", "john.smith@emaildomainwayytolongtofit.com")
+      );
+
+      Set<FieldBehavior<?>> behaviorsToOmit = Set.of(ValueTooLongBehavior.ERROR);
+      ValueBehaviorApplier.applyFieldBehaviors(ValueBehaviorApplier.Action.INSERT, qInstance, table, recordList, behaviorsToOmit);
+
+      ///////////////////////////////////////////////////////////////////////////////////////////
+      // the third error behavior was set to be omitted, so no errors should be on that record //
+      ///////////////////////////////////////////////////////////////////////////////////////////
+      assertEquals("First name", getRecordById(recordList, 1).getValueString("firstName"));
+      assertEquals("Last na...", getRecordById(recordList, 2).getValueString("lastName"));
+      assertEquals("john.smith@emaildomainwayytolongtofit.com", getRecordById(recordList, 3).getValueString("email"));
+      assertTrue(getRecordById(recordList, 3).getErrors().isEmpty());
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/
@@ -95,7 +130,7 @@ class ValueBehaviorApplierTest extends BaseTest
          new QRecord().withValue("id", 1).withValue("firstName", "First name too long").withValue("lastName", null).withValue("email", "john@smith.com"),
          new QRecord().withValue("id", 2).withValue("firstName", "").withValue("lastName", "Last name too long").withValue("email", "john@smith.com")
       );
-      ValueBehaviorApplier.applyFieldBehaviors(ValueBehaviorApplier.Action.INSERT, qInstance, table, recordList);
+      ValueBehaviorApplier.applyFieldBehaviors(ValueBehaviorApplier.Action.INSERT, qInstance, table, recordList, null);
 
       assertEquals("First name too long", getRecordById(recordList, 1).getValueString("firstName"));
       assertNull(getRecordById(recordList, 1).getValueString("lastName"));
@@ -118,4 +153,4 @@ class ValueBehaviorApplierTest extends BaseTest
       return (recordOpt.get());
    }
 
-}
+}

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/metadata/fields/DynamicDefaultValueBehaviorTest.java

@@ -24,6 +24,7 @@ package com.kingsrook.qqq.backend.core.model.metadata.fields;
 
 import java.time.LocalDate;
 import java.util.List;
+import java.util.Set;
 import com.kingsrook.qqq.backend.core.BaseTest;
 import com.kingsrook.qqq.backend.core.actions.values.ValueBehaviorApplier;
 import com.kingsrook.qqq.backend.core.context.QContext;
@@ -38,7 +39,7 @@ import static org.junit.jupiter.api.Assertions.assertNull;
 
 
 /*******************************************************************************
- ** Unit test for DynamicDefaultValueBehavior 
+ ** Unit test for DynamicDefaultValueBehavior
  *******************************************************************************/
 class DynamicDefaultValueBehaviorTest extends BaseTest
 {
@@ -53,7 +54,7 @@ class DynamicDefaultValueBehaviorTest extends BaseTest
       QTableMetaData table     = qInstance.getTable(TestUtils.TABLE_NAME_PERSON_MEMORY);
 
       QRecord record = new QRecord().withValue("id", 1);
-      ValueBehaviorApplier.applyFieldBehaviors(ValueBehaviorApplier.Action.INSERT, qInstance, table, List.of(record));
+      ValueBehaviorApplier.applyFieldBehaviors(ValueBehaviorApplier.Action.INSERT, qInstance, table, List.of(record), null);
 
       assertNotNull(record.getValue("createDate"));
       assertNotNull(record.getValue("modifyDate"));
@@ -71,7 +72,7 @@ class DynamicDefaultValueBehaviorTest extends BaseTest
       QTableMetaData table     = qInstance.getTable(TestUtils.TABLE_NAME_PERSON_MEMORY);
 
       QRecord record = new QRecord().withValue("id", 1);
-      ValueBehaviorApplier.applyFieldBehaviors(ValueBehaviorApplier.Action.UPDATE, qInstance, table, List.of(record));
+      ValueBehaviorApplier.applyFieldBehaviors(ValueBehaviorApplier.Action.UPDATE, qInstance, table, List.of(record), null);
 
       assertNull(record.getValue("createDate"));
       assertNotNull(record.getValue("modifyDate"));
@@ -79,6 +80,25 @@ class DynamicDefaultValueBehaviorTest extends BaseTest
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testOmitModifyDateUpdate()
+   {
+      QInstance      qInstance = QContext.getQInstance();
+      QTableMetaData table     = qInstance.getTable(TestUtils.TABLE_NAME_PERSON_MEMORY);
+
+      Set<FieldBehavior<?>> behaviorsToOmit = Set.of(DynamicDefaultValueBehavior.MODIFY_DATE);
+      QRecord               record          = new QRecord().withValue("id", 1);
+      ValueBehaviorApplier.applyFieldBehaviors(ValueBehaviorApplier.Action.UPDATE, qInstance, table, List.of(record), behaviorsToOmit);
+
+      assertNull(record.getValue("createDate"));
+      assertNull(record.getValue("modifyDate"));
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/
@@ -92,11 +112,11 @@ class DynamicDefaultValueBehaviorTest extends BaseTest
 
       QRecord record = new QRecord().withValue("id", 1);
 
-      ValueBehaviorApplier.applyFieldBehaviors(ValueBehaviorApplier.Action.INSERT, qInstance, table, List.of(record));
+      ValueBehaviorApplier.applyFieldBehaviors(ValueBehaviorApplier.Action.INSERT, qInstance, table, List.of(record), null);
       assertNull(record.getValue("createDate"));
       assertNull(record.getValue("modifyDate"));
 
-      ValueBehaviorApplier.applyFieldBehaviors(ValueBehaviorApplier.Action.UPDATE, qInstance, table, List.of(record));
+      ValueBehaviorApplier.applyFieldBehaviors(ValueBehaviorApplier.Action.UPDATE, qInstance, table, List.of(record), null);
       assertNull(record.getValue("createDate"));
       assertNull(record.getValue("modifyDate"));
    }
@@ -114,7 +134,7 @@ class DynamicDefaultValueBehaviorTest extends BaseTest
       table.getField("createDate").withType(QFieldType.DATE);
 
       QRecord record = new QRecord().withValue("id", 1);
-      ValueBehaviorApplier.applyFieldBehaviors(ValueBehaviorApplier.Action.INSERT, qInstance, table, List.of(record));
+      ValueBehaviorApplier.applyFieldBehaviors(ValueBehaviorApplier.Action.INSERT, qInstance, table, List.of(record), null);
       assertNotNull(record.getValue("createDate"));
       assertThat(record.getValue("createDate")).isInstanceOf(LocalDate.class);
    }
@@ -132,8 +152,8 @@ class DynamicDefaultValueBehaviorTest extends BaseTest
       table.getField("firstName").withBehavior(DynamicDefaultValueBehavior.CREATE_DATE);
 
       QRecord record = new QRecord().withValue("id", 1);
-      ValueBehaviorApplier.applyFieldBehaviors(ValueBehaviorApplier.Action.INSERT, qInstance, table, List.of(record));
+      ValueBehaviorApplier.applyFieldBehaviors(ValueBehaviorApplier.Action.INSERT, qInstance, table, List.of(record), null);
       assertNull(record.getValue("firstName"));
    }
 
-}
+}

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/automation/HealBadRecordAutomationStatusesProcessStepTest.java

@@ -103,7 +103,7 @@ class HealBadRecordAutomationStatusesProcessStepTest extends BaseTest
     **
     *******************************************************************************/
    @Test
-   void testOldRunning() throws QException
+   void testOldRunningUpdates() throws QException
    {
       /////////////////////////////////////////////////
       // temporarily remove the modify-date behavior //
@@ -160,6 +160,72 @@ class HealBadRecordAutomationStatusesProcessStepTest extends BaseTest
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testOldRunningInserts() throws QException
+   {
+      ///////////////////////////////////////////////////////////////
+      // temporarily remove the create-date & modify-date behavior //
+      ///////////////////////////////////////////////////////////////
+      QContext.getQInstance().getTable(tableName).getField("modifyDate").withBehavior(DynamicDefaultValueBehavior.NONE);
+      QContext.getQInstance().getTable(tableName).getField("createDate").withBehavior(DynamicDefaultValueBehavior.NONE);
+
+      //////////////////////////////////////////////////////////////////////////
+      // insert 2 records, one with an old createDate, one with 6 minutes ago //
+      // but set both with modifyDate very recent                             //
+      //////////////////////////////////////////////////////////////////////////
+      Instant old    = Instant.parse("2023-01-01T12:00:00Z");
+      Instant recent = Instant.now().minus(6, ChronoUnit.MINUTES);
+      new InsertAction().execute(new InsertInput(tableName).withRecords(List.of(
+         new QRecord().withValue("firstName", "Darin").withValue("createDate", old).withValue("modifyDate", recent),
+         new QRecord().withValue("firstName", "Tim").withValue("createDate", recent).withValue("modifyDate", recent)
+      )));
+      List<QRecord> records = queryAllRecords();
+
+      ///////////////////////////////////////////////////////
+      // put those records both in status: running-inserts //
+      ///////////////////////////////////////////////////////
+      RecordAutomationStatusUpdater.setAutomationStatusInRecordsAndUpdate(QContext.getQInstance().getTable(tableName), records, AutomationStatus.RUNNING_INSERT_AUTOMATIONS, null);
+
+      assertThat(queryAllRecords())
+         .allMatch(r -> AutomationStatus.RUNNING_INSERT_AUTOMATIONS.getId().equals(getAutomationStatus(r)));
+
+      //////////////////////////////////////////////////
+      // restore the createDate & modifyDate behavior //
+      //////////////////////////////////////////////////
+      QContext.getQInstance().getTable(tableName).getField("modifyDate").withBehavior(DynamicDefaultValueBehavior.MODIFY_DATE);
+      QContext.getQInstance().getTable(tableName).getField("createDate").withBehavior(DynamicDefaultValueBehavior.CREATE_DATE);
+
+      /////////////////////////
+      // run code under test //
+      /////////////////////////
+      RunBackendStepOutput output = runProcessStep();
+
+      /////////////////////////////////////////////////////////////////////////////////////////////
+      // assert we updated 1 (the old one) to pending-inserts, the other left as running-inserts //
+      /////////////////////////////////////////////////////////////////////////////////////////////
+      assertEquals(1, output.getValueInteger("totalRecordsUpdated"));
+      assertThat(queryAllRecords())
+         .anyMatch(r -> AutomationStatus.PENDING_INSERT_AUTOMATIONS.getId().equals(getAutomationStatus(r)))
+         .anyMatch(r -> AutomationStatus.RUNNING_INSERT_AUTOMATIONS.getId().equals(getAutomationStatus(r)));
+
+      /////////////////////////////////
+      // re-run, with 3-minute limit //
+      /////////////////////////////////
+      output = runProcessStep(new RunBackendStepInput().withValues(Map.of("minutesOldLimit", 3)));
+
+      /////////////////////////////////////////////////////////////////
+      // assert that one updated too, and all are now pending-insert //
+      /////////////////////////////////////////////////////////////////
+      assertEquals(1, output.getValueInteger("totalRecordsUpdated"));
+      assertThat(queryAllRecords())
+         .allMatch(r -> AutomationStatus.PENDING_INSERT_AUTOMATIONS.getId().equals(getAutomationStatus(r)));
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/

qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/importer/FilesystemImporterStep.java

@@ -362,7 +362,7 @@ public class FilesystemImporterStep implements BackendStep
          + "-" + sourceFileName.replaceAll(".*" + File.separator, "");
       path = AbstractBaseFilesystemAction.stripDuplicatedSlashes(path);
 
-      LOG.info("Archiving file", logPair("path", path));
+      LOG.info("Archiving file", logPair("path", path), logPair("archiveBackendName", archiveBackend.getName()), logPair("archiveTableName", archiveTable.getName()));
       archiveActionBase.writeFile(archiveBackend, path, bytes);
 
       return (path);

qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/AbstractS3Action.java

@@ -42,6 +42,7 @@ import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.AbstractF
 import com.kingsrook.qqq.backend.module.filesystem.exceptions.FilesystemException;
 import com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata.S3BackendMetaData;
 import com.kingsrook.qqq.backend.module.filesystem.s3.utils.S3Utils;
+import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
 
 
 /*******************************************************************************
@@ -162,9 +163,18 @@ public class AbstractS3Action extends AbstractBaseFilesystemAction<S3ObjectSumma
    @Override
    public void writeFile(QBackendMetaData backendMetaData, String path, byte[] contents) throws IOException
    {
-      path = stripLeadingSlash(stripDuplicatedSlashes(path));
       String bucketName = ((S3BackendMetaData) backendMetaData).getBucketName();
-      getS3Utils().writeFile(bucketName, path, contents);
+
+      try
+      {
+         path = stripLeadingSlash(stripDuplicatedSlashes(path));
+         getS3Utils().writeFile(bucketName, path, contents);
+      }
+      catch(Exception e)
+      {
+         LOG.warn("Error writing file", e, logPair("path", path), logPair("bucketName", bucketName));
+         throw (new IOException("Error writing file", e));
+      }
    }