bulk insert & delete w/ pre-validation and warnings and errors and such

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/AbstractPreDeleteCustomizer.java

@@ -23,6 +23,7 @@ package com.kingsrook.qqq.backend.core.actions.customizers;
 
 
 import java.util.List;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
 
@@ -31,6 +32,11 @@ import com.kingsrook.qqq.backend.core.model.data.QRecord;
  ** Abstract class that a table can specify an implementation of, to provide
  ** custom actions before a delete takes place.
  **
+ ** It's important for implementations to be aware of the isPreview field, which
+ ** is set to true when the code is running to give users advice, e.g., on a review
+ ** screen - vs. being false when the action is ACTUALLY happening.  So, if you're doing
+ ** things like storing data, you don't want to do that if isPreview is true!!
+ **
  ** General implementation would be, to iterate over the records (which the DeleteAction
  ** would look up based on the inputs to the delete action), and look at their values:
  ** - possibly adding Errors (`addError`) or Warnings (`addWarning`) to the records
@@ -43,20 +49,19 @@ import com.kingsrook.qqq.backend.core.model.data.QRecord;
  **
  ** Note that the full deleteInput is available as a field in this class.
  **
- ** A future enhancement here may be to take (as fields in this class) the list of
- ** records that the delete action marked in error - the user might want to do
- ** something special with them (idk, try some other way to delete them?)
  *******************************************************************************/
 public abstract class AbstractPreDeleteCustomizer
 {
    protected DeleteInput deleteInput;
 
+   protected boolean isPreview = false;
+
 
 
    /*******************************************************************************
     **
     *******************************************************************************/
-   public abstract List<QRecord> apply(List<QRecord> records);
+   public abstract List<QRecord> apply(List<QRecord> records) throws QException;
 
 
 
@@ -80,4 +85,35 @@ public abstract class AbstractPreDeleteCustomizer
       this.deleteInput = deleteInput;
    }
 
+
+
+   /*******************************************************************************
+    ** Getter for isPreview
+    *******************************************************************************/
+   public boolean getIsPreview()
+   {
+      return (this.isPreview);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for isPreview
+    *******************************************************************************/
+   public void setIsPreview(boolean isPreview)
+   {
+      this.isPreview = isPreview;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for isPreview
+    *******************************************************************************/
+   public AbstractPreDeleteCustomizer withIsPreview(boolean isPreview)
+   {
+      this.isPreview = isPreview;
+      return (this);
+   }
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/AbstractPreInsertCustomizer.java

@@ -32,6 +32,11 @@ import com.kingsrook.qqq.backend.core.model.data.QRecord;
  ** Abstract class that a table can specify an implementation of, to provide
  ** custom actions before an insert takes place.
  **
+ ** It's important for implementations to be aware of the isPreview field, which
+ ** is set to true when the code is running to give users advice, e.g., on a review
+ ** screen - vs. being false when the action is ACTUALLY happening.  So, if you're doing
+ ** things like storing data, you don't want to do that if isPreview is true!!
+ **
  ** General implementation would be, to iterate over the records (the inputs to
  ** the insert action), and look at their values:
  ** - possibly adding Errors (`addError`) or Warnings (`addWarning`) to the records
@@ -46,6 +51,8 @@ public abstract class AbstractPreInsertCustomizer
 {
    protected InsertInput insertInput;
 
+   protected boolean isPreview = false;
+
 
 
    /*******************************************************************************
@@ -74,4 +81,36 @@ public abstract class AbstractPreInsertCustomizer
    {
       this.insertInput = insertInput;
    }
+
+
+
+   /*******************************************************************************
+    ** Getter for isPreview
+    *******************************************************************************/
+   public boolean getIsPreview()
+   {
+      return (this.isPreview);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for isPreview
+    *******************************************************************************/
+   public void setIsPreview(boolean isPreview)
+   {
+      this.isPreview = isPreview;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for isPreview
+    *******************************************************************************/
+   public AbstractPreInsertCustomizer withIsPreview(boolean isPreview)
+   {
+      this.isPreview = isPreview;
+      return (this);
+   }
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/AbstractPreUpdateCustomizer.java

@@ -35,6 +35,11 @@ import com.kingsrook.qqq.backend.core.model.data.QRecord;
  ** Abstract class that a table can specify an implementation of, to provide
  ** custom actions before an update takes place.
  **
+ ** It's important for implementations to be aware of the isPreview field, which
+ ** is set to true when the code is running to give users advice, e.g., on a review
+ ** screen - vs. being false when the action is ACTUALLY happening.  So, if you're doing
+ ** things like storing data, you don't want to do that if isPreview is true!!
+ **
  ** General implementation would be, to iterate over the records (the inputs to
  ** the update action), and look at their values:
  ** - possibly adding Errors (`addError`) or Warnings (`addWarning`) to the records

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/DeleteAction.java

@@ -26,8 +26,10 @@ import java.io.Serializable;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.HashSet;
+import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Objects;
 import java.util.Optional;
 import java.util.Set;
 import java.util.stream.Collectors;
@@ -79,14 +81,32 @@ public class DeleteAction
    {
       ActionHelper.validateSession(deleteInput);
 
-      QTableMetaData table           = deleteInput.getTable();
+      QTableMetaData table               = deleteInput.getTable();
-      String         primaryKeyField = table.getPrimaryKeyField();
+      String         primaryKeyFieldName = table.getPrimaryKeyField();
+      QFieldMetaData primaryKeyField     = table.getField(primaryKeyFieldName);
 
-      if(CollectionUtils.nullSafeHasContents(deleteInput.getPrimaryKeys()) && deleteInput.getQueryFilter() != null)
+      List<Serializable> primaryKeys = deleteInput.getPrimaryKeys();
+      if(CollectionUtils.nullSafeHasContents(primaryKeys) && deleteInput.getQueryFilter() != null)
       {
          throw (new QException("A delete request may not contain both a list of primary keys and a query filter."));
       }
 
+      ////////////////////////////////////////////////////////
+      // make sure the primary keys are of the correct type //
+      ////////////////////////////////////////////////////////
+      if(CollectionUtils.nullSafeHasContents(primaryKeys))
+      {
+         for(int i = 0; i < primaryKeys.size(); i++)
+         {
+            Serializable primaryKey       = primaryKeys.get(i);
+            Serializable valueAsFieldType = ValueUtils.getValueAsFieldType(primaryKeyField.getType(), primaryKey);
+            if(!Objects.equals(primaryKey, valueAsFieldType))
+            {
+               primaryKeys.set(i, valueAsFieldType);
+            }
+         }
+      }
+
       //////////////////////////////////////////////////////
       // load the backend module and its delete interface //
       //////////////////////////////////////////////////////
@@ -119,50 +139,32 @@ public class DeleteAction
       ////////////////////////////////////////////////////////////////////////////////
       Optional<List<QRecord>> oldRecordList = fetchOldRecords(deleteInput, deleteInterface);
 
-      List<QRecord> recordsWithValidationErrors   = new ArrayList<>();
+      List<QRecord>              customizerResult              = performValidations(deleteInput, oldRecordList, false);
-      List<QRecord> recordsWithValidationWarnings = new ArrayList<>();
+      List<QRecord>              recordsWithValidationErrors   = new ArrayList<>();
-      if(oldRecordList.isPresent())
+      Map<Serializable, QRecord> recordsWithValidationWarnings = new LinkedHashMap<>();
+
+      ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      // check if any records got errors in the customizer - if so, remove them from the input list of pkeys to delete //
+      ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      if(customizerResult != null)
       {
-         recordsWithValidationErrors = validateRecordsExistAndCanBeAccessed(deleteInput, oldRecordList.get());
-      }
-
-      ///////////////////////////////////////////////////////////////////////////
-      // after all validations, run the pre-delete customizer, if there is one //
-      ///////////////////////////////////////////////////////////////////////////
-      Optional<AbstractPreDeleteCustomizer> preDeleteCustomizer = QCodeLoader.getTableCustomizer(AbstractPreDeleteCustomizer.class, table, TableCustomizers.PRE_DELETE_RECORD.getRole());
-      if(preDeleteCustomizer.isPresent() && oldRecordList.isPresent())
-      {
-         ////////////////////////////////////////////////////////////////////////////
-         // make list of records that are still good - to pass into the customizer //
-         ////////////////////////////////////////////////////////////////////////////
-         List<QRecord> recordsForCustomizer = makeListOfRecordsNotInErrorList(primaryKeyField, oldRecordList.get(), recordsWithValidationErrors);
-
-         preDeleteCustomizer.get().setDeleteInput(deleteInput);
-         List<QRecord> customizerResult = preDeleteCustomizer.get().apply(recordsForCustomizer);
-
-         ///////////////////////////////////////////////////////
-         // check if any records got errors in the customizer //
-         ///////////////////////////////////////////////////////
          Set<Serializable> primaryKeysToRemoveFromInput = new HashSet<>();
          for(QRecord record : customizerResult)
          {
             if(CollectionUtils.nullSafeHasContents(record.getErrors()))
             {
                recordsWithValidationErrors.add(record);
-               primaryKeysToRemoveFromInput.add(record.getValue(primaryKeyField));
+               primaryKeysToRemoveFromInput.add(record.getValue(primaryKeyFieldName));
             }
             else if(CollectionUtils.nullSafeHasContents(record.getWarnings()))
             {
-               recordsWithValidationWarnings.add(record);
+               recordsWithValidationWarnings.put(record.getValue(primaryKeyFieldName), record);
             }
          }
 
-         /////////////////////////////////////////////////////////////////
-         // do one mass removal of any bad keys from the input key list //
-         /////////////////////////////////////////////////////////////////
          if(!primaryKeysToRemoveFromInput.isEmpty())
          {
-            deleteInput.getPrimaryKeys().removeAll(primaryKeysToRemoveFromInput);
+            primaryKeys.removeAll(primaryKeysToRemoveFromInput);
          }
       }
 
@@ -171,24 +173,34 @@ public class DeleteAction
       ////////////////////////////////////
       DeleteOutput deleteOutput = deleteInterface.execute(deleteInput);
 
-      /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-      // merge the backend's output with any validation errors we found (whose ids wouldn't have gotten into the backend delete) //
+      // merge the backend's output with any validation errors we found (whose pkeys wouldn't have gotten into the backend delete) //
-      /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-      List<QRecord> outputRecordsWithErrors = deleteOutput.getRecordsWithErrors();
+      List<QRecord> outputRecordsWithErrors = Objects.requireNonNullElseGet(deleteOutput.getRecordsWithErrors(), () -> new ArrayList<>());
-      if(outputRecordsWithErrors == null)
-      {
-         deleteOutput.setRecordsWithErrors(new ArrayList<>());
-         outputRecordsWithErrors = deleteOutput.getRecordsWithErrors();
-      }
       outputRecordsWithErrors.addAll(recordsWithValidationErrors);
 
-      List<QRecord> outputRecordsWithWarnings = deleteOutput.getRecordsWithWarnings();
+      ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-      if(outputRecordsWithWarnings == null)
+      // if a record had a validation warning, but then an execution error, remove it from the warning list - so it's only in one of them. //
+      ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      for(QRecord outputRecordWithError : outputRecordsWithErrors)
       {
-         deleteOutput.setRecordsWithWarnings(new ArrayList<>());
+         Serializable pkey = outputRecordWithError.getValue(primaryKeyFieldName);
-         outputRecordsWithWarnings = deleteOutput.getRecordsWithWarnings();
+         recordsWithValidationWarnings.remove(pkey);
+      }
+
+      ///////////////////////////////////////////////////////////////////////////////////////////
+      // combine the warning list from validation to that from execution - avoiding duplicates //
+      // use a map to manage this list for the rest of this method                             //
+      ///////////////////////////////////////////////////////////////////////////////////////////
+      Map<Serializable, QRecord> outputRecordsWithWarningMap = CollectionUtils.nullSafeIsEmpty(deleteOutput.getRecordsWithWarnings()) ? new LinkedHashMap<>()
+         : deleteOutput.getRecordsWithWarnings().stream().collect(Collectors.toMap(r -> r.getValue(primaryKeyFieldName), r -> r, (a, b) -> a, () -> new LinkedHashMap<>()));
+      for(Map.Entry<Serializable, QRecord> entry : recordsWithValidationWarnings.entrySet())
+      {
+         if(!outputRecordsWithWarningMap.containsKey(entry.getKey()))
+         {
+            outputRecordsWithWarningMap.put(entry.getKey(), entry.getValue());
+         }
       }
-      outputRecordsWithWarnings.addAll(recordsWithValidationWarnings);
 
       ////////////////////////////////////////
       // delete associations, if applicable //
@@ -212,25 +224,27 @@ public class DeleteAction
          ////////////////////////////////////////////////////////////////////////////
          // make list of records that are still good - to pass into the customizer //
          ////////////////////////////////////////////////////////////////////////////
-         List<QRecord> recordsForCustomizer = makeListOfRecordsNotInErrorList(primaryKeyField, oldRecordList.get(), outputRecordsWithErrors);
+         List<QRecord> recordsForCustomizer = makeListOfRecordsNotInErrorList(primaryKeyFieldName, oldRecordList.get(), outputRecordsWithErrors);
 
          try
          {
             postDeleteCustomizer.get().setDeleteInput(deleteInput);
-            List<QRecord> customizerResult = postDeleteCustomizer.get().apply(recordsForCustomizer);
+            List<QRecord> postCustomizerResult = postDeleteCustomizer.get().apply(recordsForCustomizer);
 
             ///////////////////////////////////////////////////////
             // check if any records got errors in the customizer //
             ///////////////////////////////////////////////////////
-            for(QRecord record : customizerResult)
+            for(QRecord record : postCustomizerResult)
             {
+               Serializable pkey = record.getValue(primaryKeyFieldName);
                if(CollectionUtils.nullSafeHasContents(record.getErrors()))
                {
                   outputRecordsWithErrors.add(record);
+                  outputRecordsWithWarningMap.remove(pkey);
                }
                else if(CollectionUtils.nullSafeHasContents(record.getWarnings()))
                {
-                  outputRecordsWithWarnings.add(record);
+                  outputRecordsWithWarningMap.put(pkey, record);
                }
             }
          }
@@ -239,16 +253,65 @@ public class DeleteAction
             for(QRecord record : recordsForCustomizer)
             {
                record.addWarning(new QWarningMessage("An error occurred after the delete: " + e.getMessage()));
-               outputRecordsWithWarnings.add(record);
+               outputRecordsWithWarningMap.put(record.getValue(primaryKeyFieldName), record);
             }
          }
       }
 
+      deleteOutput.setRecordsWithErrors(outputRecordsWithErrors);
+      deleteOutput.setRecordsWithWarnings(new ArrayList<>(outputRecordsWithWarningMap.values()));
+
       return deleteOutput;
    }
 
 
 
+   /*******************************************************************************
+    ** this method takes in the deleteInput, and the list of old records that matched
+    ** the pkeys in that input.
+    **
+    ** it'll check if any of those pkeys aren't found (in a sub-method) - a record
+    ** with an error message will be added to oldRecordList for any such records.
+    **
+    ** it'll also then call the pre-customizer, if there is one - taking in the
+    ** oldRecordList.  it can add other errors or warnings to records.
+    **
+    ** The return value here is basically oldRecordList - possibly with some new
+    ** entries for the pkey-not-founds, and possibly w/ errors and warnings from the
+    ** customizer.
+    *******************************************************************************/
+   public List<QRecord> performValidations(DeleteInput deleteInput, Optional<List<QRecord>> oldRecordList, boolean isPreview) throws QException
+   {
+      if(oldRecordList.isEmpty())
+      {
+         return (null);
+      }
+
+      QTableMetaData table               = deleteInput.getTable();
+      List<QRecord>  primaryKeysNotFound = validateRecordsExistAndCanBeAccessed(deleteInput, oldRecordList.get());
+
+      ///////////////////////////////////////////////////////////////////////////
+      // after all validations, run the pre-delete customizer, if there is one //
+      ///////////////////////////////////////////////////////////////////////////
+      Optional<AbstractPreDeleteCustomizer> preDeleteCustomizer = QCodeLoader.getTableCustomizer(AbstractPreDeleteCustomizer.class, table, TableCustomizers.PRE_DELETE_RECORD.getRole());
+      List<QRecord>                         customizerResult    = oldRecordList.get();
+      if(preDeleteCustomizer.isPresent())
+      {
+         preDeleteCustomizer.get().setDeleteInput(deleteInput);
+         preDeleteCustomizer.get().setIsPreview(isPreview);
+         customizerResult = preDeleteCustomizer.get().apply(oldRecordList.get());
+      }
+
+      /////////////////////////////////////////////////////////////////////////
+      // add any pkey-not-found records to the front of the customizerResult //
+      /////////////////////////////////////////////////////////////////////////
+      customizerResult.addAll(primaryKeysNotFound);
+
+      return customizerResult;
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/
@@ -344,9 +407,9 @@ public class DeleteAction
     ** records that you can't see because of security - that they won't be found
     ** by the query here, so it's the same to you as if they don't exist at all!
     **
-    ** This method, if it finds any missing records, will:
+    ** If this method identifies any missing records (e.g., from PKeys that are
-    ** - remove those ids from the deleteInput
+    ** requested to be deleted, but don't exist (or can't be seen)), then it will
-    ** - create a QRecord with that id and a not-found error message.
+    ** return those as new QRecords, with error messages.
     *******************************************************************************/
    private List<QRecord> validateRecordsExistAndCanBeAccessed(DeleteInput deleteInput, List<QRecord> oldRecordList) throws QException
    {
@@ -355,64 +418,28 @@ public class DeleteAction
       QTableMetaData table           = deleteInput.getTable();
       QFieldMetaData primaryKeyField = table.getField(table.getPrimaryKeyField());
 
-      Set<Serializable> primaryKeysToRemoveFromInput = new HashSet<>();
-
       List<List<Serializable>> pages = CollectionUtils.getPages(deleteInput.getPrimaryKeys(), 1000);
       for(List<Serializable> page : pages)
       {
-         List<Serializable> primaryKeysToLookup = new ArrayList<>();
+         Map<Serializable, QRecord> oldRecordMapByPrimaryKey = new HashMap<>();
-         for(Serializable primaryKeyValue : page)
+         for(QRecord record : oldRecordList)
          {
-            if(primaryKeyValue != null)
+            Serializable primaryKeyValue = record.getValue(table.getPrimaryKeyField());
-            {
+            primaryKeyValue = ValueUtils.getValueAsFieldType(primaryKeyField.getType(), primaryKeyValue);
-               primaryKeysToLookup.add(primaryKeyValue);
+            oldRecordMapByPrimaryKey.put(primaryKeyValue, record);
-            }
-         }
-
-         Map<Serializable, QRecord> lookedUpRecords = new HashMap<>();
-         if(CollectionUtils.nullSafeHasContents(oldRecordList))
-         {
-            for(QRecord record : oldRecordList)
-            {
-               Serializable primaryKeyValue = record.getValue(table.getPrimaryKeyField());
-               primaryKeyValue = ValueUtils.getValueAsFieldType(primaryKeyField.getType(), primaryKeyValue);
-               lookedUpRecords.put(primaryKeyValue, record);
-            }
-         }
-         else if(!primaryKeysToLookup.isEmpty())
-         {
-            QueryInput queryInput = new QueryInput();
-            queryInput.setTransaction(deleteInput.getTransaction());
-            queryInput.setTableName(table.getName());
-            queryInput.setFilter(new QQueryFilter(new QFilterCriteria(table.getPrimaryKeyField(), QCriteriaOperator.IN, primaryKeysToLookup)));
-            QueryOutput queryOutput = new QueryAction().execute(queryInput);
-            for(QRecord record : queryOutput.getRecords())
-            {
-               lookedUpRecords.put(record.getValue(table.getPrimaryKeyField()), record);
-            }
          }
 
          for(Serializable primaryKeyValue : page)
          {
             primaryKeyValue = ValueUtils.getValueAsFieldType(primaryKeyField.getType(), primaryKeyValue);
-            if(!lookedUpRecords.containsKey(primaryKeyValue))
+            if(!oldRecordMapByPrimaryKey.containsKey(primaryKeyValue))
             {
                QRecord recordWithError = new QRecord();
                recordsWithErrors.add(recordWithError);
                recordWithError.setValue(primaryKeyField.getName(), primaryKeyValue);
                recordWithError.addError(new NotFoundStatusMessage("No record was found to delete for " + primaryKeyField.getLabel() + " = " + primaryKeyValue));
-               primaryKeysToRemoveFromInput.add(primaryKeyValue);
             }
          }
-
-         /////////////////////////////////////////////////////////////////
-         // do one mass removal of any bad keys from the input key list //
-         /////////////////////////////////////////////////////////////////
-         if(!primaryKeysToRemoveFromInput.isEmpty())
-         {
-            deleteInput.getPrimaryKeys().removeAll(primaryKeysToRemoveFromInput);
-            primaryKeysToRemoveFromInput.clear();
-         }
       }
 
       return (recordsWithErrors);

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/InsertAction.java

@@ -101,20 +101,7 @@ public class InsertAction extends AbstractQActionFunction<InsertInput, InsertOut
       /////////////////////////////
       // run standard validators //
       /////////////////////////////
-      ValueBehaviorApplier.applyFieldBehaviors(insertInput.getInstance(), table, insertInput.getRecords());
+      performValidations(insertInput, false);
-      setErrorsIfUniqueKeyErrors(insertInput, table);
-      validateRequiredFields(insertInput);
-      ValidateRecordSecurityLockHelper.validateSecurityFields(insertInput.getTable(), insertInput.getRecords(), ValidateRecordSecurityLockHelper.Action.INSERT);
-
-      ///////////////////////////////////////////////////////////////////////////
-      // after all validations, run the pre-insert customizer, if there is one //
-      ///////////////////////////////////////////////////////////////////////////
-      Optional<AbstractPreInsertCustomizer> preInsertCustomizer = QCodeLoader.getTableCustomizer(AbstractPreInsertCustomizer.class, table, TableCustomizers.PRE_INSERT_RECORD.getRole());
-      if(preInsertCustomizer.isPresent())
-      {
-         preInsertCustomizer.get().setInsertInput(insertInput);
-         insertInput.setRecords(preInsertCustomizer.get().apply(insertInput.getRecords()));
-      }
 
       ////////////////////////////////////
       // have the backend do the insert //
@@ -172,6 +159,32 @@ public class InsertAction extends AbstractQActionFunction<InsertInput, InsertOut
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public void performValidations(InsertInput insertInput, boolean isPreview) throws QException
+   {
+      QTableMetaData table = insertInput.getTable();
+
+      ValueBehaviorApplier.applyFieldBehaviors(insertInput.getInstance(), table, insertInput.getRecords());
+      setErrorsIfUniqueKeyErrors(insertInput, table);
+      validateRequiredFields(insertInput);
+      ValidateRecordSecurityLockHelper.validateSecurityFields(insertInput.getTable(), insertInput.getRecords(), ValidateRecordSecurityLockHelper.Action.INSERT);
+
+      ///////////////////////////////////////////////////////////////////////////
+      // after all validations, run the pre-insert customizer, if there is one //
+      ///////////////////////////////////////////////////////////////////////////
+      Optional<AbstractPreInsertCustomizer> preInsertCustomizer = QCodeLoader.getTableCustomizer(AbstractPreInsertCustomizer.class, table, TableCustomizers.PRE_INSERT_RECORD.getRole());
+      if(preInsertCustomizer.isPresent())
+      {
+         preInsertCustomizer.get().setInsertInput(insertInput);
+         preInsertCustomizer.get().setIsPreview(isPreview);
+         insertInput.setRecords(preInsertCustomizer.get().apply(insertInput.getRecords()));
+      }
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java

@@ -65,13 +65,13 @@ import com.kingsrook.qqq.backend.core.model.metadata.tables.QFieldSection;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QMiddlewareTableMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.Tier;
+import com.kingsrook.qqq.backend.core.processes.implementations.bulk.delete.BulkDeleteLoadStep;
 import com.kingsrook.qqq.backend.core.processes.implementations.bulk.delete.BulkDeleteTransformStep;
 import com.kingsrook.qqq.backend.core.processes.implementations.bulk.edit.BulkEditLoadStep;
 import com.kingsrook.qqq.backend.core.processes.implementations.bulk.edit.BulkEditTransformStep;
 import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkInsertExtractStep;
 import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkInsertTransformStep;
 import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.ExtractViaQueryStep;
-import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.LoadViaDeleteStep;
 import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.LoadViaInsertStep;
 import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess;
 import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
@@ -808,7 +808,7 @@ public class QInstanceEnricher
       QProcessMetaData process = StreamedETLWithFrontendProcess.defineProcessMetaData(
             ExtractViaQueryStep.class,
             BulkDeleteTransformStep.class,
-            LoadViaDeleteStep.class,
+            BulkDeleteLoadStep.class,
             values
          )
          .withName(processName)

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/delete/BulkDeleteLoadStep.java

@@ -0,0 +1,156 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.processes.implementations.bulk.delete;
+
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.stream.Collectors;
+import com.kingsrook.qqq.backend.core.actions.QBackendTransaction;
+import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLine;
+import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLineInterface;
+import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
+import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
+import com.kingsrook.qqq.backend.core.model.actions.processes.Status;
+import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.LoadViaDeleteStep;
+import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.ProcessSummaryProviderInterface;
+import com.kingsrook.qqq.backend.core.processes.implementations.general.ProcessSummaryWarningsAndErrorsRollup;
+import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
+
+
+/*******************************************************************************
+ ** Generic implementation of a LoadStep - that runs a Delete action for a
+ ** specified table.
+ *******************************************************************************/
+public class BulkDeleteLoadStep extends LoadViaDeleteStep implements ProcessSummaryProviderInterface
+{
+   private ProcessSummaryLine       okSummary     = new ProcessSummaryLine(Status.OK);
+   private List<ProcessSummaryLine> infoSummaries = new ArrayList<>();
+
+   private ProcessSummaryWarningsAndErrorsRollup processSummaryWarningsAndErrorsRollup = ProcessSummaryWarningsAndErrorsRollup.build("deleted");
+
+   private String tableLabel;
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public void preRun(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException
+   {
+      super.preRun(runBackendStepInput, runBackendStepOutput);
+
+      QTableMetaData table = runBackendStepInput.getInstance().getTable(runBackendStepInput.getTableName());
+      if(table != null)
+      {
+         tableLabel = table.getLabel();
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public ArrayList<ProcessSummaryLineInterface> getProcessSummary(RunBackendStepOutput runBackendStepOutput, boolean isForResultScreen)
+   {
+      ArrayList<ProcessSummaryLineInterface> rs = new ArrayList<>();
+
+      String noWarningsSuffix = processSummaryWarningsAndErrorsRollup.countWarnings() == 0 ? "" : " with no warnings";
+
+      okSummary.setSingularPastMessage(tableLabel + " record was deleted" + noWarningsSuffix + ".");
+      okSummary.setPluralPastMessage(tableLabel + " records were deleted" + noWarningsSuffix + ".");
+      okSummary.pickMessage(isForResultScreen);
+      okSummary.addSelfToListIfAnyCount(rs);
+
+      processSummaryWarningsAndErrorsRollup.addToList(rs);
+      rs.addAll(infoSummaries);
+      return (rs);
+   }
+
+
+
+   /*******************************************************************************
+    ** Execute the backend step - using the request as input, and the result as output.
+    **
+    *******************************************************************************/
+   @Override
+   public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException
+   {
+      ////////////////////////////
+      // have base class delete //
+      ////////////////////////////
+      super.run(runBackendStepInput, runBackendStepOutput);
+
+      QTableMetaData             table               = runBackendStepInput.getInstance().getTable(runBackendStepInput.getTableName());
+      String                     primaryKeyFieldName = table.getPrimaryKeyField();
+      Map<Serializable, QRecord> outputRecordMap     = runBackendStepOutput.getRecords().stream().collect(Collectors.toMap(r -> r.getValue(primaryKeyFieldName), r -> r, (a, b) -> a));
+
+      ///////////////////////////////////////////////////////////////////////////////////////////////////
+      // roll up the results, based on the input list, but looking for error/warnings from output list //
+      ///////////////////////////////////////////////////////////////////////////////////////////////////
+      for(QRecord record : runBackendStepInput.getRecords())
+      {
+         Serializable recordPrimaryKey = record.getValue(primaryKeyFieldName);
+         QRecord      outputRecord     = outputRecordMap.get(recordPrimaryKey);
+
+         if(CollectionUtils.nullSafeHasContents(outputRecord.getErrors()))
+         {
+            String message = outputRecord.getErrors().get(0).getMessage();
+            processSummaryWarningsAndErrorsRollup.addError(message, recordPrimaryKey);
+         }
+         else if(CollectionUtils.nullSafeHasContents(outputRecord.getWarnings()))
+         {
+            String message = outputRecord.getWarnings().get(0).getMessage();
+            processSummaryWarningsAndErrorsRollup.addWarning(message, recordPrimaryKey);
+         }
+         else
+         {
+            okSummary.incrementCountAndAddPrimaryKey(recordPrimaryKey);
+         }
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public Optional<QBackendTransaction> openTransaction(RunBackendStepInput runBackendStepInput) throws QException
+   {
+      InsertInput insertInput = new InsertInput();
+      insertInput.setTableName(runBackendStepInput.getValueString(FIELD_DESTINATION_TABLE));
+
+      return (Optional.of(new InsertAction().openTransaction(insertInput)));
+   }
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/delete/BulkDeleteTransformStep.java

@@ -22,16 +22,24 @@
 package com.kingsrook.qqq.backend.core.processes.implementations.bulk.delete;
 
 
+import java.io.Serializable;
 import java.util.ArrayList;
+import java.util.List;
+import java.util.Optional;
+import com.kingsrook.qqq.backend.core.actions.tables.DeleteAction;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLine;
 import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLineInterface;
 import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
 import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
 import com.kingsrook.qqq.backend.core.model.actions.processes.Status;
+import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.AbstractTransformStep;
 import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess;
+import com.kingsrook.qqq.backend.core.processes.implementations.general.ProcessSummaryWarningsAndErrorsRollup;
+import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
 
 
 /*******************************************************************************
@@ -41,6 +49,8 @@ public class BulkDeleteTransformStep extends AbstractTransformStep
 {
    private ProcessSummaryLine okSummary = new ProcessSummaryLine(Status.OK);
 
+   private ProcessSummaryWarningsAndErrorsRollup processSummaryWarningsAndErrorsRollup = ProcessSummaryWarningsAndErrorsRollup.build("deleted");
+
    private String tableLabel;
 
 
@@ -69,11 +79,15 @@ public class BulkDeleteTransformStep extends AbstractTransformStep
    @Override
    public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException
    {
+      QTableMetaData table           = runBackendStepInput.getTable();
+      String         primaryKeyField = table.getPrimaryKeyField();
+
       /////////////////////////////////////////////////////////////////////////////////////////////////////////////////
       // on the validate step, we haven't read the full file, so we don't know how many rows there are - thus        //
       // record count is null, and the ValidateStep won't be setting status counters - so - do it here in that case. //
       /////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-      if(runBackendStepInput.getStepName().equals(StreamedETLWithFrontendProcess.STEP_NAME_VALIDATE))
+      if(runBackendStepInput.getStepName().equals(StreamedETLWithFrontendProcess.STEP_NAME_VALIDATE) ||
+         runBackendStepInput.getStepName().equals(StreamedETLWithFrontendProcess.STEP_NAME_PREVIEW))
       {
          if(runBackendStepInput.getValue(StreamedETLWithFrontendProcess.FIELD_RECORD_COUNT) == null)
          {
@@ -83,6 +97,42 @@ public class BulkDeleteTransformStep extends AbstractTransformStep
          {
             runBackendStepInput.getAsyncJobCallback().updateStatus("Processing " + tableLabel + " record");
          }
+
+         ///////////////////////////////////////////////////////////////////////
+         // run the validation - critically - in preview mode (boolean param) //
+         ///////////////////////////////////////////////////////////////////////
+         DeleteAction deleteAction = new DeleteAction();
+         DeleteInput  deleteInput  = new DeleteInput();
+         deleteInput.setTableName(runBackendStepInput.getTableName());
+         deleteInput.setPrimaryKeys(runBackendStepInput.getRecords().stream().map(r -> r.getValue(primaryKeyField)).toList());
+         List<QRecord> validationResultRecords = deleteAction.performValidations(deleteInput, Optional.of(runBackendStepInput.getRecords()), true);
+
+         /////////////////////////////////////////////////////////////
+         // look at the update input to build process summary lines //
+         /////////////////////////////////////////////////////////////
+         List<QRecord> outputRecords = new ArrayList<>();
+         for(QRecord record : validationResultRecords)
+         {
+            Serializable recordPrimaryKey = record.getValue(table.getPrimaryKeyField());
+            if(CollectionUtils.nullSafeHasContents(record.getErrors()))
+            {
+               String message = record.getErrors().get(0).getMessage();
+               processSummaryWarningsAndErrorsRollup.addError(message, recordPrimaryKey);
+            }
+            else if(CollectionUtils.nullSafeHasContents(record.getWarnings()))
+            {
+               String message = record.getWarnings().get(0).getMessage();
+               processSummaryWarningsAndErrorsRollup.addWarning(message, recordPrimaryKey);
+               outputRecords.add(record);
+            }
+            else
+            {
+               okSummary.incrementCountAndAddPrimaryKey(recordPrimaryKey);
+               outputRecords.add(record);
+            }
+         }
+
+         runBackendStepOutput.setRecords(outputRecords);
       }
       else if(runBackendStepInput.getStepName().equals(StreamedETLWithFrontendProcess.STEP_NAME_EXECUTE))
       {
@@ -94,13 +144,15 @@ public class BulkDeleteTransformStep extends AbstractTransformStep
          {
             runBackendStepInput.getAsyncJobCallback().updateStatus("Deleting " + tableLabel + " records");
          }
-      }
 
-      //////////////////////////////////////////////////////////////////////////////////////////////////////////
+         //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-      // no transformation needs done - just pass records through from input to output, and assume all are OK //
+         // no transformation needs done - just pass records through from input to output, and assume errors & warnings will come from the delete action //
-      //////////////////////////////////////////////////////////////////////////////////////////////////////////
+         //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-      runBackendStepOutput.setRecords(runBackendStepInput.getRecords());
+         runBackendStepOutput.setRecords(runBackendStepInput.getRecords());
-      okSummary.incrementCount(runBackendStepInput.getRecords().size());
+
+         // i think load step will do this.
+         // okSummary.incrementCount(runBackendStepInput.getRecords().size());
+      }
    }
 
 
@@ -111,17 +163,16 @@ public class BulkDeleteTransformStep extends AbstractTransformStep
    @Override
    public ArrayList<ProcessSummaryLineInterface> getProcessSummary(RunBackendStepOutput runBackendStepOutput, boolean isForResultScreen)
    {
-      if(isForResultScreen)
-      {
-         okSummary.setMessage(tableLabel + " records were deleted.");
-      }
-      else
-      {
-         okSummary.setMessage(tableLabel + " records will be deleted.");
-      }
-
       ArrayList<ProcessSummaryLineInterface> rs = new ArrayList<>();
-      rs.add(okSummary);
+
+      String noWarningsSuffix = processSummaryWarningsAndErrorsRollup.countWarnings() == 0 ? "" : " with no warnings";
+      okSummary.setSingularFutureMessage(tableLabel + " record will be deleted" + noWarningsSuffix + ".");
+      okSummary.setPluralFutureMessage(tableLabel + " records will be deleted" + noWarningsSuffix + ".");
+      okSummary.pickMessage(isForResultScreen);
+      okSummary.addSelfToListIfAnyCount(rs);
+
+      processSummaryWarningsAndErrorsRollup.addToList(rs);
+
       return (rs);
    }
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/edit/BulkEditLoadStep.java

@@ -38,7 +38,6 @@ import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwith
 import com.kingsrook.qqq.backend.core.processes.implementations.general.ProcessSummaryWarningsAndErrorsRollup;
 import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
 import static com.kingsrook.qqq.backend.core.processes.implementations.bulk.edit.BulkEditTransformStep.buildInfoSummaryLines;
-import static com.kingsrook.qqq.backend.core.processes.implementations.bulk.edit.BulkEditTransformStep.getProcessSummaryWarningsAndErrorsRollup;
 
 
 /*******************************************************************************
@@ -51,7 +50,7 @@ public class BulkEditLoadStep extends LoadViaUpdateStep implements ProcessSummar
    private ProcessSummaryLine       okSummary     = new ProcessSummaryLine(Status.OK);
    private List<ProcessSummaryLine> infoSummaries = new ArrayList<>();
 
-   private ProcessSummaryWarningsAndErrorsRollup processSummaryWarningsAndErrorsRollup = getProcessSummaryWarningsAndErrorsRollup();
+   private ProcessSummaryWarningsAndErrorsRollup processSummaryWarningsAndErrorsRollup = ProcessSummaryWarningsAndErrorsRollup.build("edited");
 
    private String tableLabel;
 
@@ -104,9 +103,15 @@ public class BulkEditLoadStep extends LoadViaUpdateStep implements ProcessSummar
    @Override
    public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException
    {
-      QTableMetaData table = runBackendStepInput.getInstance().getTable(runBackendStepInput.getTableName());
+      ////////////////////////////
-
+      // have base class update //
+      ////////////////////////////
       super.run(runBackendStepInput, runBackendStepOutput);
+
+      ////////////////////////////////////////////////////////
+      // roll up results based on output from update action //
+      ////////////////////////////////////////////////////////
+      QTableMetaData table = runBackendStepInput.getInstance().getTable(runBackendStepInput.getTableName());
       for(QRecord record : runBackendStepOutput.getRecords())
       {
          Serializable recordPrimaryKey = record.getValue(table.getPrimaryKeyField());

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/edit/BulkEditTransformStep.java

@@ -59,7 +59,7 @@ public class BulkEditTransformStep extends AbstractTransformStep
    private ProcessSummaryLine       okSummary     = new ProcessSummaryLine(Status.OK);
    private List<ProcessSummaryLine> infoSummaries = new ArrayList<>();
 
-   private ProcessSummaryWarningsAndErrorsRollup processSummaryWarningsAndErrorsRollup = getProcessSummaryWarningsAndErrorsRollup();
+   private ProcessSummaryWarningsAndErrorsRollup processSummaryWarningsAndErrorsRollup = ProcessSummaryWarningsAndErrorsRollup.build("edited");
 
    private QTableMetaData table;
    private String         tableLabel;
@@ -71,36 +71,6 @@ public class BulkEditTransformStep extends AbstractTransformStep
 
 
 
-   /*******************************************************************************
-    ** used by Load step too
-    *******************************************************************************/
-   static ProcessSummaryWarningsAndErrorsRollup getProcessSummaryWarningsAndErrorsRollup()
-   {
-      return new ProcessSummaryWarningsAndErrorsRollup()
-         .withErrorTemplate(new ProcessSummaryLine(Status.ERROR)
-            .withSingularFutureMessage("record has an error: ")
-            .withPluralFutureMessage("records have an error: ")
-            .withSingularPastMessage("record had an error: ")
-            .withPluralPastMessage("records had an error: "))
-         .withWarningTemplate(new ProcessSummaryLine(Status.WARNING)
-            .withSingularFutureMessage("record will be edited, but has a warning: ")
-            .withPluralFutureMessage("records will be edited, but have a warning: ")
-            .withSingularPastMessage("record was edited, but had a warning: ")
-            .withPluralPastMessage("records were edited, but had a warning: "))
-         .withOtherErrorsSummary(new ProcessSummaryLine(Status.ERROR)
-            .withSingularFutureMessage("record has an other error.")
-            .withPluralFutureMessage("records have other errors.")
-            .withSingularPastMessage("record had an other error.")
-            .withPluralPastMessage("records had other errors."))
-         .withOtherWarningsSummary(new ProcessSummaryLine(Status.WARNING)
-            .withSingularFutureMessage("record will be edited, but has an other warning.")
-            .withPluralFutureMessage("records will be edited, but have other warnings.")
-            .withSingularPastMessage("record was edited, but had other warnings.")
-            .withPluralPastMessage("records were edited, but had other warnings."));
-   }
-
-
-
    /*******************************************************************************
     **
     *******************************************************************************/
@@ -169,7 +139,8 @@ public class BulkEditTransformStep extends AbstractTransformStep
             setUpdatedFieldsInRecord(runBackendStepInput, enabledFields, recordToUpdate);
          }
 
-         okSummary.incrementCount(runBackendStepInput.getRecords().size());
+         // i think load step will do this.
+         // okSummary.incrementCount(runBackendStepInput.getRecords().size());
       }
       else
       {

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTransformStep.java

@@ -30,6 +30,7 @@ import java.util.List;
 import java.util.Map;
 import java.util.Optional;
 import java.util.Set;
+import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
 import com.kingsrook.qqq.backend.core.actions.tables.helpers.UniqueKeyHelper;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLine;
@@ -37,12 +38,14 @@ import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLine
 import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
 import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
 import com.kingsrook.qqq.backend.core.model.actions.processes.Status;
+import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.UniqueKey;
 import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.AbstractTransformStep;
 import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.LoadViaInsertStep;
 import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess;
+import com.kingsrook.qqq.backend.core.processes.implementations.general.ProcessSummaryWarningsAndErrorsRollup;
 import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
 
 
@@ -51,7 +54,10 @@ import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
  *******************************************************************************/
 public class BulkInsertTransformStep extends AbstractTransformStep
 {
-   private ProcessSummaryLine                 okSummary        = new ProcessSummaryLine(Status.OK);
+   private ProcessSummaryLine okSummary = new ProcessSummaryLine(Status.OK);
+
+   private ProcessSummaryWarningsAndErrorsRollup processSummaryWarningsAndErrorsRollup = ProcessSummaryWarningsAndErrorsRollup.build("inserted");
+
    private Map<UniqueKey, ProcessSummaryLine> ukErrorSummaries = new HashMap<>();
 
    private QTableMetaData table;
@@ -112,20 +118,24 @@ public class BulkInsertTransformStep extends AbstractTransformStep
          }
       }
 
-      /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-      // no transformation needs to be done - just pass records through from input to output, if they don't violate any UK's //
+      // Note, we want to do our own UK checking here, even though InsertAction also tries to do it, because InsertAction //
-      /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      // will only be getting the records in pages, but in here, we'll track UK's across pages!!                          //
+      //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
 
-      ///////////////////////////////////////////////////
+      ////////////////////////////////////////////////////
-      // if there are no UK's, just output all records //
+      // if there are no UK's, proceed with all records //
-      ///////////////////////////////////////////////////
+      ////////////////////////////////////////////////////
+      List<QRecord> recordsWithoutUkErrors = new ArrayList<>();
       if(existingKeys.isEmpty())
       {
-         runBackendStepOutput.setRecords(runBackendStepInput.getRecords());
+         recordsWithoutUkErrors.addAll(runBackendStepInput.getRecords());
-         okSummary.incrementCount(runBackendStepInput.getRecords().size());
       }
       else
       {
+         /////////////////////////////////////////////////////////////
+         // else, only proceed with records that don't violate a UK //
+         /////////////////////////////////////////////////////////////
          for(UniqueKey uniqueKey : uniqueKeys)
          {
             keysInThisFile.computeIfAbsent(uniqueKey, x -> new HashSet<>());
@@ -162,11 +172,47 @@ public class BulkInsertTransformStep extends AbstractTransformStep
                   Optional<List<Serializable>> keyValues = UniqueKeyHelper.getKeyValues(table, uniqueKey, record);
                   keyValues.ifPresent(kv -> keysInThisFile.get(uniqueKey).add(kv));
                }
-               okSummary.incrementCount();
+               recordsWithoutUkErrors.add(record);
-               runBackendStepOutput.addRecord(record);
             }
          }
       }
+
+      /////////////////////////////////////////////////////////////////////////////////
+      // run all validation from the insert action - in Preview mode (boolean param) //
+      /////////////////////////////////////////////////////////////////////////////////
+      InsertAction insertAction = new InsertAction();
+      InsertInput  insertInput  = new InsertInput();
+      insertInput.setTableName(runBackendStepInput.getTableName());
+      insertInput.setRecords(recordsWithoutUkErrors);
+      insertInput.setSkipUniqueKeyCheck(true);
+      insertAction.performValidations(insertInput, true);
+      List<QRecord> validationResultRecords = insertInput.getRecords();
+
+      /////////////////////////////////////////////////////////////////
+      // look at validation results to build process summary results //
+      /////////////////////////////////////////////////////////////////
+      List<QRecord> outputRecords = new ArrayList<>();
+      for(QRecord record : validationResultRecords)
+      {
+         if(CollectionUtils.nullSafeHasContents(record.getErrors()))
+         {
+            String message = record.getErrors().get(0).getMessage();
+            processSummaryWarningsAndErrorsRollup.addError(message, null);
+         }
+         else if(CollectionUtils.nullSafeHasContents(record.getWarnings()))
+         {
+            String message = record.getWarnings().get(0).getMessage();
+            processSummaryWarningsAndErrorsRollup.addWarning(message, null);
+            outputRecords.add(record);
+         }
+         else
+         {
+            okSummary.incrementCountAndAddPrimaryKey(null);
+            outputRecords.add(record);
+         }
+      }
+
+      runBackendStepOutput.setRecords(outputRecords);
    }
 
 
@@ -177,22 +223,22 @@ public class BulkInsertTransformStep extends AbstractTransformStep
    @Override
    public ArrayList<ProcessSummaryLineInterface> getProcessSummary(RunBackendStepOutput runBackendStepOutput, boolean isForResultScreen)
    {
-      String tableLabel = table == null ? "" : table.getLabel();
+      ArrayList<ProcessSummaryLineInterface> rs         = new ArrayList<>();
+      String                                 tableLabel = table == null ? "" : table.getLabel();
 
-      okSummary
+      String noWarningsSuffix = processSummaryWarningsAndErrorsRollup.countWarnings() == 0 ? "" : " with no warnings";
-         .withSingularFutureMessage(tableLabel + " record will be inserted")
+      okSummary.setSingularFutureMessage(tableLabel + " record will be inserted" + noWarningsSuffix + ".");
-         .withPluralFutureMessage(tableLabel + " records will be inserted")
+      okSummary.setPluralFutureMessage(tableLabel + " records will be inserted" + noWarningsSuffix + ".");
-         .withSingularPastMessage(tableLabel + " record was inserted")
+      okSummary.setSingularPastMessage(tableLabel + " record was inserted" + noWarningsSuffix + ".");
-         .withPluralPastMessage(tableLabel + " records were inserted");
+      okSummary.setPluralPastMessage(tableLabel + " records were inserted" + noWarningsSuffix + ".");
-
+      okSummary.pickMessage(isForResultScreen);
-      ArrayList<ProcessSummaryLineInterface> rs = new ArrayList<>();
       okSummary.addSelfToListIfAnyCount(rs);
 
       for(Map.Entry<UniqueKey, ProcessSummaryLine> entry : ukErrorSummaries.entrySet())
       {
          UniqueKey          uniqueKey      = entry.getKey();
          ProcessSummaryLine ukErrorSummary = entry.getValue();
-         String             ukErrorSuffix  = " inserted, because they contain a duplicate key (" + uniqueKey.getDescription(table) + ")";
+         String             ukErrorSuffix  = " inserted, because of duplicate values in a unique key (" + uniqueKey.getDescription(table) + ")";
 
          ukErrorSummary
             .withSingularFutureMessage(tableLabel + " record will not be" + ukErrorSuffix)
@@ -203,6 +249,8 @@ public class BulkInsertTransformStep extends AbstractTransformStep
          ukErrorSummary.addSelfToListIfAnyCount(rs);
       }
 
+      processSummaryWarningsAndErrorsRollup.addToList(rs);
+
       return (rs);
    }
 

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/LoadViaDeleteStep.java

@@ -31,6 +31,7 @@ import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
 import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteOutput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 
@@ -60,7 +61,9 @@ public class LoadViaDeleteStep extends AbstractLoadStep
       deleteInput.setAsyncJobCallback(runBackendStepInput.getAsyncJobCallback());
       // todo?  can make more efficient deletes, maybe? deleteInput.setQueryFilter();
       getTransaction().ifPresent(deleteInput::setTransaction);
-      new DeleteAction().execute(deleteInput);
+      DeleteOutput deleteOutput = new DeleteAction().execute(deleteInput);
+      runBackendStepOutput.getRecords().addAll(deleteOutput.getRecordsWithErrors());
+      runBackendStepOutput.getRecords().addAll(deleteOutput.getRecordsWithWarnings());
    }
 
 

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/general/ProcessSummaryWarningsAndErrorsRollup.java

@@ -48,6 +48,36 @@ public class ProcessSummaryWarningsAndErrorsRollup
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public static ProcessSummaryWarningsAndErrorsRollup build(String pastTenseVerb)
+   {
+      return new ProcessSummaryWarningsAndErrorsRollup()
+         .withErrorTemplate(new ProcessSummaryLine(Status.ERROR)
+            .withSingularFutureMessage("record has an error: ")
+            .withPluralFutureMessage("records have an error: ")
+            .withSingularPastMessage("record had an error: ")
+            .withPluralPastMessage("records had an error: "))
+         .withWarningTemplate(new ProcessSummaryLine(Status.WARNING)
+            .withSingularFutureMessage("record will be " + pastTenseVerb + ", but has a warning: ")
+            .withPluralFutureMessage("records will be " + pastTenseVerb + ", but have a warning: ")
+            .withSingularPastMessage("record was " + pastTenseVerb + ", but had a warning: ")
+            .withPluralPastMessage("records were " + pastTenseVerb + ", but had a warning: "))
+         .withOtherErrorsSummary(new ProcessSummaryLine(Status.ERROR)
+            .withSingularFutureMessage("record has an other error.")
+            .withPluralFutureMessage("records have other errors.")
+            .withSingularPastMessage("record had an other error.")
+            .withPluralPastMessage("records had other errors."))
+         .withOtherWarningsSummary(new ProcessSummaryLine(Status.WARNING)
+            .withSingularFutureMessage("record will be " + pastTenseVerb + ", but has an other warning.")
+            .withPluralFutureMessage("records will be " + pastTenseVerb + ", but have other warnings.")
+            .withSingularPastMessage("record was " + pastTenseVerb + ", but had other warnings.")
+            .withPluralPastMessage("records were " + pastTenseVerb + ", but had other warnings."));
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/
@@ -164,7 +194,15 @@ public class ProcessSummaryWarningsAndErrorsRollup
             }
          }
       }
-      processSummaryLine.incrementCountAndAddPrimaryKey(primaryKey);
+
+      if(primaryKey == null)
+      {
+         processSummaryLine.incrementCount();
+      }
+      else
+      {
+         processSummaryLine.incrementCountAndAddPrimaryKey(primaryKey);
+      }
    }