CE-1955 - Summarize with some examples (including rows nos) for value mapping and other validation errors

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/QFieldType.java

@@ -27,6 +27,7 @@ import java.time.Instant;
 import java.time.LocalDate;
 import java.time.LocalTime;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.utils.StringUtils;
 
 
 /*******************************************************************************
@@ -100,6 +101,16 @@ public enum QFieldType
 
 
 
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   public String getMixedCaseLabel()
+   {
+      return StringUtils.allCapsToMixedCase(name());
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTransformStep.java

@@ -29,6 +29,7 @@ import java.util.HashSet;
 import java.util.LinkedHashSet;
 import java.util.List;
 import java.util.Map;
+import java.util.Objects;
 import java.util.Optional;
 import java.util.Set;
 import com.kingsrook.qqq.backend.core.actions.customizers.AbstractPreInsertCustomizer;
@@ -47,16 +48,25 @@ import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutp
 import com.kingsrook.qqq.backend.core.model.actions.processes.Status;
 import com.kingsrook.qqq.backend.core.model.actions.tables.QInputSource;
 import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
+import com.kingsrook.qqq.backend.core.model.dashboard.widgets.WidgetType;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.metadata.dashboard.QWidgetMetaDataInterface;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.Association;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QFieldSection;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.UniqueKey;
+import com.kingsrook.qqq.backend.core.model.statusmessages.QErrorMessage;
+import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping.BulkLoadRecordUtils;
+import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping.BulkLoadValueTypeError;
+import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkInsertMapping;
 import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.AbstractTransformStep;
 import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.LoadViaInsertStep;
 import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess;
 import com.kingsrook.qqq.backend.core.processes.implementations.general.ProcessSummaryWarningsAndErrorsRollup;
 import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
+import com.kingsrook.qqq.backend.core.utils.ListingHash;
 import com.kingsrook.qqq.backend.core.utils.StringUtils;
+import com.kingsrook.qqq.backend.core.utils.ValueUtils;
 
 
 /*******************************************************************************
@@ -66,7 +76,11 @@ public class BulkInsertTransformStep extends AbstractTransformStep
 {
    private ProcessSummaryLine okSummary = new ProcessSummaryLine(Status.OK);
 
-   private ProcessSummaryWarningsAndErrorsRollup processSummaryWarningsAndErrorsRollup = ProcessSummaryWarningsAndErrorsRollup.build("inserted");
+   private ProcessSummaryWarningsAndErrorsRollup processSummaryWarningsAndErrorsRollup = ProcessSummaryWarningsAndErrorsRollup.build("inserted")
+      .withDoReplaceSingletonCountLinesWithSuffixOnly(false);
+
+   private ListingHash<String, RowValue> errorToExampleRowValueMap = new ListingHash<>();
+   private ListingHash<String, String>   errorToExampleRowsMap     = new ListingHash<>();
 
    private Map<UniqueKey, ProcessSummaryLineWithUKSampleValues> ukErrorSummaries              = new HashMap<>();
    private Map<String, ProcessSummaryLine>                      associationsToInsertSummaries = new HashMap<>();
@@ -77,6 +91,7 @@ public class BulkInsertTransformStep extends AbstractTransformStep
 
    private int rowsProcessed = 0;
 
+   private final int EXAMPLE_ROW_LIMIT = 10;
 
 
    /*******************************************************************************
@@ -118,6 +133,44 @@ public class BulkInsertTransformStep extends AbstractTransformStep
       // make sure that if a saved profile was selected on a review screen, that the result screen knows about it. //
       ///////////////////////////////////////////////////////////////////////////////////////////////////////////////
       BulkInsertStepUtils.handleSavedBulkLoadProfileIdValue(runBackendStepInput, runBackendStepOutput);
+
+      /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      // set up the validationReview widget to render preview records using the table layout, and including the associations //
+      /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      runBackendStepOutput.addValue("formatPreviewRecordUsingTableLayout", table.getName());
+
+      BulkInsertMapping bulkInsertMapping = (BulkInsertMapping) runBackendStepOutput.getValue("bulkInsertMapping");
+      if(bulkInsertMapping != null)
+      {
+         ArrayList<String> previewRecordAssociatedTableNames  = new ArrayList<>();
+         ArrayList<String> previewRecordAssociatedWidgetNames = new ArrayList<>();
+         ArrayList<String> previewRecordAssociationNames      = new ArrayList<>();
+
+         for(String mappedAssociation : bulkInsertMapping.getMappedAssociations())
+         {
+            Optional<Association> association = table.getAssociations().stream().filter(a -> a.getName().equals(mappedAssociation)).findFirst();
+            if(association.isPresent())
+            {
+               for(QFieldSection section : table.getSections())
+               {
+                  QWidgetMetaDataInterface widget = QContext.getQInstance().getWidget(section.getWidgetName());
+                  if(widget != null && WidgetType.CHILD_RECORD_LIST.getType().equals(widget.getType()))
+                  {
+                     Serializable widgetJoinName = widget.getDefaultValues().get("joinName");
+                     if(Objects.equals(widgetJoinName, association.get().getJoinName()))
+                     {
+                        previewRecordAssociatedTableNames.add(association.get().getAssociatedTableName());
+                        previewRecordAssociatedWidgetNames.add(widget.getName());
+                        previewRecordAssociationNames.add(association.get().getName());
+                     }
+                  }
+               }
+            }
+         }
+         runBackendStepOutput.addValue("previewRecordAssociatedTableNames", previewRecordAssociatedTableNames);
+         runBackendStepOutput.addValue("previewRecordAssociatedWidgetNames", previewRecordAssociatedWidgetNames);
+         runBackendStepOutput.addValue("previewRecordAssociationNames", previewRecordAssociationNames);
+      }
    }
 
 
@@ -131,7 +184,9 @@ public class BulkInsertTransformStep extends AbstractTransformStep
       int            recordsInThisPage = runBackendStepInput.getRecords().size();
       QTableMetaData table             = QContext.getQInstance().getTable(runBackendStepInput.getTableName());
 
-      // split the records w/o UK errors into those w/ e
+      //////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      // split the records into 2 lists:  those w/ errors (e.g., from the bulk-load mapping), and those that are okay //
+      //////////////////////////////////////////////////////////////////////////////////////////////////////////////////
       List<QRecord> recordsWithoutAnyErrors = new ArrayList<>();
       List<QRecord> recordsWithSomeErrors   = new ArrayList<>();
       for(QRecord record : runBackendStepInput.getRecords())
@@ -153,16 +208,26 @@ public class BulkInsertTransformStep extends AbstractTransformStep
       {
          for(QRecord record : recordsWithSomeErrors)
          {
-            String message = record.getErrors().get(0).getMessage();
+            for(QErrorMessage error : record.getErrors())
-            processSummaryWarningsAndErrorsRollup.addError(message, null);
+            {
+               if(error instanceof BulkLoadValueTypeError blvte)
+               {
+                  processSummaryWarningsAndErrorsRollup.addError(blvte.getMessageToUseAsProcessSummaryRollupKey(), null);
+                  addToErrorToExampleRowValueMap(blvte, record);
+               }
+               else
+               {
+                  processSummaryWarningsAndErrorsRollup.addError(error.getMessage(), null);
+               }
+            }
          }
       }
 
       if(recordsWithoutAnyErrors.isEmpty())
       {
-         ////////////////////////////////////////////////////////////////////////////////
+         /////////////////////////////////////////////////////////////////////////////////
-         // skip th rest of this method if there aren't any records w/o errors in them //
+         // skip the rest of this method if there aren't any records w/o errors in them //
-         ////////////////////////////////////////////////////////////////////////////////
+         /////////////////////////////////////////////////////////////////////////////////
          this.rowsProcessed += recordsInThisPage;
       }
 
@@ -248,8 +313,11 @@ public class BulkInsertTransformStep extends AbstractTransformStep
       {
          if(CollectionUtils.nullSafeHasContents(record.getErrors()))
          {
-            String message = record.getErrors().get(0).getMessage();
+            for(QErrorMessage error : record.getErrors())
-            processSummaryWarningsAndErrorsRollup.addError(message, null);
+            {
+               processSummaryWarningsAndErrorsRollup.addError(error.getMessage(), null);
+               addToErrorToExampleRowMap(error.getMessage(), record);
+            }
          }
          else if(CollectionUtils.nullSafeHasContents(record.getWarnings()))
          {
@@ -277,6 +345,37 @@ public class BulkInsertTransformStep extends AbstractTransformStep
 
 
 
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   private void addToErrorToExampleRowValueMap(BulkLoadValueTypeError bulkLoadValueTypeError, QRecord record)
+   {
+      String         message   = bulkLoadValueTypeError.getMessageToUseAsProcessSummaryRollupKey();
+      List<RowValue> rowValues = errorToExampleRowValueMap.computeIfAbsent(message, k -> new ArrayList<>());
+
+      if(rowValues.size() < EXAMPLE_ROW_LIMIT)
+      {
+         rowValues.add(new RowValue(bulkLoadValueTypeError, record));
+      }
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   private void addToErrorToExampleRowMap(String message, QRecord record)
+   {
+      List<String> rowNos = errorToExampleRowsMap.computeIfAbsent(message, k -> new ArrayList<>());
+
+      if(rowNos.size() < EXAMPLE_ROW_LIMIT)
+      {
+         rowNos.add(BulkLoadRecordUtils.getRowNosString(record));
+      }
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/
@@ -411,9 +510,61 @@ public class BulkInsertTransformStep extends AbstractTransformStep
          ukErrorSummary.addSelfToListIfAnyCount(rs);
       }
 
+      ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      // for process summary lines that exist in the error-to-example-row-value map, add those example values to the lines. //
+      ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      for(Map.Entry<String, ProcessSummaryLine> entry : processSummaryWarningsAndErrorsRollup.getErrorSummaries().entrySet())
+      {
+         String message = entry.getKey();
+         if(errorToExampleRowValueMap.containsKey(message))
+         {
+            ProcessSummaryLine line = entry.getValue();
+            List<RowValue> rowValues = errorToExampleRowValueMap.get(message);
+            String exampleOrFull = rowValues.size() < line.getCount() ? "Example " : "";
+            line.setMessageSuffix(line.getMessageSuffix() + ".  " + exampleOrFull + "Values:");
+            line.setBulletsOfText(new ArrayList<>(rowValues.stream().map(String::valueOf).toList()));
+         }
+         else if(errorToExampleRowsMap.containsKey(message))
+         {
+            ProcessSummaryLine line = entry.getValue();
+            List<String> rowDescriptions = errorToExampleRowsMap.get(message);
+            String exampleOrFull = rowDescriptions.size() < line.getCount() ? "Example " : "";
+            line.setMessageSuffix(line.getMessageSuffix() + ".  " + exampleOrFull + "Records:");
+            line.setBulletsOfText(new ArrayList<>(rowDescriptions.stream().map(String::valueOf).toList()));
+         }
+      }
+
       processSummaryWarningsAndErrorsRollup.addToList(rs);
 
       return (rs);
    }
 
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   private record RowValue(String row, String value)
+   {
+
+      /***************************************************************************
+       **
+       ***************************************************************************/
+      public RowValue(BulkLoadValueTypeError bulkLoadValueTypeError, QRecord record)
+      {
+         this(BulkLoadRecordUtils.getRowNosString(record), ValueUtils.getValueAsString(bulkLoadValueTypeError.getValue()));
+      }
+
+
+
+      /***************************************************************************
+       **
+       ***************************************************************************/
+      @Override
+      public String toString()
+      {
+         return row + " [" + value + "]";
+      }
+   }
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadValueTypeError.java

@@ -0,0 +1,76 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping;
+
+
+import java.io.Serializable;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
+import com.kingsrook.qqq.backend.core.model.statusmessages.BadInputStatusMessage;
+
+
+/*******************************************************************************
+ ** Specialized error for records, for bulk-load use-cases, where we want to
+ ** report back info to the user about the field & value.
+ *******************************************************************************/
+public class BulkLoadValueTypeError extends BadInputStatusMessage
+{
+   private final String       fieldLabel;
+   private final String       fieldName;
+   private final Serializable value;
+   private final QFieldType   type;
+
+
+
+   /*******************************************************************************
+    ** Constructor
+    **
+    *******************************************************************************/
+   public BulkLoadValueTypeError(String fieldName, Serializable value, QFieldType type, String fieldLabel)
+   {
+      super("Value [" + value + "] for field [" + fieldLabel + "] could not be converted to type [" + type + "]");
+      this.fieldName = fieldName;
+      this.value = value;
+      this.type = type;
+      this.fieldLabel = fieldLabel;
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   public String getMessageToUseAsProcessSummaryRollupKey()
+   {
+      return ("Cannot convert value for field [" + fieldLabel + "] to type [" + type.getMixedCaseLabel() + "]");
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for value
+    **
+    *******************************************************************************/
+   public Serializable getValue()
+   {
+      return value;
+   }
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/ValueMapper.java

@@ -34,7 +34,6 @@ import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.Association;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
-import com.kingsrook.qqq.backend.core.model.statusmessages.BadInputStatusMessage;
 import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkInsertMapping;
 import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
 import com.kingsrook.qqq.backend.core.utils.StringUtils;
@@ -70,6 +69,9 @@ public class ValueMapper
          return;
       }
 
+      String associationNamePrefixForFields = StringUtils.hasContent(associationNameChain) ? associationNameChain + "." : "";
+      String tableLabelPrefix = StringUtils.hasContent(associationNameChain) ? table.getLabel() + ": " : "";
+
       Map<String, Map<String, Serializable>> mappingForTable = mapping.getFieldNameToValueMappingForTable(associationNameChain);
       for(QRecord record : records)
       {
@@ -102,7 +104,7 @@ public class ValueMapper
                }
                catch(Exception e)
                {
-                  record.addError(new BadInputStatusMessage("Value [" + value + "] for field [" + field.getLabel() + "] could not be converted to type [" + type + "]"));
+                  record.addError(new BulkLoadValueTypeError(associationNamePrefixForFields + field.getName(), value, type, tableLabelPrefix + field.getLabel()));
                }
             }
 

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/general/ProcessSummaryWarningsAndErrorsRollup.java

@@ -477,4 +477,13 @@ public class ProcessSummaryWarningsAndErrorsRollup
    }
 
 
+
+   /*******************************************************************************
+    ** Getter for errorSummaries
+    **
+    *******************************************************************************/
+   public Map<String, ProcessSummaryLine> getErrorSummaries()
+   {
+      return errorSummaries;
+   }
 }

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/actions/processes/ProcessSummaryAssert.java

@@ -0,0 +1,208 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.model.actions.processes;
+
+
+import java.util.ArrayList;
+import java.util.List;
+import com.kingsrook.qqq.backend.core.utils.StringUtils;
+import org.assertj.core.api.AbstractAssert;
+import org.assertj.core.api.Assertions;
+import static org.junit.jupiter.api.Assertions.fail;
+
+
+/*******************************************************************************
+ ** AssertJ assert class for ProcessSummary - that is - a list of ProcessSummaryLineInterface's
+ *******************************************************************************/
+public class ProcessSummaryAssert extends AbstractAssert<ProcessSummaryAssert, List<ProcessSummaryLineInterface>>
+{
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   protected ProcessSummaryAssert(List<ProcessSummaryLineInterface> actual, Class<?> selfType)
+   {
+      super(actual, selfType);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @SuppressWarnings("unchecked")
+   public static ProcessSummaryAssert assertThat(RunProcessOutput runProcessOutput)
+   {
+      List<ProcessSummaryLineInterface> processResults = (List<ProcessSummaryLineInterface>) runProcessOutput.getValue("processResults");
+      if(processResults == null)
+      {
+         processResults = (List<ProcessSummaryLineInterface>) runProcessOutput.getValue("validationSummary");
+      }
+
+      return (new ProcessSummaryAssert(processResults, ProcessSummaryAssert.class));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @SuppressWarnings("unchecked")
+   public static ProcessSummaryAssert assertThat(RunBackendStepOutput runBackendStepOutput)
+   {
+      List<ProcessSummaryLineInterface> processResults = (List<ProcessSummaryLineInterface>) runBackendStepOutput.getValue("processResults");
+      if(processResults == null)
+      {
+         processResults = (List<ProcessSummaryLineInterface>) runBackendStepOutput.getValue("validationSummary");
+      }
+
+      if(processResults == null)
+      {
+         fail("Could not find process results in backend step output.");
+      }
+
+      return (new ProcessSummaryAssert(processResults, ProcessSummaryAssert.class));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public static ProcessSummaryAssert assertThat(List<ProcessSummaryLineInterface> actual)
+   {
+      return (new ProcessSummaryAssert(actual, ProcessSummaryAssert.class));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public ProcessSummaryAssert hasSize(int expectedSize)
+   {
+      Assertions.assertThat(actual).hasSize(expectedSize);
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public ProcessSummaryLineInterfaceAssert hasLineWithMessageMatching(String regExp)
+   {
+      List<String> foundMessages = new ArrayList<>();
+      for(ProcessSummaryLineInterface processSummaryLineInterface : actual)
+      {
+         if(processSummaryLineInterface.getMessage() == null)
+         {
+            processSummaryLineInterface.prepareForFrontend(false);
+         }
+
+         if(processSummaryLineInterface.getMessage() != null && processSummaryLineInterface.getMessage().matches(regExp))
+         {
+            return (new ProcessSummaryLineInterfaceAssert(processSummaryLineInterface, ProcessSummaryLineInterfaceAssert.class));
+         }
+         else
+         {
+            foundMessages.add(processSummaryLineInterface.getMessage());
+         }
+      }
+
+      failWithMessage("Failed to find a ProcessSummaryLine with message matching [" + regExp + "].\nFound messages were:\n" + StringUtils.join("\n", foundMessages));
+      return (null);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public ProcessSummaryLineInterfaceAssert hasLineWithMessageContaining(String substr)
+   {
+      List<String> foundMessages = new ArrayList<>();
+      for(ProcessSummaryLineInterface processSummaryLineInterface : actual)
+      {
+         if(processSummaryLineInterface.getMessage() == null)
+         {
+            processSummaryLineInterface.prepareForFrontend(false);
+         }
+
+         if(processSummaryLineInterface.getMessage() != null && processSummaryLineInterface.getMessage().contains(substr))
+         {
+            return (new ProcessSummaryLineInterfaceAssert(processSummaryLineInterface, ProcessSummaryLineInterfaceAssert.class));
+         }
+         else
+         {
+            foundMessages.add(processSummaryLineInterface.getMessage());
+         }
+      }
+
+      failWithMessage("Failed to find a ProcessSummaryLine with message containing [" + substr + "].\nFound messages were:\n" + StringUtils.join("\n", foundMessages));
+      return (null);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public ProcessSummaryLineInterfaceAssert hasLineWithStatus(Status status)
+   {
+      List<String> foundStatuses = new ArrayList<>();
+      for(ProcessSummaryLineInterface processSummaryLineInterface : actual)
+      {
+         if(status.equals(processSummaryLineInterface.getStatus()))
+         {
+            return (new ProcessSummaryLineInterfaceAssert(processSummaryLineInterface, ProcessSummaryLineInterfaceAssert.class));
+         }
+         else
+         {
+            foundStatuses.add(String.valueOf(processSummaryLineInterface.getStatus()));
+         }
+      }
+
+      failWithMessage("Failed to find a ProcessSummaryLine with status [" + status + "].\nFound statuses were:\n" + StringUtils.join("\n", foundStatuses));
+      return (null);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public ProcessSummaryAssert hasNoLineWithStatus(Status status)
+   {
+      for(ProcessSummaryLineInterface processSummaryLineInterface : actual)
+      {
+         if(status.equals(processSummaryLineInterface.getStatus()))
+         {
+            failWithMessage("Found a ProcessSummaryLine with status [" + status + "], which was not supposed to happen.");
+            return (null);
+         }
+      }
+
+      return (this);
+   }
+
+}

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/actions/processes/ProcessSummaryLineInterfaceAssert.java

@@ -0,0 +1,189 @@
+
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.model.actions.processes;
+
+
+import org.assertj.core.api.AbstractAssert;
+import org.assertj.core.api.Assertions;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+
+/*******************************************************************************
+ ** AssertJ assert class for ProcessSummaryLine.
+ *******************************************************************************/
+public class ProcessSummaryLineInterfaceAssert extends AbstractAssert<ProcessSummaryLineInterfaceAssert, ProcessSummaryLineInterface>
+{
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   protected ProcessSummaryLineInterfaceAssert(ProcessSummaryLineInterface actual, Class<?> selfType)
+   {
+      super(actual, selfType);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public static ProcessSummaryLineInterfaceAssert assertThat(ProcessSummaryLineInterface actual)
+   {
+      return (new ProcessSummaryLineInterfaceAssert(actual, ProcessSummaryLineInterfaceAssert.class));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public ProcessSummaryLineInterfaceAssert hasCount(Integer count)
+   {
+      if(actual instanceof ProcessSummaryLine psl)
+      {
+         assertEquals(count, psl.getCount(), "Expected count in process summary line");
+      }
+      else
+      {
+         failWithMessage("ProcessSummaryLineInterface is not of concrete type ProcessSummaryLine (is: " + actual.getClass().getSimpleName() + ")");
+      }
+
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public ProcessSummaryLineInterfaceAssert hasStatus(Status status)
+   {
+      assertEquals(status, actual.getStatus(), "Expected status in process summary line");
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public ProcessSummaryLineInterfaceAssert hasMessageMatching(String regExp)
+   {
+      if(actual.getMessage() == null)
+      {
+         actual.prepareForFrontend(false);
+      }
+
+      Assertions.assertThat(actual.getMessage()).matches(regExp);
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public ProcessSummaryLineInterfaceAssert hasMessageContaining(String substring)
+   {
+      if(actual.getMessage() == null)
+      {
+         actual.prepareForFrontend(false);
+      }
+
+      Assertions.assertThat(actual.getMessage()).contains(substring);
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public ProcessSummaryLineInterfaceAssert doesNotHaveMessageMatching(String regExp)
+   {
+      if(actual.getMessage() == null)
+      {
+         actual.prepareForFrontend(false);
+      }
+
+      Assertions.assertThat(actual.getMessage()).doesNotMatch(regExp);
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public ProcessSummaryLineInterfaceAssert doesNotHaveMessageContaining(String substring)
+   {
+      if(actual.getMessage() == null)
+      {
+         actual.prepareForFrontend(false);
+      }
+
+      Assertions.assertThat(actual.getMessage()).doesNotContain(substring);
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public ProcessSummaryLineInterfaceAssert hasAnyBulletsOfTextContaining(String substring)
+   {
+      if(actual instanceof ProcessSummaryLine psl)
+      {
+         Assertions.assertThat(psl.getBulletsOfText())
+            .isNotNull()
+            .anyMatch(s -> s.contains(substring));
+      }
+      else
+      {
+         Assertions.fail("Process Summary Line was not the expected type.");
+      }
+
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public ProcessSummaryLineInterfaceAssert doesNotHaveAnyBulletsOfTextContaining(String substring)
+   {
+      if(actual instanceof ProcessSummaryLine psl)
+      {
+         if(psl.getBulletsOfText() != null)
+         {
+            Assertions.assertThat(psl.getBulletsOfText())
+               .noneMatch(s -> s.contains(substring));
+         }
+      }
+
+      return (this);
+   }
+
+}

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTransformStepTest.java

@@ -22,10 +22,14 @@
 package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert;
 
 
+import java.io.Serializable;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.stream.Collectors;
 import com.kingsrook.qqq.backend.core.BaseTest;
+import com.kingsrook.qqq.backend.core.context.QContext;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryAssert;
 import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLine;
 import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLineInterface;
 import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
@@ -38,8 +42,12 @@ import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.UniqueKey;
 import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryRecordStore;
+import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping.BulkLoadRecordUtils;
+import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping.BulkLoadValueTypeError;
+import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow;
 import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess;
 import com.kingsrook.qqq.backend.core.utils.TestUtils;
+import com.kingsrook.qqq.backend.core.utils.collections.ListBuilder;
 import org.junit.jupiter.api.AfterEach;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
@@ -87,9 +95,9 @@ class BulkInsertTransformStepTest extends BaseTest
       // insert some records that will cause some UK violations //
       ////////////////////////////////////////////////////////////
       TestUtils.insertRecords(table, List.of(
-         newQRecord("uuid-A", "SKU-1", 1),
+         newUkTestQRecord("uuid-A", "SKU-1", 1),
-         newQRecord("uuid-B", "SKU-2", 1),
+         newUkTestQRecord("uuid-B", "SKU-2", 1),
-         newQRecord("uuid-C", "SKU-2", 2)
+         newUkTestQRecord("uuid-C", "SKU-2", 2)
       ));
 
       ///////////////////////////////////////////
@@ -102,13 +110,13 @@ class BulkInsertTransformStepTest extends BaseTest
       input.setTableName(TABLE_NAME);
       input.setStepName(StreamedETLWithFrontendProcess.STEP_NAME_VALIDATE);
       input.setRecords(List.of(
-         newQRecord("uuid-1", "SKU-A", 1), // OK.
+         newUkTestQRecord("uuid-1", "SKU-A", 1), // OK.
-         newQRecord("uuid-1", "SKU-B", 1), // violate uuid UK in this set
+         newUkTestQRecord("uuid-1", "SKU-B", 1), // violate uuid UK in this set
-         newQRecord("uuid-2", "SKU-C", 1), // OK.
+         newUkTestQRecord("uuid-2", "SKU-C", 1), // OK.
-         newQRecord("uuid-3", "SKU-C", 2), // OK.
+         newUkTestQRecord("uuid-3", "SKU-C", 2), // OK.
-         newQRecord("uuid-4", "SKU-C", 1), // violate sku/storeId UK in this set
+         newUkTestQRecord("uuid-4", "SKU-C", 1), // violate sku/storeId UK in this set
-         newQRecord("uuid-A", "SKU-X", 1), // violate uuid UK from pre-existing records
+         newUkTestQRecord("uuid-A", "SKU-X", 1), // violate uuid UK from pre-existing records
-         newQRecord("uuid-D", "SKU-2", 1)  // violate sku/storeId UK from pre-existing records
+         newUkTestQRecord("uuid-D", "SKU-2", 1)  // violate sku/storeId UK from pre-existing records
       ));
       bulkInsertTransformStep.preRun(input, output);
       bulkInsertTransformStep.runOnePage(input, output);
@@ -171,9 +179,9 @@ class BulkInsertTransformStepTest extends BaseTest
       // insert some records that will cause some UK violations //
       ////////////////////////////////////////////////////////////
       TestUtils.insertRecords(table, List.of(
-         newQRecord("uuid-A", "SKU-1", 1),
+         newUkTestQRecord("uuid-A", "SKU-1", 1),
-         newQRecord("uuid-B", "SKU-2", 1),
+         newUkTestQRecord("uuid-B", "SKU-2", 1),
-         newQRecord("uuid-C", "SKU-2", 2)
+         newUkTestQRecord("uuid-C", "SKU-2", 2)
       ));
 
       ///////////////////////////////////////////
@@ -186,20 +194,20 @@ class BulkInsertTransformStepTest extends BaseTest
       input.setTableName(TABLE_NAME);
       input.setStepName(StreamedETLWithFrontendProcess.STEP_NAME_VALIDATE);
       input.setRecords(List.of(
-         newQRecord("uuid-1", "SKU-A", 1), // OK.
+         newUkTestQRecord("uuid-1", "SKU-A", 1), // OK.
-         newQRecord("uuid-1", "SKU-B", 1), // violate uuid UK in this set
+         newUkTestQRecord("uuid-1", "SKU-B", 1), // violate uuid UK in this set
-         newQRecord("uuid-2", "SKU-C", 1), // OK.
+         newUkTestQRecord("uuid-2", "SKU-C", 1), // OK.
-         newQRecord("uuid-3", "SKU-C", 2), // OK.
+         newUkTestQRecord("uuid-3", "SKU-C", 2), // OK.
-         newQRecord("uuid-4", "SKU-C", 1), // violate sku/storeId UK in this set
+         newUkTestQRecord("uuid-4", "SKU-C", 1), // violate sku/storeId UK in this set
-         newQRecord("uuid-A", "SKU-X", 1), // violate uuid UK from pre-existing records
+         newUkTestQRecord("uuid-A", "SKU-X", 1), // violate uuid UK from pre-existing records
-         newQRecord("uuid-D", "SKU-2", 1)  // violate sku/storeId UK from pre-existing records
+         newUkTestQRecord("uuid-D", "SKU-2", 1)  // violate sku/storeId UK from pre-existing records
       ));
       bulkInsertTransformStep.preRun(input, output);
       bulkInsertTransformStep.runOnePage(input, output);
 
-      ///////////////////////////////////////////////////////
+      ///////////////////////////////////
-      // assert that all records pass.
+      // assert that all records pass. //
-      ///////////////////////////////////////////////////////
+      ///////////////////////////////////
       assertEquals(7, output.getRecords().size());
    }
 
@@ -211,8 +219,8 @@ class BulkInsertTransformStepTest extends BaseTest
    private boolean recordEquals(QRecord record, String uuid, String sku, Integer storeId)
    {
       return (record.getValue("uuid").equals(uuid)
-         && record.getValue("sku").equals(sku)
+              && record.getValue("sku").equals(sku)
-         && record.getValue("storeId").equals(storeId));
+              && record.getValue("storeId").equals(storeId));
    }
 
 
@@ -220,7 +228,7 @@ class BulkInsertTransformStepTest extends BaseTest
    /*******************************************************************************
     **
     *******************************************************************************/
-   private QRecord newQRecord(String uuid, String sku, int storeId)
+   private QRecord newUkTestQRecord(String uuid, String sku, int storeId)
    {
       return new QRecord()
          .withValue("uuid", uuid)
@@ -229,4 +237,134 @@ class BulkInsertTransformStepTest extends BaseTest
          .withValue("name", "Some Item");
    }
 
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testValueMappingTypeErrors() throws QException
+   {
+      ///////////////////////////////////////////
+      // setup & run the bulk insert transform //
+      ///////////////////////////////////////////
+      BulkInsertTransformStep bulkInsertTransformStep = new BulkInsertTransformStep();
+      RunBackendStepInput     input                   = new RunBackendStepInput();
+      RunBackendStepOutput    output                  = new RunBackendStepOutput();
+      Serializable[]          emptyValues             = new Serializable[0];
+
+      input.setTableName(TestUtils.TABLE_NAME_ORDER);
+      input.setStepName(StreamedETLWithFrontendProcess.STEP_NAME_VALIDATE);
+      input.setRecords(ListBuilder.of(
+         BulkLoadRecordUtils.addBackendDetailsAboutFileRows(new QRecord(), new BulkLoadFileRow(emptyValues, 1))
+            .withError(new BulkLoadValueTypeError("storeId", "A", QFieldType.INTEGER, "Store"))
+            .withError(new BulkLoadValueTypeError("orderDate", "47", QFieldType.DATE, "Order Date")),
+         BulkLoadRecordUtils.addBackendDetailsAboutFileRows(new QRecord(), new BulkLoadFileRow(emptyValues, 2))
+            .withError(new BulkLoadValueTypeError("storeId", "BCD", QFieldType.INTEGER, "Store"))
+      ));
+
+      ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      // add 102 records with an error in the total field - which is more than the number of examples that should be given //
+      ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      for(int i = 0; i < 102; i++)
+      {
+         input.getRecords().add(BulkLoadRecordUtils.addBackendDetailsAboutFileRows(new QRecord(), new BulkLoadFileRow(emptyValues, 3 + i))
+            .withError(new BulkLoadValueTypeError("total", "three-fifty-" + i, QFieldType.DECIMAL, "Total")));
+      }
+
+      bulkInsertTransformStep.preRun(input, output);
+      bulkInsertTransformStep.runOnePage(input, output);
+      ArrayList<ProcessSummaryLineInterface> processSummary = bulkInsertTransformStep.getProcessSummary(output, false);
+
+      ProcessSummaryAssert.assertThat(processSummary)
+         .hasLineWithMessageContaining("Cannot convert value for field [Store] to type [Integer]")
+         .hasMessageContaining("Values:")
+         .doesNotHaveMessageContaining("Example Values:")
+         .hasAnyBulletsOfTextContaining("Row 1 [A]")
+         .hasAnyBulletsOfTextContaining("Row 2 [BCD]")
+         .hasStatus(Status.ERROR)
+         .hasCount(2);
+
+      ProcessSummaryAssert.assertThat(processSummary)
+         .hasLineWithMessageContaining("Cannot convert value for field [Order Date] to type [Date]")
+         .hasMessageContaining("Values:")
+         .doesNotHaveMessageContaining("Example Values:")
+         .hasAnyBulletsOfTextContaining("Row 1 [47]")
+         .hasStatus(Status.ERROR)
+         .hasCount(1);
+
+      ProcessSummaryAssert.assertThat(processSummary)
+         .hasLineWithMessageContaining("Cannot convert value for field [Total] to type [Decimal]")
+         .hasMessageContaining("Example Values:")
+         .hasAnyBulletsOfTextContaining("Row 3 [three-fifty-0]")
+         .hasAnyBulletsOfTextContaining("Row 4 [three-fifty-1]")
+         .hasAnyBulletsOfTextContaining("Row 5 [three-fifty-2]")
+         .doesNotHaveAnyBulletsOfTextContaining("three-fifty-101")
+         .hasStatus(Status.ERROR)
+         .hasCount(102);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testRollupOfValidationErrors() throws QException
+   {
+      QContext.getQSession().withSecurityKeyValue(TestUtils.SECURITY_KEY_TYPE_STORE, 1);
+
+      ///////////////////////////////////////////
+      // setup & run the bulk insert transform //
+      ///////////////////////////////////////////
+      BulkInsertTransformStep bulkInsertTransformStep = new BulkInsertTransformStep();
+      RunBackendStepInput     input                   = new RunBackendStepInput();
+      RunBackendStepOutput    output                  = new RunBackendStepOutput();
+      Serializable[]          emptyValues             = new Serializable[0];
+
+      String tooLong = ".".repeat(201);
+
+      input.setTableName(TestUtils.TABLE_NAME_ORDER);
+      input.setStepName(StreamedETLWithFrontendProcess.STEP_NAME_VALIDATE);
+      input.setRecords(ListBuilder.of(
+         BulkLoadRecordUtils.addBackendDetailsAboutFileRows(new QRecord().withValue("shipToName", tooLong), new BulkLoadFileRow(emptyValues, 1)),
+         BulkLoadRecordUtils.addBackendDetailsAboutFileRows(new QRecord().withValue("shipToName", "OK").withValue("storeId", 1), new BulkLoadFileRow(emptyValues, 2))
+      ));
+
+      /////////////////////////////////////////////////////////////////////
+      // add 102 records with no security key - which should be an error //
+      /////////////////////////////////////////////////////////////////////
+      for(int i = 0; i < 102; i++)
+      {
+         input.getRecords().add(BulkLoadRecordUtils.addBackendDetailsAboutFileRows(new QRecord(), new BulkLoadFileRow(emptyValues, 3 + i)));
+      }
+
+      bulkInsertTransformStep.preRun(input, output);
+      bulkInsertTransformStep.runOnePage(input, output);
+      ArrayList<ProcessSummaryLineInterface> processSummary = bulkInsertTransformStep.getProcessSummary(output, false);
+
+      ProcessSummaryAssert.assertThat(processSummary)
+         .hasLineWithMessageContaining("value for Ship To Name is too long")
+         .hasMessageContaining("Records:")
+         .doesNotHaveMessageContaining("Example Records:")
+         .hasAnyBulletsOfTextContaining("Row 1")
+         .hasStatus(Status.ERROR)
+         .hasCount(1);
+
+      ProcessSummaryAssert.assertThat(processSummary)
+         .hasLineWithMessageContaining("without a value in the field: Store Id")
+         .hasMessageContaining("Example Records:")
+         .hasAnyBulletsOfTextContaining("Row 1")
+         .hasAnyBulletsOfTextContaining("Row 3")
+         .hasAnyBulletsOfTextContaining("Row 4")
+         .doesNotHaveAnyBulletsOfTextContaining("Row 101")
+         .hasStatus(Status.ERROR)
+         .hasCount(103); // the 102, plus row 1.
+
+      ProcessSummaryAssert.assertThat(processSummary)
+         .hasLineWithMessageContaining("Order record will be inserted")
+         .hasStatus(Status.OK)
+         .hasCount(1);
+   }
+
 }

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java

@@ -70,6 +70,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.dashboard.QWidgetMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.DisplayFormat;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.ValueTooLongBehavior;
 import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinOn;
 import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinType;
 import com.kingsrook.qqq.backend.core.model.metadata.joins.QJoinMetaData;
@@ -662,7 +663,7 @@ public class TestUtils
          .withField(new QFieldMetaData("createDate", QFieldType.DATE_TIME).withIsEditable(false))
          .withField(new QFieldMetaData("modifyDate", QFieldType.DATE_TIME).withIsEditable(false))
          .withField(new QFieldMetaData("orderNo", QFieldType.STRING))
-         .withField(new QFieldMetaData("shipToName", QFieldType.STRING))
+         .withField(new QFieldMetaData("shipToName", QFieldType.STRING).withMaxLength(200).withBehavior(ValueTooLongBehavior.ERROR))
          .withField(new QFieldMetaData("orderDate", QFieldType.DATE))
          .withField(new QFieldMetaData("storeId", QFieldType.INTEGER))
          .withField(new QFieldMetaData("total", QFieldType.DECIMAL).withDisplayFormat(DisplayFormat.CURRENCY).withFieldSecurityLock(new FieldSecurityLock()