Merged dev into feature/join-record-enhancements

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/processes/RunProcessActionTest.java

@@ -23,10 +23,15 @@ package com.kingsrook.qqq.backend.core.actions.processes;
 
 
 import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicInteger;
 import com.kingsrook.qqq.backend.core.BaseTest;
 import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessState;
 import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput;
 import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessOutput;
 import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReferenceLambda;
@@ -35,6 +40,8 @@ import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaD
 import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendStepMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.processes.QStateMachineStep;
+import com.kingsrook.qqq.backend.core.model.metadata.processes.QStepMetaData;
+import com.kingsrook.qqq.backend.core.utils.collections.MultiLevelMapHelper;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
 import static org.assertj.core.api.Assertions.assertThat;
@@ -73,14 +80,14 @@ class RunProcessActionTest extends BaseTest
          /////////////////////////////////////////////////////////////////
          // two-steps - a, points at b; b has no next-step, so it exits //
          /////////////////////////////////////////////////////////////////
-         .addStep(QStateMachineStep.backendOnly("a", new QBackendStepMetaData().withName("aBackend")
+         .withStep(QStateMachineStep.backendOnly("a", new QBackendStepMetaData().withName("aBackend")
             .withCode(new QCodeReferenceLambda<BackendStep>((runBackendStepInput, runBackendStepOutput) ->
             {
                log.add("in StepA");
                runBackendStepOutput.getProcessState().setNextStepName("b");
             }))))
 
-         .addStep(QStateMachineStep.backendOnly("b", new QBackendStepMetaData().withName("bBackend")
+         .withStep(QStateMachineStep.backendOnly("b", new QBackendStepMetaData().withName("bBackend")
             .withCode(new QCodeReferenceLambda<BackendStep>((runBackendStepInput, runBackendStepOutput) ->
             {
                log.add("in StepB");
@@ -109,8 +116,8 @@ class RunProcessActionTest extends BaseTest
    {
       QProcessMetaData process = new QProcessMetaData().withName("test")
 
-         .addStep(QStateMachineStep.frontendOnly("a", new QFrontendStepMetaData().withName("aFrontend")).withDefaultNextStepName("b"))
-         .addStep(QStateMachineStep.frontendOnly("b", new QFrontendStepMetaData().withName("bFrontend")))
+         .withStep(QStateMachineStep.frontendOnly("a", new QFrontendStepMetaData().withName("aFrontend")).withDefaultNextStepName("b"))
+         .withStep(QStateMachineStep.frontendOnly("b", new QFrontendStepMetaData().withName("bFrontend")))
 
          .withStepFlow(ProcessStepFlow.STATE_MACHINE);
 
@@ -150,7 +157,7 @@ class RunProcessActionTest extends BaseTest
          // since it never goes to the frontend, it'll stack overflow //
          // (though we'll catch it ourselves before JVM does)         //
          ///////////////////////////////////////////////////////////////
-         .addStep(QStateMachineStep.backendOnly("a", new QBackendStepMetaData().withName("aBackend")
+         .withStep(QStateMachineStep.backendOnly("a", new QBackendStepMetaData().withName("aBackend")
             .withCode(new QCodeReferenceLambda<BackendStep>((runBackendStepInput, runBackendStepOutput) ->
             {
                log.add("in StepA");
@@ -193,14 +200,14 @@ class RunProcessActionTest extends BaseTest
          // since it never goes to the frontend, it'll stack overflow //
          // (though we'll catch it ourselves before JVM does)         //
          ///////////////////////////////////////////////////////////////
-         .addStep(QStateMachineStep.backendOnly("a", new QBackendStepMetaData().withName("aBackend")
+         .withStep(QStateMachineStep.backendOnly("a", new QBackendStepMetaData().withName("aBackend")
             .withCode(new QCodeReferenceLambda<BackendStep>((runBackendStepInput, runBackendStepOutput) ->
             {
                log.add("in StepA");
                runBackendStepOutput.getProcessState().setNextStepName("b");
             }))))
 
-         .addStep(QStateMachineStep.backendOnly("b", new QBackendStepMetaData().withName("bBackend")
+         .withStep(QStateMachineStep.backendOnly("b", new QBackendStepMetaData().withName("bBackend")
             .withCode(new QCodeReferenceLambda<BackendStep>((runBackendStepInput, runBackendStepOutput) ->
             {
                log.add("in StepB");
@@ -238,7 +245,7 @@ class RunProcessActionTest extends BaseTest
    {
       QProcessMetaData process = new QProcessMetaData().withName("test")
 
-         .addStep(QStateMachineStep.frontendThenBackend("a",
+         .withStep(QStateMachineStep.frontendThenBackend("a",
             new QFrontendStepMetaData().withName("aFrontend"),
             new QBackendStepMetaData().withName("aBackend")
                .withCode(new QCodeReferenceLambda<BackendStep>((runBackendStepInput, runBackendStepOutput) ->
@@ -247,7 +254,7 @@ class RunProcessActionTest extends BaseTest
                   runBackendStepOutput.getProcessState().setNextStepName("b");
                }))))
 
-         .addStep(QStateMachineStep.frontendThenBackend("b",
+         .withStep(QStateMachineStep.frontendThenBackend("b",
             new QFrontendStepMetaData().withName("bFrontend"),
             new QBackendStepMetaData().withName("bBackend")
                .withCode(new QCodeReferenceLambda<BackendStep>((runBackendStepInput, runBackendStepOutput) ->
@@ -256,7 +263,7 @@ class RunProcessActionTest extends BaseTest
                   runBackendStepOutput.getProcessState().setNextStepName("c");
                }))))
 
-         .addStep(QStateMachineStep.frontendThenBackend("c",
+         .withStep(QStateMachineStep.frontendThenBackend("c",
             new QFrontendStepMetaData().withName("cFrontend"),
             new QBackendStepMetaData().withName("cBackend")
                .withCode(new QCodeReferenceLambda<BackendStep>((runBackendStepInput, runBackendStepOutput) ->
@@ -265,7 +272,7 @@ class RunProcessActionTest extends BaseTest
                   runBackendStepOutput.getProcessState().setNextStepName("d");
                }))))
 
-         .addStep(QStateMachineStep.frontendOnly("d",
+         .withStep(QStateMachineStep.frontendOnly("d",
             new QFrontendStepMetaData().withName("dFrontend")))
 
          .withStepFlow(ProcessStepFlow.STATE_MACHINE);
@@ -321,7 +328,132 @@ class RunProcessActionTest extends BaseTest
       runProcessOutput = new RunProcessAction().execute(input);
       assertEquals(List.of("in StepA", "in StepB", "in StepC"), log);
       assertThat(runProcessOutput.getProcessState().getNextStepName()).isEmpty();
+   }
 
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testGoingBack() throws QException
+   {
+      AtomicInteger        backCount     = new AtomicInteger(0);
+      Map<String, Integer> stepRunCounts = new HashMap<>();
+
+      BackendStep backendStep = (runBackendStepInput, runBackendStepOutput) ->
+      {
+         //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+         // shared backend-step lambda, that will do the same thing for both - but using step name to count how many times each is executed. //
+         //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+         MultiLevelMapHelper.getOrPutAndIncrement(stepRunCounts, runBackendStepInput.getStepName());
+         if(runBackendStepInput.getIsStepBack())
+         {
+            backCount.incrementAndGet();
+         }
+      };
+
+      ///////////////////////////////////////////////////////////
+      // normal flow here:  a -> b -> c                        //
+      // but, b can go back to a, as in: a -> b -> a -> b -> c //
+      ///////////////////////////////////////////////////////////
+      QProcessMetaData process = new QProcessMetaData().withName("test")
+         .withStep(new QBackendStepMetaData()
+            .withName("a")
+            .withCode(new QCodeReferenceLambda<>(backendStep)))
+         .withStep(new QFrontendStepMetaData()
+            .withName("b")
+            .withBackStepName("a"))
+         .withStep(new QBackendStepMetaData()
+            .withName("c")
+            .withCode(new QCodeReferenceLambda<>(backendStep)))
+         .withStepFlow(ProcessStepFlow.LINEAR);
+
+      QContext.getQInstance().addProcess(process);
+
+      RunProcessInput input = new RunProcessInput();
+      input.setProcessName("test");
+      input.setFrontendStepBehavior(RunProcessInput.FrontendStepBehavior.BREAK);
+
+      ///////////////////////////////////////////////////////////
+      // start the process - we should be sent to b (frontend) //
+      ///////////////////////////////////////////////////////////
+      RunProcessOutput runProcessOutput = new RunProcessAction().execute(input);
+      assertThat(runProcessOutput.getProcessState().getNextStepName())
+         .isPresent().get()
+         .isEqualTo("b");
+
+      assertEquals(0, backCount.get());
+      assertEquals(Map.of("a", 1), stepRunCounts);
+
+      ////////////////////////////////////////////////////////////////
+      // resume after b, but in back-mode - should end up back at b //
+      ////////////////////////////////////////////////////////////////
+      input.setStartAfterStep(null);
+      input.setStartAtStep("a");
+      runProcessOutput = new RunProcessAction().execute(input);
+      assertThat(runProcessOutput.getProcessState().getNextStepName())
+         .isPresent().get()
+         .isEqualTo("b");
+
+      assertEquals(1, backCount.get());
+      assertEquals(Map.of("a", 2), stepRunCounts);
+
+      ////////////////////////////////////////////////////////////////////////////
+      // resume after b, in regular (forward) mode - should wrap up the process //
+      ////////////////////////////////////////////////////////////////////////////
+      input.setStartAfterStep("b");
+      input.setStartAtStep(null);
+      runProcessOutput = new RunProcessAction().execute(input);
+      assertThat(runProcessOutput.getProcessState().getNextStepName())
+         .isEmpty();
+
+      assertEquals(1, backCount.get());
+      assertEquals(Map.of("a", 2, "c", 1), stepRunCounts);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testGetAvailableStepList() throws QException
+   {
+      QProcessMetaData process = new QProcessMetaData()
+         .withStep(new QBackendStepMetaData().withName("A"))
+         .withStep(new QBackendStepMetaData().withName("B"))
+         .withStep(new QBackendStepMetaData().withName("C"))
+         .withStep(new QBackendStepMetaData().withName("D"))
+         .withStep(new QBackendStepMetaData().withName("E"));
+
+      ProcessState processState = new ProcessState();
+      processState.setStepList(process.getStepList().stream().map(s -> s.getName()).toList());
+
+      assertStepListNames(List.of("A", "B", "C", "D", "E"), RunProcessAction.getAvailableStepList(processState, process, null, false));
+      assertStepListNames(List.of("A", "B", "C", "D", "E"), RunProcessAction.getAvailableStepList(processState, process, null, true));
+
+      assertStepListNames(List.of("B", "C", "D", "E"), RunProcessAction.getAvailableStepList(processState, process, "A", false));
+      assertStepListNames(List.of("A", "B", "C", "D", "E"), RunProcessAction.getAvailableStepList(processState, process, "A", true));
+
+      assertStepListNames(List.of("D", "E"), RunProcessAction.getAvailableStepList(processState, process, "C", false));
+      assertStepListNames(List.of("C", "D", "E"), RunProcessAction.getAvailableStepList(processState, process, "C", true));
+
+      assertStepListNames(Collections.emptyList(), RunProcessAction.getAvailableStepList(processState, process, "E", false));
+      assertStepListNames(List.of("E"), RunProcessAction.getAvailableStepList(processState, process, "E", true));
+
+      assertStepListNames(Collections.emptyList(), RunProcessAction.getAvailableStepList(processState, process, "Z", false));
+      assertStepListNames(Collections.emptyList(), RunProcessAction.getAvailableStepList(processState, process, "Z", true));
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   private void assertStepListNames(List<String> expectedNames, List<QStepMetaData> actualSteps)
+   {
+      assertEquals(expectedNames, actualSteps.stream().map(s -> s.getName()).toList());
    }
 
 }

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/reporting/GenerateReportActionTest.java

@@ -90,7 +90,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals;
  *******************************************************************************/
 public class GenerateReportActionTest extends BaseTest
 {
-   private static final String REPORT_NAME = "personReport1";
+   public static final String REPORT_NAME = "personReport1";
 
 
 
@@ -655,7 +655,7 @@ public class GenerateReportActionTest extends BaseTest
       Iterator<Map<String, String>> iterator = list.iterator();
       Map<String, String>           row      = iterator.next();
       assertEquals(5, list.size());
-      assertThat(row).containsOnlyKeys("Id", "First Name", "Last Name");
+      assertThat(row).containsOnlyKeys("Id", "First Name", "Last Name", "Birth Date");
    }
 
 
@@ -663,7 +663,7 @@ public class GenerateReportActionTest extends BaseTest
    /*******************************************************************************
     **
     *******************************************************************************/
-   private static QReportMetaData defineTableOnlyReport()
+   public static QReportMetaData defineTableOnlyReport()
    {
       QReportMetaData report = new QReportMetaData()
          .withName(REPORT_NAME)
@@ -686,7 +686,9 @@ public class GenerateReportActionTest extends BaseTest
                .withColumns(List.of(
                   new QReportField().withName("id"),
                   new QReportField().withName("firstName"),
-                  new QReportField().withName("lastName")))));
+                  new QReportField().withName("lastName"),
+                  new QReportField().withName("birthDate")
+               ))));
 
       return report;
    }

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/values/SearchPossibleValueSourceActionTest.java

@@ -217,6 +217,63 @@ class SearchPossibleValueSourceActionTest extends BaseTest
    }
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testSearchPvsAction_tableByLabels() throws QException
+   {
+      {
+         SearchPossibleValueSourceOutput output = getSearchPossibleValueSourceOutputByLabels(List.of("Square", "Circle"), TestUtils.POSSIBLE_VALUE_SOURCE_SHAPE);
+         assertEquals(2, output.getResults().size());
+         assertThat(output.getResults()).anyMatch(pv -> pv.getId().equals(2) && pv.getLabel().equals("Square"));
+         assertThat(output.getResults()).anyMatch(pv -> pv.getId().equals(3) && pv.getLabel().equals("Circle"));
+      }
+
+      {
+         SearchPossibleValueSourceOutput output = getSearchPossibleValueSourceOutputByLabels(List.of(), TestUtils.POSSIBLE_VALUE_SOURCE_SHAPE);
+         assertEquals(0, output.getResults().size());
+      }
+
+      {
+         SearchPossibleValueSourceOutput output = getSearchPossibleValueSourceOutputByLabels(List.of("notFound"), TestUtils.POSSIBLE_VALUE_SOURCE_SHAPE);
+         assertEquals(0, output.getResults().size());
+      }
+   }
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testSearchPvsAction_enumByLabel() throws QException
+   {
+      {
+         SearchPossibleValueSourceOutput output = getSearchPossibleValueSourceOutputByLabels(List.of("IL", "MO", "XX"), TestUtils.POSSIBLE_VALUE_SOURCE_STATE);
+         assertEquals(2, output.getResults().size());
+         assertThat(output.getResults()).anyMatch(pv -> pv.getId().equals(1) && pv.getLabel().equals("IL"));
+         assertThat(output.getResults()).anyMatch(pv -> pv.getId().equals(2) && pv.getLabel().equals("MO"));
+      }
+
+      {
+         SearchPossibleValueSourceOutput output = getSearchPossibleValueSourceOutputByLabels(List.of("Il", "mo", "XX"), TestUtils.POSSIBLE_VALUE_SOURCE_STATE);
+         assertEquals(2, output.getResults().size());
+         assertThat(output.getResults()).anyMatch(pv -> pv.getId().equals(1) && pv.getLabel().equals("IL"));
+         assertThat(output.getResults()).anyMatch(pv -> pv.getId().equals(2) && pv.getLabel().equals("MO"));
+      }
+
+      {
+         SearchPossibleValueSourceOutput output = getSearchPossibleValueSourceOutputByLabels(List.of(), TestUtils.POSSIBLE_VALUE_SOURCE_STATE);
+         assertEquals(0, output.getResults().size());
+      }
+
+      {
+         SearchPossibleValueSourceOutput output = getSearchPossibleValueSourceOutputByLabels(List.of("not-found"), TestUtils.POSSIBLE_VALUE_SOURCE_STATE);
+         assertEquals(0, output.getResults().size());
+      }
+   }
+
+
 
    /*******************************************************************************
     **
@@ -414,4 +471,18 @@ class SearchPossibleValueSourceActionTest extends BaseTest
       return output;
    }
 
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private SearchPossibleValueSourceOutput getSearchPossibleValueSourceOutputByLabels(List<String> labels, String possibleValueSourceName) throws QException
+   {
+      SearchPossibleValueSourceInput input = new SearchPossibleValueSourceInput();
+      input.setLabelList(labels);
+      input.setPossibleValueSourceName(possibleValueSourceName);
+      SearchPossibleValueSourceOutput output = new SearchPossibleValueSourceAction().execute(input);
+      return output;
+   }
+
 }

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricherTest.java

@@ -27,6 +27,7 @@ import java.util.Collections;
 import java.util.List;
 import java.util.Optional;
 import com.kingsrook.qqq.backend.core.BaseTest;
+import com.kingsrook.qqq.backend.core.instances.enrichment.plugins.QInstanceEnricherPluginInterface;
 import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
 import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.AdornmentType;
@@ -47,6 +48,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.tables.QFieldSection;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.Tier;
 import com.kingsrook.qqq.backend.core.utils.TestUtils;
+import org.junit.jupiter.api.AfterEach;
 import org.junit.jupiter.api.Test;
 import static com.kingsrook.qqq.backend.core.utils.TestUtils.APP_NAME_GREETINGS;
 import static com.kingsrook.qqq.backend.core.utils.TestUtils.APP_NAME_MISCELLANEOUS;
@@ -66,6 +68,17 @@ import static org.junit.jupiter.api.Assertions.assertTrue;
 class QInstanceEnricherTest extends BaseTest
 {
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @AfterEach
+   void afterEach()
+   {
+      QInstanceEnricher.removeAllEnricherPlugins();
+   }
+
+
+
    /*******************************************************************************
     ** Test that a table missing a label gets the default label applied (name w/ UC-first).
     **
@@ -572,4 +585,37 @@ class QInstanceEnricherTest extends BaseTest
       assertEquals("My Field", qInstance.getProcess("test").getFrontendStep("screen").getViewFields().get(0).getLabel());
    }
 
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testFieldPlugIn()
+   {
+      QInstance qInstance = TestUtils.defineInstance();
+
+      QInstanceEnricher.addEnricherPlugin(new QInstanceEnricherPluginInterface<QFieldMetaData>()
+      {
+         /***************************************************************************
+          *
+          ***************************************************************************/
+         @Override
+         public void enrich(QFieldMetaData field, QInstance qInstance)
+         {
+            if(field != null)
+            {
+               field.setLabel(field.getLabel() + " Plugged");
+            }
+         }
+      });
+
+      new QInstanceEnricher(qInstance).enrich();
+
+      qInstance.getTables().values().forEach(table -> table.getFields().values().forEach(field -> assertThat(field.getLabel()).endsWith("Plugged")));
+      qInstance.getProcesses().values().forEach(process -> process.getInputFields().forEach(field -> assertThat(field.getLabel()).endsWith("Plugged")));
+      qInstance.getProcesses().values().forEach(process -> process.getOutputFields().forEach(field -> assertThat(field.getLabel()).endsWith("Plugged")));
+
+   }
+
 }

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QInstanceHelpContentManagerTest.java

@@ -47,6 +47,8 @@ import com.kingsrook.qqq.backend.core.model.metadata.dashboard.QWidgetMetaDataIn
 import com.kingsrook.qqq.backend.core.model.metadata.help.HelpRole;
 import com.kingsrook.qqq.backend.core.model.metadata.help.QHelpContent;
 import com.kingsrook.qqq.backend.core.model.metadata.help.QHelpRole;
+import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
 import com.kingsrook.qqq.backend.core.utils.TestUtils;
 import org.junit.jupiter.api.Test;
@@ -292,6 +294,49 @@ class QInstanceHelpContentManagerTest extends BaseTest
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testWildcardProcessField() throws QException
+   {
+      /////////////////////////////////////
+      // get the instance from base test //
+      /////////////////////////////////////
+      QInstance qInstance = QContext.getQInstance();
+      new HelpContentMetaDataProvider().defineAll(qInstance, TestUtils.MEMORY_BACKEND_NAME, null);
+
+      HelpContent recordEntity = new HelpContent()
+         .withId(1)
+         .withKey("process:*.bulkInsert;step:upload")
+         .withContent("v1")
+         .withRole(HelpContentRole.PROCESS_SCREEN.getId());
+      new InsertAction().execute(new InsertInput(HelpContent.TABLE_NAME).withRecordEntity(recordEntity));
+
+      /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      // now - post-insert customizer should have automatically added help content to the instance - to all bulkInsert processes //
+      /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      int hitCount = 0;
+      for(QTableMetaData table : qInstance.getTables().values())
+      {
+         QProcessMetaData process = qInstance.getProcess(table.getName() + ".bulkInsert");
+         if(process == null)
+         {
+            return;
+         }
+
+         List<QHelpContent> helpContents = process.getFrontendStep("upload").getHelpContents();
+         assertEquals(1, helpContents.size());
+         assertEquals("v1", helpContents.get(0).getContent());
+         assertEquals(Set.of(QHelpRole.PROCESS_SCREEN), helpContents.get(0).getRoles());
+         hitCount++;
+      }
+
+      assertThat(hitCount).isGreaterThanOrEqualTo(3);
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/
@@ -411,7 +456,7 @@ class QInstanceHelpContentManagerTest extends BaseTest
 
       QInstanceHelpContentManager.processHelpContentRecord(qInstance, helpContentCreator.apply("foo;bar:baz"));
       assertThat(collectingLogger.getCollectedMessages()).hasSize(1);
-      assertThat(collectingLogger.getCollectedMessages().get(0).getMessage()).contains("Discarding help content with key that does not contain name:value format");
+      assertThat(collectingLogger.getCollectedMessages().get(0).getMessage()).contains("Discarding help content with key-part that does not contain name:value format");
       collectingLogger.clear();
 
       QInstanceHelpContentManager.processHelpContentRecord(qInstance, helpContentCreator.apply(null));

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidatorTest.java

@@ -1185,10 +1185,12 @@ public class QInstanceValidatorTest extends BaseTest
          "should not have searchFields",
          "should not have orderByFields",
          "should not have a customCodeReference",
-         "is missing enum values");
+         "is missing enum values",
+         "is missing its idType.");
 
       assertValidationFailureReasons((qInstance) -> qInstance.getPossibleValueSource(TestUtils.POSSIBLE_VALUE_SOURCE_STATE).setEnumValues(new ArrayList<>()),
-         "is missing enum values");
+         "is missing enum values",
+         "is missing its idType.");
    }
 
 
@@ -1213,10 +1215,12 @@ public class QInstanceValidatorTest extends BaseTest
          "should not have a customCodeReference",
          "is missing a tableName",
          "is missing searchFields",
-         "is missing orderByFields");
+         "is missing orderByFields",
+         "is missing its idType.");
 
       assertValidationFailureReasons((qInstance) -> qInstance.getPossibleValueSource(TestUtils.POSSIBLE_VALUE_SOURCE_SHAPE).setTableName("Not a table"),
-         "Unrecognized table");
+         "Unrecognized table",
+         "is missing its idType.");
 
       assertValidationFailureReasons((qInstance) -> qInstance.getPossibleValueSource(TestUtils.POSSIBLE_VALUE_SOURCE_SHAPE).setSearchFields(List.of("id", "notAField", "name")),
          "unrecognized searchField: notAField");
@@ -1244,11 +1248,13 @@ public class QInstanceValidatorTest extends BaseTest
          "should not have a tableName",
          "should not have searchFields",
          "should not have orderByFields",
-         "is missing a customCodeReference");
+         "is missing a customCodeReference",
+         "is missing its idType.");
 
       assertValidationFailureReasons((qInstance) -> qInstance.getPossibleValueSource(TestUtils.POSSIBLE_VALUE_SOURCE_CUSTOM).setCustomCodeReference(new QCodeReference()),
          "missing a code reference name",
-         "missing a code type");
+         "missing a code type",
+         "is missing its idType.");
    }
 
 

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/actions/processes/ProcessSummaryAssert.java

@@ -0,0 +1,208 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.model.actions.processes;
+
+
+import java.util.ArrayList;
+import java.util.List;
+import com.kingsrook.qqq.backend.core.utils.StringUtils;
+import org.assertj.core.api.AbstractAssert;
+import org.assertj.core.api.Assertions;
+import static org.junit.jupiter.api.Assertions.fail;
+
+
+/*******************************************************************************
+ ** AssertJ assert class for ProcessSummary - that is - a list of ProcessSummaryLineInterface's
+ *******************************************************************************/
+public class ProcessSummaryAssert extends AbstractAssert<ProcessSummaryAssert, List<ProcessSummaryLineInterface>>
+{
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   protected ProcessSummaryAssert(List<ProcessSummaryLineInterface> actual, Class<?> selfType)
+   {
+      super(actual, selfType);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @SuppressWarnings("unchecked")
+   public static ProcessSummaryAssert assertThat(RunProcessOutput runProcessOutput)
+   {
+      List<ProcessSummaryLineInterface> processResults = (List<ProcessSummaryLineInterface>) runProcessOutput.getValue("processResults");
+      if(processResults == null)
+      {
+         processResults = (List<ProcessSummaryLineInterface>) runProcessOutput.getValue("validationSummary");
+      }
+
+      return (new ProcessSummaryAssert(processResults, ProcessSummaryAssert.class));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @SuppressWarnings("unchecked")
+   public static ProcessSummaryAssert assertThat(RunBackendStepOutput runBackendStepOutput)
+   {
+      List<ProcessSummaryLineInterface> processResults = (List<ProcessSummaryLineInterface>) runBackendStepOutput.getValue("processResults");
+      if(processResults == null)
+      {
+         processResults = (List<ProcessSummaryLineInterface>) runBackendStepOutput.getValue("validationSummary");
+      }
+
+      if(processResults == null)
+      {
+         fail("Could not find process results in backend step output.");
+      }
+
+      return (new ProcessSummaryAssert(processResults, ProcessSummaryAssert.class));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public static ProcessSummaryAssert assertThat(List<ProcessSummaryLineInterface> actual)
+   {
+      return (new ProcessSummaryAssert(actual, ProcessSummaryAssert.class));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public ProcessSummaryAssert hasSize(int expectedSize)
+   {
+      Assertions.assertThat(actual).hasSize(expectedSize);
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public ProcessSummaryLineInterfaceAssert hasLineWithMessageMatching(String regExp)
+   {
+      List<String> foundMessages = new ArrayList<>();
+      for(ProcessSummaryLineInterface processSummaryLineInterface : actual)
+      {
+         if(processSummaryLineInterface.getMessage() == null)
+         {
+            processSummaryLineInterface.prepareForFrontend(false);
+         }
+
+         if(processSummaryLineInterface.getMessage() != null && processSummaryLineInterface.getMessage().matches(regExp))
+         {
+            return (new ProcessSummaryLineInterfaceAssert(processSummaryLineInterface, ProcessSummaryLineInterfaceAssert.class));
+         }
+         else
+         {
+            foundMessages.add(processSummaryLineInterface.getMessage());
+         }
+      }
+
+      failWithMessage("Failed to find a ProcessSummaryLine with message matching [" + regExp + "].\nFound messages were:\n" + StringUtils.join("\n", foundMessages));
+      return (null);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public ProcessSummaryLineInterfaceAssert hasLineWithMessageContaining(String substr)
+   {
+      List<String> foundMessages = new ArrayList<>();
+      for(ProcessSummaryLineInterface processSummaryLineInterface : actual)
+      {
+         if(processSummaryLineInterface.getMessage() == null)
+         {
+            processSummaryLineInterface.prepareForFrontend(false);
+         }
+
+         if(processSummaryLineInterface.getMessage() != null && processSummaryLineInterface.getMessage().contains(substr))
+         {
+            return (new ProcessSummaryLineInterfaceAssert(processSummaryLineInterface, ProcessSummaryLineInterfaceAssert.class));
+         }
+         else
+         {
+            foundMessages.add(processSummaryLineInterface.getMessage());
+         }
+      }
+
+      failWithMessage("Failed to find a ProcessSummaryLine with message containing [" + substr + "].\nFound messages were:\n" + StringUtils.join("\n", foundMessages));
+      return (null);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public ProcessSummaryLineInterfaceAssert hasLineWithStatus(Status status)
+   {
+      List<String> foundStatuses = new ArrayList<>();
+      for(ProcessSummaryLineInterface processSummaryLineInterface : actual)
+      {
+         if(status.equals(processSummaryLineInterface.getStatus()))
+         {
+            return (new ProcessSummaryLineInterfaceAssert(processSummaryLineInterface, ProcessSummaryLineInterfaceAssert.class));
+         }
+         else
+         {
+            foundStatuses.add(String.valueOf(processSummaryLineInterface.getStatus()));
+         }
+      }
+
+      failWithMessage("Failed to find a ProcessSummaryLine with status [" + status + "].\nFound statuses were:\n" + StringUtils.join("\n", foundStatuses));
+      return (null);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public ProcessSummaryAssert hasNoLineWithStatus(Status status)
+   {
+      for(ProcessSummaryLineInterface processSummaryLineInterface : actual)
+      {
+         if(status.equals(processSummaryLineInterface.getStatus()))
+         {
+            failWithMessage("Found a ProcessSummaryLine with status [" + status + "], which was not supposed to happen.");
+            return (null);
+         }
+      }
+
+      return (this);
+   }
+
+}

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/actions/processes/ProcessSummaryLineInterfaceAssert.java

@@ -0,0 +1,188 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.model.actions.processes;
+
+
+import org.assertj.core.api.AbstractAssert;
+import org.assertj.core.api.Assertions;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+
+/*******************************************************************************
+ ** AssertJ assert class for ProcessSummaryLine.
+ *******************************************************************************/
+public class ProcessSummaryLineInterfaceAssert extends AbstractAssert<ProcessSummaryLineInterfaceAssert, ProcessSummaryLineInterface>
+{
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   protected ProcessSummaryLineInterfaceAssert(ProcessSummaryLineInterface actual, Class<?> selfType)
+   {
+      super(actual, selfType);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public static ProcessSummaryLineInterfaceAssert assertThat(ProcessSummaryLineInterface actual)
+   {
+      return (new ProcessSummaryLineInterfaceAssert(actual, ProcessSummaryLineInterfaceAssert.class));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public ProcessSummaryLineInterfaceAssert hasCount(Integer count)
+   {
+      if(actual instanceof ProcessSummaryLine psl)
+      {
+         assertEquals(count, psl.getCount(), "Expected count in process summary line");
+      }
+      else
+      {
+         failWithMessage("ProcessSummaryLineInterface is not of concrete type ProcessSummaryLine (is: " + actual.getClass().getSimpleName() + ")");
+      }
+
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public ProcessSummaryLineInterfaceAssert hasStatus(Status status)
+   {
+      assertEquals(status, actual.getStatus(), "Expected status in process summary line");
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public ProcessSummaryLineInterfaceAssert hasMessageMatching(String regExp)
+   {
+      if(actual.getMessage() == null)
+      {
+         actual.prepareForFrontend(false);
+      }
+
+      Assertions.assertThat(actual.getMessage()).matches(regExp);
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public ProcessSummaryLineInterfaceAssert hasMessageContaining(String substring)
+   {
+      if(actual.getMessage() == null)
+      {
+         actual.prepareForFrontend(false);
+      }
+
+      Assertions.assertThat(actual.getMessage()).contains(substring);
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public ProcessSummaryLineInterfaceAssert doesNotHaveMessageMatching(String regExp)
+   {
+      if(actual.getMessage() == null)
+      {
+         actual.prepareForFrontend(false);
+      }
+
+      Assertions.assertThat(actual.getMessage()).doesNotMatch(regExp);
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public ProcessSummaryLineInterfaceAssert doesNotHaveMessageContaining(String substring)
+   {
+      if(actual.getMessage() == null)
+      {
+         actual.prepareForFrontend(false);
+      }
+
+      Assertions.assertThat(actual.getMessage()).doesNotContain(substring);
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public ProcessSummaryLineInterfaceAssert hasAnyBulletsOfTextContaining(String substring)
+   {
+      if(actual instanceof ProcessSummaryLine psl)
+      {
+         Assertions.assertThat(psl.getBulletsOfText())
+            .isNotNull()
+            .anyMatch(s -> s.contains(substring));
+      }
+      else
+      {
+         Assertions.fail("Process Summary Line was not the expected type.");
+      }
+
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public ProcessSummaryLineInterfaceAssert doesNotHaveAnyBulletsOfTextContaining(String substring)
+   {
+      if(actual instanceof ProcessSummaryLine psl)
+      {
+         if(psl.getBulletsOfText() != null)
+         {
+            Assertions.assertThat(psl.getBulletsOfText())
+               .noneMatch(s -> s.contains(substring));
+         }
+      }
+
+      return (this);
+   }
+
+}

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/data/QRecordEntityTest.java

@@ -24,8 +24,10 @@ package com.kingsrook.qqq.backend.core.model.data;
 
 import java.math.BigDecimal;
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.List;
 import com.kingsrook.qqq.backend.core.BaseTest;
+import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.model.data.testentities.Item;
 import com.kingsrook.qqq.backend.core.model.data.testentities.ItemWithPrimitives;
@@ -35,7 +37,10 @@ import com.kingsrook.qqq.backend.core.model.metadata.fields.DisplayFormat;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
+import static org.assertj.core.api.Assertions.assertThat;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertFalse;
 import static org.junit.jupiter.api.Assertions.assertNotNull;
@@ -49,6 +54,31 @@ import static org.junit.jupiter.api.Assertions.assertTrue;
 class QRecordEntityTest extends BaseTest
 {
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @BeforeEach
+   void beforeEach() throws QException
+   {
+      QContext.getQInstance().addTable(new QTableMetaData()
+         .withName(Item.TABLE_NAME)
+         .withFieldsFromEntity(Item.class)
+      );
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @AfterEach
+   void afterEach()
+   {
+      QContext.getQInstance().getTables().remove(Item.TABLE_NAME);
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/
@@ -68,6 +98,19 @@ class QRecordEntityTest extends BaseTest
       assertEquals(47, qRecord.getValueInteger("quantity"));
       assertEquals(new BigDecimal("3.50"), qRecord.getValueBigDecimal("price"));
       assertTrue(qRecord.getValueBoolean("featured"));
+
+      /////////////////////////////////////////////////////////////////////////////////////////////////////////
+      // assert that, if we had no lists of associations in the entity, that we also have none in the record //
+      /////////////////////////////////////////////////////////////////////////////////////////////////////////
+      assertThat(qRecord.getAssociatedRecords()).isNullOrEmpty();
+
+      ///////////////////////////////////////////////////////////////////////
+      // now assert that an empty list translates through to an empty list //
+      ///////////////////////////////////////////////////////////////////////
+      item.setItemAlternates(Collections.emptyList());
+      qRecord = item.toQRecord();
+      assertTrue(qRecord.getAssociatedRecords().containsKey(Item.ASSOCIATION_ITEM_ALTERNATES_NAME));
+      assertTrue(qRecord.getAssociatedRecords().get(Item.ASSOCIATION_ITEM_ALTERNATES_NAME).isEmpty());
    }
 
 
@@ -76,9 +119,40 @@ class QRecordEntityTest extends BaseTest
     **
     *******************************************************************************/
    @Test
-   void testItemToQRecordOnlyChangedFields() throws QException
+   void testItemToQRecordWithAssociations() throws QException
+   {
+      Item item = new Item();
+      item.setSku("ABC-123");
+      item.setQuantity(47);
+      item.setItemAlternates(List.of(
+         new Item().withSku("DEF"),
+         new Item().withSku("GHI").withQuantity(3)
+      ));
+
+      QRecord qRecord = item.toQRecord();
+      assertEquals("ABC-123", qRecord.getValueString("sku"));
+      assertEquals(47, qRecord.getValueInteger("quantity"));
+
+      List<QRecord> associatedRecords = qRecord.getAssociatedRecords().get(Item.ASSOCIATION_ITEM_ALTERNATES_NAME);
+      assertEquals(2, associatedRecords.size());
+      assertEquals("DEF", associatedRecords.get(0).getValue("sku"));
+      assertTrue(associatedRecords.get(0).getValues().containsKey("quantity"));
+      assertNull(associatedRecords.get(0).getValue("quantity"));
+      assertEquals("GHI", associatedRecords.get(1).getValue("sku"));
+      assertEquals(3, associatedRecords.get(1).getValue("quantity"));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @SuppressWarnings("deprecation")
+   @Test
+   void testItemToQRecordOnlyChangedFieldsEntityThatCameFromQRecord() throws QException
    {
       Item item = new Item(new QRecord()
+         .withValue("id", 1701)
          .withValue("sku", "ABC-123")
          .withValue("description", null)
          .withValue("quantity", 47)
@@ -88,11 +162,20 @@ class QRecordEntityTest extends BaseTest
       QRecord qRecordOnlyChangedFields = item.toQRecordOnlyChangedFields();
       assertTrue(qRecordOnlyChangedFields.getValues().isEmpty());
 
+      QRecord qRecordOnlyChangedFieldsIncludePKey = item.toQRecordOnlyChangedFields(true);
+      assertEquals(1, qRecordOnlyChangedFieldsIncludePKey.getValues().size());
+      assertEquals(1701, qRecordOnlyChangedFieldsIncludePKey.getValue("id"));
+
       item.setDescription("My Changed Item");
-      qRecordOnlyChangedFields = item.toQRecordOnlyChangedFields();
+      qRecordOnlyChangedFields = item.toQRecordOnlyChangedFields(false);
       assertEquals(1, qRecordOnlyChangedFields.getValues().size());
       assertEquals("My Changed Item", qRecordOnlyChangedFields.getValueString("description"));
 
+      qRecordOnlyChangedFieldsIncludePKey = item.toQRecordOnlyChangedFields(true);
+      assertEquals(2, qRecordOnlyChangedFieldsIncludePKey.getValues().size());
+      assertEquals("My Changed Item", qRecordOnlyChangedFieldsIncludePKey.getValueString("description"));
+      assertEquals(1701, qRecordOnlyChangedFieldsIncludePKey.getValue("id"));
+
       item.setPrice(null);
       qRecordOnlyChangedFields = item.toQRecordOnlyChangedFields();
       assertEquals(2, qRecordOnlyChangedFields.getValues().size());
@@ -101,6 +184,81 @@ class QRecordEntityTest extends BaseTest
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @SuppressWarnings("deprecation")
+   @Test
+   void testItemToQRecordOnlyChangedFieldsFromNewEntity() throws QException
+   {
+      Item item = new Item()
+         .withId(1701)
+         .withSku("ABC-123");
+
+      QRecord qRecordOnlyChangedFields = item.toQRecordOnlyChangedFields();
+      assertEquals(2, qRecordOnlyChangedFields.getValues().size());
+      assertEquals(1701, qRecordOnlyChangedFields.getValue("id"));
+      assertEquals("ABC-123", qRecordOnlyChangedFields.getValue("sku"));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @SuppressWarnings("deprecation")
+   @Test
+   void testItemToQRecordOnlyChangedFieldsWithAssociations() throws QException
+   {
+      Item item = new Item(new QRecord()
+         .withValue("id", 1701)
+         .withValue("sku", "ABC-123")
+         .withAssociatedRecord(Item.ASSOCIATION_ITEM_ALTERNATES_NAME, new Item(new QRecord()
+            .withValue("id", 1702)
+            .withValue("sku", "DEF")
+            .withValue("quantity", 3)
+            .withValue("price", new BigDecimal("3.50"))
+         ).toQRecord())
+      );
+
+      ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      // if no values were changed in the entities, from when they were constructed (from records), then value maps should be empty //
+      ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      QRecord qRecordOnlyChangedFields = item.toQRecordOnlyChangedFields(false);
+      assertTrue(qRecordOnlyChangedFields.getValues().isEmpty());
+      List<QRecord> associatedRecords = qRecordOnlyChangedFields.getAssociatedRecords().get(Item.ASSOCIATION_ITEM_ALTERNATES_NAME);
+      assertTrue(associatedRecords.get(0).getValues().isEmpty());
+
+      ///////////////////////////////////////////////////////
+      // but - if pkeys are requested, confirm we get them //
+      ///////////////////////////////////////////////////////
+      qRecordOnlyChangedFields = item.toQRecordOnlyChangedFields(true);
+      assertEquals(1, qRecordOnlyChangedFields.getValues().size());
+      assertEquals(1701, qRecordOnlyChangedFields.getValue("id"));
+      associatedRecords = qRecordOnlyChangedFields.getAssociatedRecords().get(Item.ASSOCIATION_ITEM_ALTERNATES_NAME);
+      assertEquals(1, associatedRecords.get(0).getValues().size());
+      assertEquals(1702, associatedRecords.get(0).getValue("id"));
+
+      ////////////////////////////////////////////
+      // change some properties in the entities //
+      ////////////////////////////////////////////
+      item.setDescription("My Changed Item");
+      item.getItemAlternates().get(0).setQuantity(4);
+      item.getItemAlternates().get(0).setPrice(null);
+
+      qRecordOnlyChangedFields = item.toQRecordOnlyChangedFields(true);
+      assertEquals(2, qRecordOnlyChangedFields.getValues().size());
+      assertEquals(1701, qRecordOnlyChangedFields.getValue("id"));
+      assertEquals("My Changed Item", qRecordOnlyChangedFields.getValue("description"));
+      associatedRecords = qRecordOnlyChangedFields.getAssociatedRecords().get(Item.ASSOCIATION_ITEM_ALTERNATES_NAME);
+      assertEquals(3, associatedRecords.get(0).getValues().size());
+      assertEquals(1702, associatedRecords.get(0).getValue("id"));
+      assertEquals(4, associatedRecords.get(0).getValue("quantity"));
+      assertNull(associatedRecords.get(0).getValue("price"));
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/data/testentities/Item.java

@@ -23,6 +23,8 @@ package com.kingsrook.qqq.backend.core.model.data.testentities;
 
 
 import java.math.BigDecimal;
+import java.util.List;
+import com.kingsrook.qqq.backend.core.model.data.QAssociation;
 import com.kingsrook.qqq.backend.core.model.data.QField;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
 import com.kingsrook.qqq.backend.core.model.data.QRecordEntity;
@@ -34,6 +36,13 @@ import com.kingsrook.qqq.backend.core.model.metadata.fields.DisplayFormat;
  *******************************************************************************/
 public class Item extends QRecordEntity
 {
+   public static final String TABLE_NAME = "item";
+
+   public static final String ASSOCIATION_ITEM_ALTERNATES_NAME = "itemAlternates";
+
+   @QField(isPrimaryKey = true)
+   private Integer id;
+
    @QField(isRequired = true, label = "SKU")
    private String sku;
 
@@ -49,6 +58,9 @@ public class Item extends QRecordEntity
    @QField(backendName = "is_featured")
    private Boolean featured;
 
+   @QAssociation(name = ASSOCIATION_ITEM_ALTERNATES_NAME)
+   private List<Item> itemAlternates;
+
 
 
    /*******************************************************************************
@@ -179,4 +191,122 @@ public class Item extends QRecordEntity
    {
       this.featured = featured;
    }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for sku
+    *******************************************************************************/
+   public Item withSku(String sku)
+   {
+      this.sku = sku;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for description
+    *******************************************************************************/
+   public Item withDescription(String description)
+   {
+      this.description = description;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for quantity
+    *******************************************************************************/
+   public Item withQuantity(Integer quantity)
+   {
+      this.quantity = quantity;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for price
+    *******************************************************************************/
+   public Item withPrice(BigDecimal price)
+   {
+      this.price = price;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for featured
+    *******************************************************************************/
+   public Item withFeatured(Boolean featured)
+   {
+      this.featured = featured;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for itemAlternates
+    *******************************************************************************/
+   public List<Item> getItemAlternates()
+   {
+      return (this.itemAlternates);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for itemAlternates
+    *******************************************************************************/
+   public void setItemAlternates(List<Item> itemAlternates)
+   {
+      this.itemAlternates = itemAlternates;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for itemAlternates
+    *******************************************************************************/
+   public Item withItemAlternates(List<Item> itemAlternates)
+   {
+      this.itemAlternates = itemAlternates;
+      return (this);
+   }
+
+
+   /*******************************************************************************
+    ** Getter for id
+    *******************************************************************************/
+   public Integer getId()
+   {
+      return (this.id);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for id
+    *******************************************************************************/
+   public void setId(Integer id)
+   {
+      this.id = id;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for id
+    *******************************************************************************/
+   public Item withId(Integer id)
+   {
+      this.id = id;
+      return (this);
+   }
+
+
 }

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/session/QSessionTest.java

@@ -76,7 +76,7 @@ class QSessionTest extends BaseTest
    void testMixedValueTypes()
    {
       QSession session = new QSession().withSecurityKeyValues(Map.of(
-         "storeId", List.of("100", "200", 300)
+         "storeId", List.of("100", "200", 300, "four-hundred")
       ));
 
       for(int i : List.of(100, 200, 300))
@@ -86,6 +86,18 @@ class QSessionTest extends BaseTest
          assertTrue(session.hasSecurityKeyValue("storeId", i, QFieldType.STRING), "Should contain: " + i);
          assertTrue(session.hasSecurityKeyValue("storeId", String.valueOf(i), QFieldType.STRING), "Should contain: " + i);
       }
+
+      ////////////////////////////////////////////////////////////////////////////
+      // next two blocks - used to throw exceptions - now, gracefully be false. //
+      ////////////////////////////////////////////////////////////////////////////
+      int i = 400;
+      assertFalse(session.hasSecurityKeyValue("storeId", i, QFieldType.INTEGER), "Should not contain: " + i);
+      assertFalse(session.hasSecurityKeyValue("storeId", String.valueOf(i), QFieldType.INTEGER), "Should not contain: " + i);
+      assertFalse(session.hasSecurityKeyValue("storeId", i, QFieldType.STRING), "Should not contain: " + i);
+      assertFalse(session.hasSecurityKeyValue("storeId", String.valueOf(i), QFieldType.STRING), "Should not contain: " + i);
+
+      assertFalse(session.hasSecurityKeyValue("storeId", "one-hundred", QFieldType.INTEGER), "Should not contain: " + i);
+      assertFalse(session.hasSecurityKeyValue("storeId", "one-hundred", QFieldType.STRING), "Should not contain: " + i);
    }
 
 

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertFullProcessTest.java

@@ -0,0 +1,252 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert;
+
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
+import java.util.function.Consumer;
+import com.kingsrook.qqq.backend.core.BaseTest;
+import com.kingsrook.qqq.backend.core.actions.processes.RunProcessAction;
+import com.kingsrook.qqq.backend.core.actions.tables.StorageAction;
+import com.kingsrook.qqq.backend.core.context.QContext;
+import com.kingsrook.qqq.backend.core.instances.QInstanceEnricher;
+import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput;
+import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessOutput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.storage.StorageInput;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.frontend.QFrontendFieldMetaData;
+import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryRecordStore;
+import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadProfile;
+import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadProfileField;
+import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess;
+import com.kingsrook.qqq.backend.core.utils.JsonUtils;
+import com.kingsrook.qqq.backend.core.utils.TestUtils;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertNull;
+
+
+/*******************************************************************************
+ ** Unit test for full bulk insert process
+ *******************************************************************************/
+class BulkInsertFullProcessTest extends BaseTest
+{
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @BeforeEach
+   @AfterEach
+   void beforeAndAfterEach()
+   {
+      MemoryRecordStore.getInstance().reset();
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public static String getPersonCsvRow1()
+   {
+      return ("""
+         "0","2021-10-26 14:39:37","2021-10-26 14:39:37","John","Doe","1980-01-01","john@doe.com","Missouri",42
+         """);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public static String getPersonCsvRow2()
+   {
+      return ("""
+         "0","2021-10-26 14:39:37","2021-10-26 14:39:37","Jane","Doe","1981-01-01","john@doe.com","Illinois",
+         """);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public static String getPersonCsvHeaderUsingLabels()
+   {
+      return ("""
+         "Id","Create Date","Modify Date","First Name","Last Name","Birth Date","Email","Home State",noOfShoes
+         """);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void test() throws Exception
+   {
+      String defaultEmail = "noone@kingsrook.com";
+
+      ///////////////////////////////////////
+      // make sure table is empty to start //
+      ///////////////////////////////////////
+      assertThat(TestUtils.queryTable(TestUtils.TABLE_NAME_PERSON_MEMORY)).isEmpty();
+
+      QInstance qInstance   = QContext.getQInstance();
+      String    processName = "PersonBulkInsertV2";
+      new QInstanceEnricher(qInstance).defineTableBulkInsert(qInstance, qInstance.getTable(TestUtils.TABLE_NAME_PERSON_MEMORY), processName);
+
+      /////////////////////////////////////////////////////////
+      // start the process - expect to go to the upload step //
+      /////////////////////////////////////////////////////////
+      RunProcessInput runProcessInput = new RunProcessInput();
+      runProcessInput.setProcessName(processName);
+      runProcessInput.addValue("tableName", TestUtils.TABLE_NAME_PERSON_MEMORY);
+      RunProcessOutput runProcessOutput = new RunProcessAction().execute(runProcessInput);
+      String           processUUID      = runProcessOutput.getProcessUUID();
+      assertThat(runProcessOutput.getProcessState().getNextStepName()).isPresent().get().isEqualTo("upload");
+
+      //////////////////////////////
+      // simulate the file upload //
+      //////////////////////////////
+      String       storageReference = UUID.randomUUID() + ".csv";
+      StorageInput storageInput     = new StorageInput(TestUtils.TABLE_NAME_MEMORY_STORAGE).withReference(storageReference);
+      try(OutputStream outputStream = new StorageAction().createOutputStream(storageInput))
+      {
+         outputStream.write((getPersonCsvHeaderUsingLabels() + getPersonCsvRow1() + getPersonCsvRow2()).getBytes());
+      }
+      catch(IOException e)
+      {
+         throw (e);
+      }
+
+      //////////////////////////
+      // continue post-upload //
+      //////////////////////////
+      runProcessInput.setProcessUUID(processUUID);
+      runProcessInput.setStartAfterStep("upload");
+      runProcessInput.addValue("theFile", new ArrayList<>(List.of(storageInput)));
+      runProcessOutput = new RunProcessAction().execute(runProcessInput);
+      assertEquals(List.of("Id", "Create Date", "Modify Date", "First Name", "Last Name", "Birth Date", "Email", "Home State", "noOfShoes"), runProcessOutput.getValue("headerValues"));
+      assertEquals(List.of("A", "B", "C", "D", "E", "F", "G", "H", "I"), runProcessOutput.getValue("headerLetters"));
+
+      //////////////////////////////////////////////////////
+      // assert about the suggested mapping that was done //
+      //////////////////////////////////////////////////////
+      Serializable bulkLoadProfile = runProcessOutput.getValue("bulkLoadProfile");
+      assertThat(bulkLoadProfile).isInstanceOf(BulkLoadProfile.class);
+      assertThat(((BulkLoadProfile) bulkLoadProfile).getFieldList()).hasSizeGreaterThan(5);
+      assertEquals("firstName", ((BulkLoadProfile) bulkLoadProfile).getFieldList().get(0).getFieldName());
+      assertEquals(3, ((BulkLoadProfile) bulkLoadProfile).getFieldList().get(0).getColumnIndex());
+      assertEquals("lastName", ((BulkLoadProfile) bulkLoadProfile).getFieldList().get(1).getFieldName());
+      assertEquals(4, ((BulkLoadProfile) bulkLoadProfile).getFieldList().get(1).getColumnIndex());
+      assertEquals("birthDate", ((BulkLoadProfile) bulkLoadProfile).getFieldList().get(2).getFieldName());
+      assertEquals(5, ((BulkLoadProfile) bulkLoadProfile).getFieldList().get(2).getColumnIndex());
+
+      assertThat(runProcessOutput.getProcessState().getNextStepName()).isPresent().get().isEqualTo("fileMapping");
+
+      ////////////////////////////////////////////////////////////////////////////////
+      // all subsequent steps will want these data - so set up a lambda to set them //
+      ////////////////////////////////////////////////////////////////////////////////
+      Consumer<RunProcessInput> addProfileToRunProcessInput = (RunProcessInput input) ->
+      {
+         input.addValue("version", "v1");
+         input.addValue("layout", "FLAT");
+         input.addValue("hasHeaderRow", "true");
+         input.addValue("fieldListJSON", JsonUtils.toJson(List.of(
+            new BulkLoadProfileField().withFieldName("firstName").withColumnIndex(3),
+            new BulkLoadProfileField().withFieldName("lastName").withColumnIndex(4),
+            new BulkLoadProfileField().withFieldName("email").withDefaultValue(defaultEmail),
+            new BulkLoadProfileField().withFieldName("homeStateId").withColumnIndex(7).withDoValueMapping(true).withValueMappings(Map.of("Illinois", 1)),
+            new BulkLoadProfileField().withFieldName("noOfShoes").withColumnIndex(8)
+         )));
+      };
+
+      ////////////////////////////////
+      // continue post file-mapping //
+      ////////////////////////////////
+      runProcessInput.setStartAfterStep("fileMapping");
+      addProfileToRunProcessInput.accept(runProcessInput);
+      runProcessOutput = new RunProcessAction().execute(runProcessInput);
+      Serializable valueMappingField = runProcessOutput.getValue("valueMappingField");
+      assertThat(valueMappingField).isInstanceOf(QFrontendFieldMetaData.class);
+      assertEquals("homeStateId", ((QFrontendFieldMetaData) valueMappingField).getName());
+      assertEquals(List.of("Missouri", "Illinois"), runProcessOutput.getValue("fileValues"));
+      assertEquals(List.of("homeStateId"), runProcessOutput.getValue("fieldNamesToDoValueMapping"));
+      assertEquals(Map.of(1, "IL"), runProcessOutput.getValue("mappedValueLabels"));
+      assertEquals(0, runProcessOutput.getValue("valueMappingFieldIndex"));
+      assertThat(runProcessOutput.getProcessState().getNextStepName()).isPresent().get().isEqualTo("valueMapping");
+
+      /////////////////////////////////
+      // continue post value-mapping //
+      /////////////////////////////////
+      runProcessInput.setStartAfterStep("valueMapping");
+      runProcessInput.addValue("mappedValuesJSON", JsonUtils.toJson(Map.of("Illinois", 1, "Missouri", 2)));
+      addProfileToRunProcessInput.accept(runProcessInput);
+      runProcessOutput = new RunProcessAction().execute(runProcessInput);
+      assertThat(runProcessOutput.getProcessState().getNextStepName()).isPresent().get().isEqualTo("review");
+
+      /////////////////////////////////
+      // continue post review screen //
+      /////////////////////////////////
+      runProcessInput.setStartAfterStep("review");
+      addProfileToRunProcessInput.accept(runProcessInput);
+      runProcessOutput = new RunProcessAction().execute(runProcessInput);
+      assertThat(runProcessOutput.getRecords()).hasSize(2);
+      assertThat(runProcessOutput.getProcessState().getNextStepName()).isPresent().get().isEqualTo("result");
+      assertThat(runProcessOutput.getValues().get(StreamedETLWithFrontendProcess.FIELD_PROCESS_SUMMARY)).isNotNull().isInstanceOf(List.class);
+      assertThat(runProcessOutput.getException()).isEmpty();
+
+      ////////////////////////////////////
+      // query for the inserted records //
+      ////////////////////////////////////
+      List<QRecord> records = TestUtils.queryTable(TestUtils.TABLE_NAME_PERSON_MEMORY);
+
+      assertEquals("John", records.get(0).getValueString("firstName"));
+      assertEquals("Jane", records.get(1).getValueString("firstName"));
+
+      assertNotNull(records.get(0).getValue("id"));
+      assertNotNull(records.get(1).getValue("id"));
+
+      assertEquals(2, records.get(0).getValueInteger("homeStateId"));
+      assertEquals(1, records.get(1).getValueInteger("homeStateId"));
+
+      assertEquals(defaultEmail, records.get(0).getValueString("email"));
+      assertEquals(defaultEmail, records.get(1).getValueString("email"));
+
+      assertEquals(42, records.get(0).getValueInteger("noOfShoes"));
+      assertNull(records.get(1).getValue("noOfShoes"));
+   }
+
+}

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertPrepareFileMappingStepTest.java

@@ -0,0 +1,69 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert;
+
+
+import com.kingsrook.qqq.backend.core.BaseTest;
+import org.junit.jupiter.api.Test;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+
+/*******************************************************************************
+ ** Unit test for BulkInsertPrepareMappingStep 
+ *******************************************************************************/
+class BulkInsertPrepareFileMappingStepTest extends BaseTest
+{
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @SuppressWarnings("PointlessArithmeticExpression")
+   @Test
+   void testToHeaderLetter()
+   {
+      assertEquals("A", BulkInsertPrepareFileMappingStep.toHeaderLetter(0));
+      assertEquals("B", BulkInsertPrepareFileMappingStep.toHeaderLetter(1));
+      assertEquals("Z", BulkInsertPrepareFileMappingStep.toHeaderLetter(25));
+
+      assertEquals("AA", BulkInsertPrepareFileMappingStep.toHeaderLetter(26 + 0));
+      assertEquals("AB", BulkInsertPrepareFileMappingStep.toHeaderLetter(26 + 1));
+      assertEquals("AZ", BulkInsertPrepareFileMappingStep.toHeaderLetter(26 + 25));
+
+      assertEquals("BA", BulkInsertPrepareFileMappingStep.toHeaderLetter(2 * 26 + 0));
+      assertEquals("BB", BulkInsertPrepareFileMappingStep.toHeaderLetter(2 * 26 + 1));
+      assertEquals("BZ", BulkInsertPrepareFileMappingStep.toHeaderLetter(2 * 26 + 25));
+
+      assertEquals("ZA", BulkInsertPrepareFileMappingStep.toHeaderLetter(26 * 26 + 0));
+      assertEquals("ZB", BulkInsertPrepareFileMappingStep.toHeaderLetter(26 * 26 + 1));
+      assertEquals("ZZ", BulkInsertPrepareFileMappingStep.toHeaderLetter(26 * 26 + 25));
+
+      assertEquals("AAA", BulkInsertPrepareFileMappingStep.toHeaderLetter(27 * 26 + 0));
+      assertEquals("AAB", BulkInsertPrepareFileMappingStep.toHeaderLetter(27 * 26 + 1));
+      assertEquals("AAC", BulkInsertPrepareFileMappingStep.toHeaderLetter(27 * 26 + 2));
+
+      assertEquals("ABA", BulkInsertPrepareFileMappingStep.toHeaderLetter(28 * 26 + 0));
+      assertEquals("ABB", BulkInsertPrepareFileMappingStep.toHeaderLetter(28 * 26 + 1));
+
+      assertEquals("BAA", BulkInsertPrepareFileMappingStep.toHeaderLetter(2 * 26 * 26 + 26 + 0));
+   }
+
+}

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertPrepareValueMappingStepTest.java

@@ -0,0 +1,55 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert;
+
+
+import com.kingsrook.qqq.backend.core.BaseTest;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.utils.TestUtils;
+import org.junit.jupiter.api.Test;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+
+/*******************************************************************************
+ ** Unit test for BulkInsertPrepareValueMappingStep 
+ *******************************************************************************/
+class BulkInsertPrepareValueMappingStepTest extends BaseTest
+{
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void test() throws QException
+   {
+      assertEquals(TestUtils.TABLE_NAME_ORDER, BulkInsertPrepareValueMappingStep.getTableAndField(TestUtils.TABLE_NAME_ORDER, "orderNo").table().getName());
+      assertEquals("orderNo", BulkInsertPrepareValueMappingStep.getTableAndField(TestUtils.TABLE_NAME_ORDER, "orderNo").field().getName());
+
+      assertEquals(TestUtils.TABLE_NAME_LINE_ITEM, BulkInsertPrepareValueMappingStep.getTableAndField(TestUtils.TABLE_NAME_ORDER, "orderLine.sku").table().getName());
+      assertEquals("sku", BulkInsertPrepareValueMappingStep.getTableAndField(TestUtils.TABLE_NAME_ORDER, "orderLine.sku").field().getName());
+
+      assertEquals(TestUtils.TABLE_NAME_LINE_ITEM_EXTRINSIC, BulkInsertPrepareValueMappingStep.getTableAndField(TestUtils.TABLE_NAME_ORDER, "orderLine.extrinsics.key").table().getName());
+      assertEquals("key", BulkInsertPrepareValueMappingStep.getTableAndField(TestUtils.TABLE_NAME_ORDER, "orderLine.extrinsics.key").field().getName());
+
+   }
+
+}

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTest.java

@@ -1,157 +0,0 @@
-/*
- * QQQ - Low-code Application Framework for Engineers.
- * Copyright (C) 2021-2022.  Kingsrook, LLC
- * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
- * contact@kingsrook.com
- * https://github.com/Kingsrook/
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program.  If not, see <https://www.gnu.org/licenses/>.
- */
-
-package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert;
-
-
-import java.util.List;
-import com.kingsrook.qqq.backend.core.BaseTest;
-import com.kingsrook.qqq.backend.core.actions.processes.RunProcessAction;
-import com.kingsrook.qqq.backend.core.exceptions.QException;
-import com.kingsrook.qqq.backend.core.model.actions.processes.QUploadedFile;
-import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput;
-import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessOutput;
-import com.kingsrook.qqq.backend.core.model.data.QRecord;
-import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryRecordStore;
-import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess;
-import com.kingsrook.qqq.backend.core.state.StateType;
-import com.kingsrook.qqq.backend.core.state.TempFileStateProvider;
-import com.kingsrook.qqq.backend.core.state.UUIDAndTypeStateKey;
-import com.kingsrook.qqq.backend.core.utils.TestUtils;
-import org.junit.jupiter.api.AfterEach;
-import org.junit.jupiter.api.BeforeEach;
-import org.junit.jupiter.api.Test;
-import static org.assertj.core.api.Assertions.assertThat;
-import static org.junit.jupiter.api.Assertions.assertEquals;
-import static org.junit.jupiter.api.Assertions.assertNotNull;
-
-
-/*******************************************************************************
- ** Unit test for full bulk insert process
- *******************************************************************************/
-class BulkInsertTest extends BaseTest
-{
-
-   /*******************************************************************************
-    **
-    *******************************************************************************/
-   @BeforeEach
-   @AfterEach
-   void beforeAndAfterEach()
-   {
-      MemoryRecordStore.getInstance().reset();
-   }
-
-
-
-   /*******************************************************************************
-    **
-    *******************************************************************************/
-   public static String getPersonCsvRow1()
-   {
-      return ("""
-         "0","2021-10-26 14:39:37","2021-10-26 14:39:37","John","Doe","1980-01-01","john@doe.com"
-         """);
-   }
-
-
-
-   /*******************************************************************************
-    **
-    *******************************************************************************/
-   public static String getPersonCsvRow2()
-   {
-      return ("""
-         "0","2021-10-26 14:39:37","2021-10-26 14:39:37","Jane","Doe","1981-01-01","john@doe.com"
-         """);
-   }
-
-
-
-   /*******************************************************************************
-    **
-    *******************************************************************************/
-   public static String getPersonCsvHeaderUsingLabels()
-   {
-      return ("""
-         "Id","Create Date","Modify Date","First Name","Last Name","Birth Date","Email"
-         """);
-   }
-
-
-
-   /*******************************************************************************
-    **
-    *******************************************************************************/
-   @Test
-   void test() throws QException
-   {
-      ///////////////////////////////////////
-      // make sure table is empty to start //
-      ///////////////////////////////////////
-      assertThat(TestUtils.queryTable(TestUtils.TABLE_NAME_PERSON_MEMORY)).isEmpty();
-
-      ////////////////////////////////////////////////////////////////
-      // create an uploaded file, similar to how an http server may //
-      ////////////////////////////////////////////////////////////////
-      QUploadedFile qUploadedFile = new QUploadedFile();
-      qUploadedFile.setBytes((getPersonCsvHeaderUsingLabels() + getPersonCsvRow1() + getPersonCsvRow2()).getBytes());
-      qUploadedFile.setFilename("test.csv");
-      UUIDAndTypeStateKey uploadedFileKey = new UUIDAndTypeStateKey(StateType.UPLOADED_FILE);
-      TempFileStateProvider.getInstance().put(uploadedFileKey, qUploadedFile);
-
-      RunProcessInput runProcessInput = new RunProcessInput();
-      runProcessInput.setProcessName(TestUtils.TABLE_NAME_PERSON_MEMORY + ".bulkInsert");
-      RunProcessOutput runProcessOutput = new RunProcessAction().execute(runProcessInput);
-      String           processUUID      = runProcessOutput.getProcessUUID();
-
-      runProcessInput.setProcessUUID(processUUID);
-      runProcessInput.setStartAfterStep("upload");
-      runProcessInput.addValue(QUploadedFile.DEFAULT_UPLOADED_FILE_FIELD_NAME, uploadedFileKey);
-      runProcessOutput = new RunProcessAction().execute(runProcessInput);
-      assertThat(runProcessOutput.getRecords()).hasSize(2);
-      assertThat(runProcessOutput.getProcessState().getNextStepName()).isPresent().get().isEqualTo("review");
-
-      runProcessInput.addValue(StreamedETLWithFrontendProcess.FIELD_DO_FULL_VALIDATION, true);
-      runProcessInput.setStartAfterStep("review");
-      runProcessOutput = new RunProcessAction().execute(runProcessInput);
-      assertThat(runProcessOutput.getRecords()).hasSize(2);
-      assertThat(runProcessOutput.getProcessState().getNextStepName()).isPresent().get().isEqualTo("review");
-      assertThat(runProcessOutput.getValues().get(StreamedETLWithFrontendProcess.FIELD_VALIDATION_SUMMARY)).isNotNull().isInstanceOf(List.class);
-
-      runProcessInput.setStartAfterStep("review");
-      runProcessOutput = new RunProcessAction().execute(runProcessInput);
-      assertThat(runProcessOutput.getRecords()).hasSize(2);
-      assertThat(runProcessOutput.getProcessState().getNextStepName()).isPresent().get().isEqualTo("result");
-      assertThat(runProcessOutput.getValues().get(StreamedETLWithFrontendProcess.FIELD_PROCESS_SUMMARY)).isNotNull().isInstanceOf(List.class);
-      assertThat(runProcessOutput.getException()).isEmpty();
-
-      ////////////////////////////////////
-      // query for the inserted records //
-      ////////////////////////////////////
-      List<QRecord> records = TestUtils.queryTable(TestUtils.TABLE_NAME_PERSON_MEMORY);
-      assertEquals("John", records.get(0).getValueString("firstName"));
-      assertEquals("Jane", records.get(1).getValueString("firstName"));
-      assertNotNull(records.get(0).getValue("id"));
-      assertNotNull(records.get(1).getValue("id"));
-   }
-
-}

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTransformStepTest.java

@@ -22,10 +22,14 @@
 package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert;
 
 
+import java.io.Serializable;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.stream.Collectors;
 import com.kingsrook.qqq.backend.core.BaseTest;
+import com.kingsrook.qqq.backend.core.context.QContext;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryAssert;
 import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLine;
 import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLineInterface;
 import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
@@ -37,9 +41,14 @@ import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.UniqueKey;
+import com.kingsrook.qqq.backend.core.model.statusmessages.BadInputStatusMessage;
 import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryRecordStore;
+import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping.BulkLoadRecordUtils;
+import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping.BulkLoadValueTypeError;
+import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow;
 import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess;
 import com.kingsrook.qqq.backend.core.utils.TestUtils;
+import com.kingsrook.qqq.backend.core.utils.collections.ListBuilder;
 import org.junit.jupiter.api.AfterEach;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
@@ -87,9 +96,9 @@ class BulkInsertTransformStepTest extends BaseTest
       // insert some records that will cause some UK violations //
       ////////////////////////////////////////////////////////////
       TestUtils.insertRecords(table, List.of(
-         newQRecord("uuid-A", "SKU-1", 1),
-         newQRecord("uuid-B", "SKU-2", 1),
-         newQRecord("uuid-C", "SKU-2", 2)
+         newUkTestQRecord("uuid-A", "SKU-1", 1),
+         newUkTestQRecord("uuid-B", "SKU-2", 1),
+         newUkTestQRecord("uuid-C", "SKU-2", 2)
       ));
 
       ///////////////////////////////////////////
@@ -102,13 +111,13 @@ class BulkInsertTransformStepTest extends BaseTest
       input.setTableName(TABLE_NAME);
       input.setStepName(StreamedETLWithFrontendProcess.STEP_NAME_VALIDATE);
       input.setRecords(List.of(
-         newQRecord("uuid-1", "SKU-A", 1), // OK.
-         newQRecord("uuid-1", "SKU-B", 1), // violate uuid UK in this set
-         newQRecord("uuid-2", "SKU-C", 1), // OK.
-         newQRecord("uuid-3", "SKU-C", 2), // OK.
-         newQRecord("uuid-4", "SKU-C", 1), // violate sku/storeId UK in this set
-         newQRecord("uuid-A", "SKU-X", 1), // violate uuid UK from pre-existing records
-         newQRecord("uuid-D", "SKU-2", 1)  // violate sku/storeId UK from pre-existing records
+         newUkTestQRecord("uuid-1", "SKU-A", 1), // OK.
+         newUkTestQRecord("uuid-1", "SKU-B", 1), // violate uuid UK in this set
+         newUkTestQRecord("uuid-2", "SKU-C", 1), // OK.
+         newUkTestQRecord("uuid-3", "SKU-C", 2), // OK.
+         newUkTestQRecord("uuid-4", "SKU-C", 1), // violate sku/storeId UK in this set
+         newUkTestQRecord("uuid-A", "SKU-X", 1), // violate uuid UK from pre-existing records
+         newUkTestQRecord("uuid-D", "SKU-2", 1)  // violate sku/storeId UK from pre-existing records
       ));
       bulkInsertTransformStep.preRun(input, output);
       bulkInsertTransformStep.runOnePage(input, output);
@@ -171,9 +180,9 @@ class BulkInsertTransformStepTest extends BaseTest
       // insert some records that will cause some UK violations //
       ////////////////////////////////////////////////////////////
       TestUtils.insertRecords(table, List.of(
-         newQRecord("uuid-A", "SKU-1", 1),
-         newQRecord("uuid-B", "SKU-2", 1),
-         newQRecord("uuid-C", "SKU-2", 2)
+         newUkTestQRecord("uuid-A", "SKU-1", 1),
+         newUkTestQRecord("uuid-B", "SKU-2", 1),
+         newUkTestQRecord("uuid-C", "SKU-2", 2)
       ));
 
       ///////////////////////////////////////////
@@ -186,20 +195,20 @@ class BulkInsertTransformStepTest extends BaseTest
       input.setTableName(TABLE_NAME);
       input.setStepName(StreamedETLWithFrontendProcess.STEP_NAME_VALIDATE);
       input.setRecords(List.of(
-         newQRecord("uuid-1", "SKU-A", 1), // OK.
-         newQRecord("uuid-1", "SKU-B", 1), // violate uuid UK in this set
-         newQRecord("uuid-2", "SKU-C", 1), // OK.
-         newQRecord("uuid-3", "SKU-C", 2), // OK.
-         newQRecord("uuid-4", "SKU-C", 1), // violate sku/storeId UK in this set
-         newQRecord("uuid-A", "SKU-X", 1), // violate uuid UK from pre-existing records
-         newQRecord("uuid-D", "SKU-2", 1)  // violate sku/storeId UK from pre-existing records
+         newUkTestQRecord("uuid-1", "SKU-A", 1), // OK.
+         newUkTestQRecord("uuid-1", "SKU-B", 1), // violate uuid UK in this set
+         newUkTestQRecord("uuid-2", "SKU-C", 1), // OK.
+         newUkTestQRecord("uuid-3", "SKU-C", 2), // OK.
+         newUkTestQRecord("uuid-4", "SKU-C", 1), // violate sku/storeId UK in this set
+         newUkTestQRecord("uuid-A", "SKU-X", 1), // violate uuid UK from pre-existing records
+         newUkTestQRecord("uuid-D", "SKU-2", 1)  // violate sku/storeId UK from pre-existing records
       ));
       bulkInsertTransformStep.preRun(input, output);
       bulkInsertTransformStep.runOnePage(input, output);
 
-      ///////////////////////////////////////////////////////
-      // assert that all records pass.
-      ///////////////////////////////////////////////////////
+      ///////////////////////////////////
+      // assert that all records pass. //
+      ///////////////////////////////////
       assertEquals(7, output.getRecords().size());
    }
 
@@ -211,8 +220,8 @@ class BulkInsertTransformStepTest extends BaseTest
    private boolean recordEquals(QRecord record, String uuid, String sku, Integer storeId)
    {
       return (record.getValue("uuid").equals(uuid)
-         && record.getValue("sku").equals(sku)
-         && record.getValue("storeId").equals(storeId));
+              && record.getValue("sku").equals(sku)
+              && record.getValue("storeId").equals(storeId));
    }
 
 
@@ -220,7 +229,7 @@ class BulkInsertTransformStepTest extends BaseTest
    /*******************************************************************************
     **
     *******************************************************************************/
-   private QRecord newQRecord(String uuid, String sku, int storeId)
+   private QRecord newUkTestQRecord(String uuid, String sku, int storeId)
    {
       return new QRecord()
          .withValue("uuid", uuid)
@@ -229,4 +238,191 @@ class BulkInsertTransformStepTest extends BaseTest
          .withValue("name", "Some Item");
    }
 
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testValueMappingTypeErrors() throws QException
+   {
+      ///////////////////////////////////////////
+      // setup & run the bulk insert transform //
+      ///////////////////////////////////////////
+      BulkInsertTransformStep bulkInsertTransformStep = new BulkInsertTransformStep();
+      RunBackendStepInput     input                   = new RunBackendStepInput();
+      RunBackendStepOutput    output                  = new RunBackendStepOutput();
+      Serializable[]          emptyValues             = new Serializable[0];
+
+      input.setTableName(TestUtils.TABLE_NAME_ORDER);
+      input.setStepName(StreamedETLWithFrontendProcess.STEP_NAME_VALIDATE);
+      input.setRecords(ListBuilder.of(
+         BulkLoadRecordUtils.addBackendDetailsAboutFileRows(new QRecord(), new BulkLoadFileRow(emptyValues, 1))
+            .withError(new BulkLoadValueTypeError("storeId", "A", QFieldType.INTEGER, "Store"))
+            .withError(new BulkLoadValueTypeError("orderDate", "47", QFieldType.DATE, "Order Date")),
+         BulkLoadRecordUtils.addBackendDetailsAboutFileRows(new QRecord(), new BulkLoadFileRow(emptyValues, 2))
+            .withError(new BulkLoadValueTypeError("storeId", "BCD", QFieldType.INTEGER, "Store"))
+      ));
+
+      ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      // add 102 records with an error in the total field - which is more than the number of examples that should be given //
+      ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      for(int i = 0; i < 102; i++)
+      {
+         input.getRecords().add(BulkLoadRecordUtils.addBackendDetailsAboutFileRows(new QRecord(), new BulkLoadFileRow(emptyValues, 3 + i))
+            .withError(new BulkLoadValueTypeError("total", "three-fifty-" + i, QFieldType.DECIMAL, "Total")));
+      }
+
+      bulkInsertTransformStep.preRun(input, output);
+      bulkInsertTransformStep.runOnePage(input, output);
+      ArrayList<ProcessSummaryLineInterface> processSummary = bulkInsertTransformStep.getProcessSummary(output, false);
+
+      ProcessSummaryAssert.assertThat(processSummary)
+         .hasLineWithMessageContaining("Cannot convert value for field [Store] to type [Integer]")
+         .hasMessageContaining("Values:")
+         .doesNotHaveMessageContaining("Example Values:")
+         .hasAnyBulletsOfTextContaining("Row 1 [A]")
+         .hasAnyBulletsOfTextContaining("Row 2 [BCD]")
+         .hasStatus(Status.ERROR)
+         .hasCount(2);
+
+      ProcessSummaryAssert.assertThat(processSummary)
+         .hasLineWithMessageContaining("Cannot convert value for field [Order Date] to type [Date]")
+         .hasMessageContaining("Values:")
+         .doesNotHaveMessageContaining("Example Values:")
+         .hasAnyBulletsOfTextContaining("Row 1 [47]")
+         .hasStatus(Status.ERROR)
+         .hasCount(1);
+
+      ProcessSummaryAssert.assertThat(processSummary)
+         .hasLineWithMessageContaining("Cannot convert value for field [Total] to type [Decimal]")
+         .hasMessageContaining("Example Values:")
+         .hasAnyBulletsOfTextContaining("Row 3 [three-fifty-0]")
+         .hasAnyBulletsOfTextContaining("Row 4 [three-fifty-1]")
+         .hasAnyBulletsOfTextContaining("Row 5 [three-fifty-2]")
+         .doesNotHaveAnyBulletsOfTextContaining("three-fifty-101")
+         .hasStatus(Status.ERROR)
+         .hasCount(102);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testRollupOfValidationErrors() throws QException
+   {
+      QContext.getQSession().withSecurityKeyValue(TestUtils.SECURITY_KEY_TYPE_STORE, 1);
+
+      ///////////////////////////////////////////
+      // setup & run the bulk insert transform //
+      ///////////////////////////////////////////
+      BulkInsertTransformStep bulkInsertTransformStep = new BulkInsertTransformStep();
+      RunBackendStepInput     input                   = new RunBackendStepInput();
+      RunBackendStepOutput    output                  = new RunBackendStepOutput();
+      Serializable[]          emptyValues             = new Serializable[0];
+
+      String tooLong = ".".repeat(201);
+
+      input.setTableName(TestUtils.TABLE_NAME_ORDER);
+      input.setStepName(StreamedETLWithFrontendProcess.STEP_NAME_VALIDATE);
+      input.setRecords(ListBuilder.of(
+         BulkLoadRecordUtils.addBackendDetailsAboutFileRows(new QRecord().withValue("shipToName", tooLong), new BulkLoadFileRow(emptyValues, 1)),
+         BulkLoadRecordUtils.addBackendDetailsAboutFileRows(new QRecord().withValue("shipToName", "OK").withValue("storeId", 1), new BulkLoadFileRow(emptyValues, 2))
+      ));
+
+      /////////////////////////////////////////////////////////////////////
+      // add 102 records with no security key - which should be an error //
+      /////////////////////////////////////////////////////////////////////
+      for(int i = 0; i < 102; i++)
+      {
+         input.getRecords().add(BulkLoadRecordUtils.addBackendDetailsAboutFileRows(new QRecord(), new BulkLoadFileRow(emptyValues, 3 + i)));
+      }
+
+      bulkInsertTransformStep.preRun(input, output);
+      bulkInsertTransformStep.runOnePage(input, output);
+      ArrayList<ProcessSummaryLineInterface> processSummary = bulkInsertTransformStep.getProcessSummary(output, false);
+
+      ProcessSummaryAssert.assertThat(processSummary)
+         .hasLineWithMessageContaining("value for Ship To Name is too long")
+         .hasMessageContaining("Records:")
+         .doesNotHaveMessageContaining("Example Records:")
+         .hasAnyBulletsOfTextContaining("Row 1")
+         .hasStatus(Status.ERROR)
+         .hasCount(1);
+
+      ProcessSummaryAssert.assertThat(processSummary)
+         .hasLineWithMessageContaining("without a value in the field: Store Id")
+         .hasMessageContaining("Example Records:")
+         .hasAnyBulletsOfTextContaining("Row 1")
+         .hasAnyBulletsOfTextContaining("Row 3")
+         .hasAnyBulletsOfTextContaining("Row 4")
+         .doesNotHaveAnyBulletsOfTextContaining("Row 101")
+         .hasStatus(Status.ERROR)
+         .hasCount(103); // the 102, plus row 1.
+
+      ProcessSummaryAssert.assertThat(processSummary)
+         .hasLineWithMessageContaining("Order record will be inserted")
+         .hasStatus(Status.OK)
+         .hasCount(1);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testPropagationOfErrorsFromAssociations() throws QException
+   {
+      ////////////////////////////////////////////////
+      // set line item lineNumber field as required //
+      ////////////////////////////////////////////////
+      QInstance instance = TestUtils.defineInstance();
+      instance.getTable(TestUtils.TABLE_NAME_LINE_ITEM).getField("lineNumber").setIsRequired(true);
+      reInitInstanceInContext(instance);
+
+      QContext.getQSession().withSecurityKeyValue(TestUtils.SECURITY_KEY_TYPE_STORE, 1);
+
+      ///////////////////////////////////////////
+      // setup & run the bulk insert transform //
+      ///////////////////////////////////////////
+      BulkInsertTransformStep bulkInsertTransformStep = new BulkInsertTransformStep();
+      RunBackendStepInput     input                   = new RunBackendStepInput();
+      RunBackendStepOutput    output                  = new RunBackendStepOutput();
+
+      input.setTableName(TestUtils.TABLE_NAME_ORDER);
+      input.setStepName(StreamedETLWithFrontendProcess.STEP_NAME_VALIDATE);
+      input.setRecords(ListBuilder.of(
+         new QRecord().withValue("storeId", 1).withAssociatedRecord("orderLine", new QRecord()),
+         new QRecord().withValue("storeId", 1).withAssociatedRecord("orderLine", new QRecord().withError(new BadInputStatusMessage("some mapping error")))
+      ));
+
+      bulkInsertTransformStep.preRun(input, output);
+      bulkInsertTransformStep.runOnePage(input, output);
+      ArrayList<ProcessSummaryLineInterface> processSummary = bulkInsertTransformStep.getProcessSummary(output, false);
+
+      ProcessSummaryAssert.assertThat(processSummary)
+         .hasLineWithMessageContaining("some mapping error")
+         .hasMessageContaining("Records:")
+         .hasStatus(Status.ERROR)
+         .hasCount(1);
+
+      ProcessSummaryAssert.assertThat(processSummary)
+         .hasLineWithMessageContaining("records were processed from the file")
+         .hasStatus(Status.INFO)
+         .hasCount(2);
+
+      ProcessSummaryAssert.assertThat(processSummary)
+         .hasLineWithMessageContaining("Order record will be inserted")
+         .hasStatus(Status.OK)
+         .hasCount(1);
+
+      ProcessSummaryAssert.assertThat(processSummary)
+         .hasLineWithMessageContaining("Order Line record will be inserted")
+         .hasStatus(Status.OK)
+         .hasCount(1);
+   }
+
 }

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/CsvFileToRowsTest.java

@@ -0,0 +1,60 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling;
+
+
+import java.io.ByteArrayInputStream;
+import com.kingsrook.qqq.backend.core.BaseTest;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow;
+import org.junit.jupiter.api.Test;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+
+
+/*******************************************************************************
+ ** Unit test for CsvFileToRows 
+ *******************************************************************************/
+class CsvFileToRowsTest extends BaseTest
+{
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void test() throws QException
+   {
+      byte[] csvBytes = """
+         one,two,three
+         1,2,3,4
+         """.getBytes();
+      FileToRowsInterface fileToRowsInterface = FileToRowsInterface.forFile("someFile.csv", new ByteArrayInputStream(csvBytes));
+
+      BulkLoadFileRow headerRow = fileToRowsInterface.next();
+      BulkLoadFileRow bodyRow   = fileToRowsInterface.next();
+
+      assertEquals(new BulkLoadFileRow(new String[] { "one", "two", "three" }, 1), headerRow);
+      assertEquals(new BulkLoadFileRow(new String[] { "1", "2", "3", "4" }, 2), bodyRow);
+      assertFalse(fileToRowsInterface.hasNext());
+   }
+
+}

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/TestFileToRows.java

@@ -0,0 +1,86 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling;
+
+
+import java.io.InputStream;
+import java.io.Serializable;
+import java.util.List;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow;
+
+
+/***************************************************************************
+ **
+ ***************************************************************************/
+public class TestFileToRows extends AbstractIteratorBasedFileToRows<Serializable[]> implements FileToRowsInterface
+{
+   private final List<Serializable[]> rows;
+
+
+
+   /*******************************************************************************
+    ** Constructor
+    **
+    *******************************************************************************/
+   public TestFileToRows(List<Serializable[]> rows)
+   {
+      this.rows = rows;
+      setIterator(this.rows.iterator());
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   @Override
+   public void init(InputStream inputStream) throws QException
+   {
+      ///////////
+      // noop! //
+      ///////////
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   @Override
+   public void close() throws Exception
+   {
+      ///////////
+      // noop! //
+      ///////////
+   }
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   @Override
+   public BulkLoadFileRow makeRow(Serializable[] values)
+   {
+      return (new BulkLoadFileRow(values, getRowNo()));
+   }
+}

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/XlsxFileToRowsTest.java

@@ -0,0 +1,161 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling;
+
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.Serializable;
+import java.time.LocalDate;
+import java.time.Month;
+import java.util.Map;
+import com.kingsrook.qqq.backend.core.BaseTest;
+import com.kingsrook.qqq.backend.core.actions.reporting.GenerateReportAction;
+import com.kingsrook.qqq.backend.core.actions.reporting.GenerateReportActionTest;
+import com.kingsrook.qqq.backend.core.actions.reporting.excel.fastexcel.ExcelFastexcelExportStreamer;
+import com.kingsrook.qqq.backend.core.context.QContext;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportDestination;
+import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportFormat;
+import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportInput;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow;
+import org.junit.jupiter.api.Test;
+import static com.kingsrook.qqq.backend.core.actions.reporting.GenerateReportActionTest.REPORT_NAME;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+
+/*******************************************************************************
+ ** Unit test for XlsxFileToRows 
+ *******************************************************************************/
+class XlsxFileToRowsTest extends BaseTest
+{
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void test() throws QException, IOException
+   {
+      byte[] byteArray = writeExcelBytes();
+
+      FileToRowsInterface fileToRowsInterface = new XlsxFileToRows();
+      fileToRowsInterface.init(new ByteArrayInputStream(byteArray));
+
+      BulkLoadFileRow headerRow = fileToRowsInterface.next();
+      BulkLoadFileRow bodyRow   = fileToRowsInterface.next();
+
+      assertEquals(new BulkLoadFileRow(new String[] { "Id", "First Name", "Last Name", "Birth Date" }, 1), headerRow);
+      assertEquals(new BulkLoadFileRow(new Serializable[] { 1, "Darin", "Jonson", LocalDate.of(1980, Month.JANUARY, 31) }, 2), bodyRow);
+
+      ///////////////////////////////////////////////////////////////////////////////////////
+      // make sure there's at least a limit (less than 20) to how many more rows there are //
+      ///////////////////////////////////////////////////////////////////////////////////////
+      int otherRowCount = 0;
+      while(fileToRowsInterface.hasNext() && otherRowCount < 20)
+      {
+         fileToRowsInterface.next();
+         otherRowCount++;
+      }
+      assertFalse(fileToRowsInterface.hasNext());
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   private static byte[] writeExcelBytes() throws QException, IOException
+   {
+      ReportFormat          format = ReportFormat.XLSX;
+      ByteArrayOutputStream baos   = new ByteArrayOutputStream();
+
+      QInstance qInstance = QContext.getQInstance();
+      qInstance.addReport(GenerateReportActionTest.defineTableOnlyReport());
+      GenerateReportActionTest.insertPersonRecords(qInstance);
+
+      ReportInput reportInput = new ReportInput();
+      reportInput.setReportName(REPORT_NAME);
+      reportInput.setReportDestination(new ReportDestination().withReportFormat(format).withReportOutputStream(baos));
+      reportInput.setInputValues(Map.of("startDate", LocalDate.of(1970, Month.MAY, 15), "endDate", LocalDate.now()));
+      reportInput.setOverrideExportStreamerSupplier(ExcelFastexcelExportStreamer::new);
+      new GenerateReportAction().execute(reportInput);
+
+      byte[] byteArray = baos.toByteArray();
+      // FileUtils.writeByteArrayToFile(new File("/tmp/xlsx.xlsx"), byteArray);
+      return byteArray;
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testDateTimeFormats()
+   {
+      assertFormatDateAndOrDateTime(true, false, "dddd, m/d/yy at h:mm");
+      assertFormatDateAndOrDateTime(true, false, "h PM, ddd mmm dd");
+      assertFormatDateAndOrDateTime(true, false, "dd/mm/yyyy hh:mm");
+      assertFormatDateAndOrDateTime(true, false, "yyyy-mm-dd hh:mm:ss.000");
+      assertFormatDateAndOrDateTime(true, false, "hh:mm dd/mm/yyyy");
+
+      assertFormatDateAndOrDateTime(false, true, "yyyy-mm-dd");
+      assertFormatDateAndOrDateTime(false, true, "mmmm d \\[dddd\\]");
+      assertFormatDateAndOrDateTime(false, true, "mmm dd, yyyy");
+      assertFormatDateAndOrDateTime(false, true, "d-mmm");
+      assertFormatDateAndOrDateTime(false, true, "dd.mm.yyyy");
+
+      assertFormatDateAndOrDateTime(false, false, "yyyy");
+      assertFormatDateAndOrDateTime(false, false, "mmm-yyyy");
+      assertFormatDateAndOrDateTime(false, false, "hh");
+      assertFormatDateAndOrDateTime(false, false, "hh:mm");
+   }
+
+
+
+   /***************************************************************************
+    *
+    ***************************************************************************/
+   private void assertFormatDateAndOrDateTime(boolean expectDateTime, boolean expectDate, String format)
+   {
+      if(XlsxFileToRows.isDateTimeFormat(format))
+      {
+         assertTrue(expectDateTime, format + " was considered a dateTime, but wasn't expected to.");
+         assertFalse(expectDate, format + " was considered a dateTime, but was expected to be a date.");
+      }
+      else if(XlsxFileToRows.isDateFormat(format))
+      {
+         assertFalse(expectDateTime, format + " was considered a date, but was expected to be a dateTime.");
+         assertTrue(expectDate, format + " was considered a date, but was expected to.");
+      }
+      else
+      {
+         assertFalse(expectDateTime, format + " was not considered a dateTime, but was expected to.");
+         assertFalse(expectDate, format + " was considered a date, but was expected to.");
+      }
+   }
+
+}

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadMappingSuggesterTest.java

@@ -0,0 +1,209 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping;
+
+
+import java.util.List;
+import com.kingsrook.qqq.backend.core.BaseTest;
+import com.kingsrook.qqq.backend.core.context.QContext;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadProfile;
+import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadProfileField;
+import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadTableStructure;
+import com.kingsrook.qqq.backend.core.utils.TestUtils;
+import org.junit.jupiter.api.Test;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNull;
+
+
+/*******************************************************************************
+ ** Unit test for BulkLoadMappingSuggester 
+ *******************************************************************************/
+class BulkLoadMappingSuggesterTest extends BaseTest
+{
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testSimpleFlat()
+   {
+      BulkLoadTableStructure tableStructure = BulkLoadTableStructureBuilder.buildTableStructure(TestUtils.TABLE_NAME_PERSON_MEMORY);
+      List<String>           headerRow      = List.of("Id", "First Name", "lastname", "email", "homestate");
+
+      BulkLoadProfile bulkLoadProfile = new BulkLoadMappingSuggester().suggestBulkLoadMappingProfile(tableStructure, headerRow);
+      assertEquals("v1", bulkLoadProfile.getVersion());
+      assertEquals("FLAT", bulkLoadProfile.getLayout());
+      assertNull(getFieldByName(bulkLoadProfile, "id"));
+      assertEquals(1, getFieldByName(bulkLoadProfile, "firstName").getColumnIndex());
+      assertEquals(2, getFieldByName(bulkLoadProfile, "lastName").getColumnIndex());
+      assertEquals(3, getFieldByName(bulkLoadProfile, "email").getColumnIndex());
+      assertEquals(4, getFieldByName(bulkLoadProfile, "homeStateId").getColumnIndex());
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testSimpleTall()
+   {
+      BulkLoadTableStructure tableStructure = BulkLoadTableStructureBuilder.buildTableStructure(TestUtils.TABLE_NAME_ORDER);
+      List<String>           headerRow      = List.of("orderNo", "shipto name", "sku", "quantity");
+
+      BulkLoadProfile bulkLoadProfile = new BulkLoadMappingSuggester().suggestBulkLoadMappingProfile(tableStructure, headerRow);
+      assertEquals("v1", bulkLoadProfile.getVersion());
+      assertEquals("TALL", bulkLoadProfile.getLayout());
+      assertEquals(0, getFieldByName(bulkLoadProfile, "orderNo").getColumnIndex());
+      assertEquals(1, getFieldByName(bulkLoadProfile, "shipToName").getColumnIndex());
+      assertEquals(2, getFieldByName(bulkLoadProfile, "orderLine.sku").getColumnIndex());
+      assertEquals(3, getFieldByName(bulkLoadProfile, "orderLine.quantity").getColumnIndex());
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testTallWithTableNamesOnAssociations()
+   {
+      BulkLoadTableStructure tableStructure = BulkLoadTableStructureBuilder.buildTableStructure(TestUtils.TABLE_NAME_ORDER);
+      List<String>           headerRow      = List.of("Order No", "Ship To Name", "Order Line: SKU", "Order Line: Quantity");
+
+      BulkLoadProfile bulkLoadProfile = new BulkLoadMappingSuggester().suggestBulkLoadMappingProfile(tableStructure, headerRow);
+      assertEquals("v1", bulkLoadProfile.getVersion());
+      assertEquals("TALL", bulkLoadProfile.getLayout());
+      assertEquals(0, getFieldByName(bulkLoadProfile, "orderNo").getColumnIndex());
+      assertEquals(1, getFieldByName(bulkLoadProfile, "shipToName").getColumnIndex());
+      assertEquals(2, getFieldByName(bulkLoadProfile, "orderLine.sku").getColumnIndex());
+      assertEquals(3, getFieldByName(bulkLoadProfile, "orderLine.quantity").getColumnIndex());
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testChallengingAddress1And2()
+   {
+      try
+      {
+         {
+            QTableMetaData table = QContext.getQInstance().getTable(TestUtils.TABLE_NAME_ORDER);
+            table.addField(new QFieldMetaData("address1", QFieldType.STRING));
+            table.addField(new QFieldMetaData("address2", QFieldType.STRING));
+
+            BulkLoadTableStructure tableStructure = BulkLoadTableStructureBuilder.buildTableStructure(TestUtils.TABLE_NAME_ORDER);
+            List<String>           headerRow      = List.of("orderNo", "ship to name", "address 1", "address 2");
+
+            BulkLoadProfile bulkLoadProfile = new BulkLoadMappingSuggester().suggestBulkLoadMappingProfile(tableStructure, headerRow);
+            assertEquals(0, getFieldByName(bulkLoadProfile, "orderNo").getColumnIndex());
+            assertEquals(1, getFieldByName(bulkLoadProfile, "shipToName").getColumnIndex());
+            assertEquals(2, getFieldByName(bulkLoadProfile, "address1").getColumnIndex());
+            assertEquals(3, getFieldByName(bulkLoadProfile, "address2").getColumnIndex());
+            reInitInstanceInContext(TestUtils.defineInstance());
+         }
+
+         {
+            QTableMetaData table = QContext.getQInstance().getTable(TestUtils.TABLE_NAME_ORDER);
+            table.addField(new QFieldMetaData("address", QFieldType.STRING));
+            table.addField(new QFieldMetaData("address2", QFieldType.STRING));
+
+            BulkLoadTableStructure tableStructure = BulkLoadTableStructureBuilder.buildTableStructure(TestUtils.TABLE_NAME_ORDER);
+            List<String>           headerRow      = List.of("orderNo", "ship to name", "address 1", "address 2");
+
+            BulkLoadProfile bulkLoadProfile = new BulkLoadMappingSuggester().suggestBulkLoadMappingProfile(tableStructure, headerRow);
+            assertEquals(0, getFieldByName(bulkLoadProfile, "orderNo").getColumnIndex());
+            assertEquals(1, getFieldByName(bulkLoadProfile, "shipToName").getColumnIndex());
+            assertEquals(2, getFieldByName(bulkLoadProfile, "address").getColumnIndex());
+            assertEquals(3, getFieldByName(bulkLoadProfile, "address2").getColumnIndex());
+            reInitInstanceInContext(TestUtils.defineInstance());
+         }
+
+         {
+            QTableMetaData table = QContext.getQInstance().getTable(TestUtils.TABLE_NAME_ORDER);
+            table.addField(new QFieldMetaData("address1", QFieldType.STRING));
+            table.addField(new QFieldMetaData("address2", QFieldType.STRING));
+
+            BulkLoadTableStructure tableStructure = BulkLoadTableStructureBuilder.buildTableStructure(TestUtils.TABLE_NAME_ORDER);
+            List<String>           headerRow      = List.of("orderNo", "ship to name", "address", "address 2");
+
+            BulkLoadProfile bulkLoadProfile = new BulkLoadMappingSuggester().suggestBulkLoadMappingProfile(tableStructure, headerRow);
+            assertEquals(0, getFieldByName(bulkLoadProfile, "orderNo").getColumnIndex());
+            assertEquals(1, getFieldByName(bulkLoadProfile, "shipToName").getColumnIndex());
+            assertEquals(2, getFieldByName(bulkLoadProfile, "address1").getColumnIndex());
+            assertEquals(3, getFieldByName(bulkLoadProfile, "address2").getColumnIndex());
+            reInitInstanceInContext(TestUtils.defineInstance());
+         }
+      }
+      finally
+      {
+         reInitInstanceInContext(TestUtils.defineInstance());
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testSimpleWide()
+   {
+      BulkLoadTableStructure tableStructure = BulkLoadTableStructureBuilder.buildTableStructure(TestUtils.TABLE_NAME_ORDER);
+      List<String>           headerRow      = List.of("orderNo", "ship to name", "sku", "quantity1", "sku 2", "quantity 2");
+
+      BulkLoadProfile bulkLoadProfile = new BulkLoadMappingSuggester().suggestBulkLoadMappingProfile(tableStructure, headerRow);
+      assertEquals("v1", bulkLoadProfile.getVersion());
+      assertEquals("WIDE", bulkLoadProfile.getLayout());
+      assertEquals(0, getFieldByName(bulkLoadProfile, "orderNo").getColumnIndex());
+      assertEquals(1, getFieldByName(bulkLoadProfile, "shipToName").getColumnIndex());
+      assertEquals(2, getFieldByName(bulkLoadProfile, "orderLine.sku,0").getColumnIndex());
+      assertEquals(3, getFieldByName(bulkLoadProfile, "orderLine.quantity,0").getColumnIndex());
+      assertEquals(4, getFieldByName(bulkLoadProfile, "orderLine.sku,1").getColumnIndex());
+      assertEquals(5, getFieldByName(bulkLoadProfile, "orderLine.quantity,1").getColumnIndex());
+
+      /////////////////////////////////////////////////////////////////
+      // assert that the order of fields matches the file's ordering //
+      /////////////////////////////////////////////////////////////////
+      assertEquals(List.of("orderNo", "shipToName", "orderLine.sku,0", "orderLine.quantity,0", "orderLine.sku,1", "orderLine.quantity,1"),
+         bulkLoadProfile.getFieldList().stream().map(f -> f.getFieldName()).toList());
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   private BulkLoadProfileField getFieldByName(BulkLoadProfile bulkLoadProfile, String fieldName)
+   {
+      return (bulkLoadProfile.getFieldList().stream()
+         .filter(f -> f.getFieldName().equals(fieldName))
+         .findFirst().orElse(null));
+   }
+
+}

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadValueMapperTest.java

@@ -0,0 +1,182 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping;
+
+
+import java.io.Serializable;
+import java.math.BigDecimal;
+import java.util.List;
+import java.util.Map;
+import com.kingsrook.qqq.backend.core.BaseTest;
+import com.kingsrook.qqq.backend.core.context.QContext;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkInsertMapping;
+import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
+import com.kingsrook.qqq.backend.core.utils.TestUtils;
+import org.json.JSONArray;
+import org.json.JSONObject;
+import org.junit.jupiter.api.Test;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+
+/*******************************************************************************
+ ** Unit test for ValueMapper 
+ *******************************************************************************/
+class BulkLoadValueMapperTest extends BaseTest
+{
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void test() throws QException
+   {
+      BulkInsertMapping mapping = new BulkInsertMapping().withFieldNameToValueMapping(Map.of(
+         "storeId", Map.of("QQQMart", 1, "Q'R'Us", 2),
+         "shipToName", Map.of("HoJu", "Homer", "Bart", "Bartholomew"),
+         "orderLine.sku", Map.of("ABC", "Alphabet"),
+         "orderLine.extrinsics.value", Map.of("foo", "bar", "bar", "baz"),
+         "extrinsics.key", Map.of("1", "one", "2", "two")
+      ));
+
+      QRecord inputRecord = new QRecord()
+         .withValue("storeId", "QQQMart")
+         .withValue("shipToName", "HoJu")
+         .withAssociatedRecord("orderLine", new QRecord()
+            .withValue("sku", "ABC")
+            .withAssociatedRecord("extrinsics", new QRecord()
+               .withValue("key", "myKey")
+               .withValue("value", "foo")
+            )
+            .withAssociatedRecord("extrinsics", new QRecord()
+               .withValue("key", "yourKey")
+               .withValue("value", "bar")
+            )
+         )
+         .withAssociatedRecord("extrinsics", new QRecord()
+            .withValue("key", 1)
+            .withValue("value", "foo")
+         );
+      JSONObject beforeJson = recordToJson(inputRecord);
+
+      QRecord expectedRecord = new QRecord()
+         .withValue("storeId", 1)
+         .withValue("shipToName", "Homer")
+         .withAssociatedRecord("orderLine", new QRecord()
+            .withValue("sku", "Alphabet")
+            .withAssociatedRecord("extrinsics", new QRecord()
+               .withValue("key", "myKey")
+               .withValue("value", "bar")
+            )
+            .withAssociatedRecord("extrinsics", new QRecord()
+               .withValue("key", "yourKey")
+               .withValue("value", "baz")
+            )
+         )
+         .withAssociatedRecord("extrinsics", new QRecord()
+            .withValue("key", "one")
+            .withValue("value", "foo")
+         );
+      JSONObject expectedJson = recordToJson(expectedRecord);
+
+      BulkLoadValueMapper.valueMapping(List.of(inputRecord), mapping, QContext.getQInstance().getTable(TestUtils.TABLE_NAME_ORDER));
+      JSONObject actualJson = recordToJson(inputRecord);
+
+      System.out.println("Before");
+      System.out.println(beforeJson.toString(3));
+      System.out.println("Actual");
+      System.out.println(actualJson.toString(3));
+      System.out.println("Expected");
+      System.out.println(expectedJson.toString(3));
+
+      assertThat(actualJson).usingRecursiveComparison().isEqualTo(expectedJson);
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   void testPossibleValue(Serializable inputValue, Serializable expectedValue, boolean expectErrors) throws QException
+   {
+      QRecord inputRecord = new QRecord().withValue("homeStateId", inputValue);
+      BulkLoadValueMapper.valueMapping(List.of(inputRecord), new BulkInsertMapping(), QContext.getQInstance().getTable(TestUtils.TABLE_NAME_PERSON_MEMORY));
+      assertEquals(expectedValue, inputRecord.getValue("homeStateId"));
+
+      if(expectErrors)
+      {
+         assertThat(inputRecord.getErrors().get(0)).isInstanceOf(BulkLoadPossibleValueError.class);
+      }
+      else
+      {
+         assertThat(inputRecord.getErrors()).isNullOrEmpty();
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testPossibleValues() throws QException
+   {
+      testPossibleValue(1, 1, false);
+      testPossibleValue("1", 1, false);
+      testPossibleValue("1.0", 1, false);
+      testPossibleValue(new BigDecimal("1.0"), 1, false);
+      testPossibleValue("IL", 1, false);
+      testPossibleValue("il", 1, false);
+
+      testPossibleValue(512, 512, true); // an id, but not in the PVS
+      testPossibleValue("USA", "USA", true);
+      testPossibleValue(true, true, true);
+      testPossibleValue(new BigDecimal("4.7"), new BigDecimal("4.7"), true);
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   public static JSONObject recordToJson(QRecord record)
+   {
+      JSONObject jsonObject = new JSONObject();
+      for(Map.Entry<String, Serializable> valueEntry : CollectionUtils.nonNullMap(record.getValues()).entrySet())
+      {
+         jsonObject.put(valueEntry.getKey(), valueEntry.getValue());
+      }
+      for(Map.Entry<String, List<QRecord>> associationEntry : CollectionUtils.nonNullMap(record.getAssociatedRecords()).entrySet())
+      {
+         JSONArray jsonArray = new JSONArray();
+         for(QRecord associationRecord : CollectionUtils.nonNullList(associationEntry.getValue()))
+         {
+            jsonArray.put(recordToJson(associationRecord));
+         }
+         jsonObject.put(associationEntry.getKey(), jsonArray);
+      }
+      return (jsonObject);
+   }
+
+}

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/FlatRowsToRecordTest.java

@@ -0,0 +1,262 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping;
+
+
+import java.io.Serializable;
+import java.math.BigDecimal;
+import java.util.List;
+import java.util.Map;
+import com.kingsrook.qqq.backend.core.BaseTest;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.TestFileToRows;
+import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkInsertMapping;
+import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow;
+import com.kingsrook.qqq.backend.core.utils.TestUtils;
+import com.kingsrook.qqq.backend.core.utils.collections.ListBuilder;
+import org.junit.jupiter.api.Test;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+
+/*******************************************************************************
+ ** Unit test for FlatRowsToRecord 
+ *******************************************************************************/
+class FlatRowsToRecordTest extends BaseTest
+{
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testFieldNameToHeaderNameMapping() throws QException
+   {
+      TestFileToRows fileToRows = new TestFileToRows(List.of(
+         new Serializable[] { "id", "firstName", "Last Name", "Ignore", "cost" },
+         new Serializable[] { 1, "Homer", "Simpson", true, "three fifty" },
+         new Serializable[] { 2, "Marge", "Simpson", false, "" },
+         new Serializable[] { 3, "Bart", "Simpson", "A", "99.95" },
+         new Serializable[] { 4, "Ned", "Flanders", 3.1, "one$" },
+         new Serializable[] { "", "", "", "", "" } // all blank row (we can get these at the bottoms of files) - make sure it doesn't become a record.
+      ));
+
+      BulkLoadFileRow header = fileToRows.next();
+
+      FlatRowsToRecord rowsToRecord = new FlatRowsToRecord();
+
+      BulkInsertMapping mapping = new BulkInsertMapping()
+         .withFieldNameToHeaderNameMap(Map.of(
+            "firstName", "firstName",
+            "lastName", "Last Name",
+            "cost", "cost"
+         ))
+         .withFieldNameToDefaultValueMap(Map.of(
+            "noOfShoes", 2
+         ))
+         .withFieldNameToValueMapping(Map.of("cost", Map.of("three fifty", new BigDecimal("3.50"), "one$", new BigDecimal("1.00"))))
+         .withTableName(TestUtils.TABLE_NAME_PERSON)
+         .withHasHeaderRow(true);
+
+      List<QRecord> records = rowsToRecord.nextPage(fileToRows, header, mapping, 1);
+      assertEquals(1, records.size());
+      assertEquals(List.of("Homer"), getValues(records, "firstName"));
+      assertEquals(List.of("Simpson"), getValues(records, "lastName"));
+      assertEquals(List.of(2), getValues(records, "noOfShoes"));
+      assertEquals(List.of(new BigDecimal("3.50")), getValues(records, "cost"));
+      assertEquals(4, records.get(0).getValues().size()); // make sure no additional values were set
+      assertEquals(1, ((List<?>) records.get(0).getBackendDetail("fileRows")).size());
+      assertEquals("Row 2", records.get(0).getBackendDetail("rowNos"));
+
+      records = rowsToRecord.nextPage(fileToRows, header, mapping, 2);
+      assertEquals(2, records.size());
+      assertEquals(List.of("Marge", "Bart"), getValues(records, "firstName"));
+      assertEquals(List.of(2, 2), getValues(records, "noOfShoes"));
+      assertEquals(ListBuilder.of(null, new BigDecimal("99.95")), getValues(records, "cost"));
+      assertEquals(1, ((List<?>) records.get(0).getBackendDetail("fileRows")).size());
+      assertEquals("Row 3", records.get(0).getBackendDetail("rowNos"));
+      assertEquals("Row 4", records.get(1).getBackendDetail("rowNos"));
+
+      records = rowsToRecord.nextPage(fileToRows, header, mapping, Integer.MAX_VALUE);
+      assertEquals(1, records.size());
+      assertEquals(List.of("Ned"), getValues(records, "firstName"));
+      assertEquals(List.of(2), getValues(records, "noOfShoes"));
+      assertEquals(ListBuilder.of(new BigDecimal("1.00")), getValues(records, "cost"));
+      assertEquals("Row 5", records.get(0).getBackendDetail("rowNos"));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testFieldNameToColumnIndexMapping() throws QException
+   {
+      TestFileToRows fileToRows = new TestFileToRows(List.of(
+         //                   0, 1,       2,         3,    4
+         new Serializable[] { 1, "Homer", "Simpson", true, "three fifty" },
+         new Serializable[] { 2, "Marge", "Simpson", false, "" },
+         new Serializable[] { 3, "Bart", "Simpson", "A", "99.95" },
+         new Serializable[] { 4, "Ned", "Flanders", 3.1, "one$" }
+      ));
+
+      FlatRowsToRecord rowsToRecord = new FlatRowsToRecord();
+
+      BulkInsertMapping mapping = new BulkInsertMapping()
+         .withFieldNameToIndexMap(Map.of(
+            "firstName", 1,
+            "lastName", 2,
+            "cost", 4
+         ))
+         .withFieldNameToDefaultValueMap(Map.of(
+            "noOfShoes", 2
+         ))
+         .withFieldNameToValueMapping(Map.of("cost", Map.of("three fifty", new BigDecimal("3.50"), "one$", new BigDecimal("1.00"))))
+         .withTableName(TestUtils.TABLE_NAME_PERSON)
+         .withHasHeaderRow(false);
+
+      List<QRecord> records = rowsToRecord.nextPage(fileToRows, null, mapping, 1);
+      assertEquals(1, records.size());
+      assertEquals(List.of("Homer"), getValues(records, "firstName"));
+      assertEquals(List.of("Simpson"), getValues(records, "lastName"));
+      assertEquals(List.of(2), getValues(records, "noOfShoes"));
+      assertEquals(List.of(new BigDecimal("3.50")), getValues(records, "cost"));
+      assertEquals(4, records.get(0).getValues().size()); // make sure no additional values were set
+      assertEquals(1, ((List<?>) records.get(0).getBackendDetail("fileRows")).size());
+      assertEquals("Row 1", records.get(0).getBackendDetail("rowNos"));
+
+      records = rowsToRecord.nextPage(fileToRows, null, mapping, 2);
+      assertEquals(2, records.size());
+      assertEquals(List.of("Marge", "Bart"), getValues(records, "firstName"));
+      assertEquals(List.of(2, 2), getValues(records, "noOfShoes"));
+      assertEquals(ListBuilder.of(null, new BigDecimal("99.95")), getValues(records, "cost"));
+      assertEquals(1, ((List<?>) records.get(0).getBackendDetail("fileRows")).size());
+      assertEquals("Row 2", records.get(0).getBackendDetail("rowNos"));
+      assertEquals("Row 3", records.get(1).getBackendDetail("rowNos"));
+
+      records = rowsToRecord.nextPage(fileToRows, null, mapping, Integer.MAX_VALUE);
+      assertEquals(1, records.size());
+      assertEquals(List.of("Ned"), getValues(records, "firstName"));
+      assertEquals(List.of(2), getValues(records, "noOfShoes"));
+      assertEquals(ListBuilder.of(new BigDecimal("1.00")), getValues(records, "cost"));
+      assertEquals("Row 4", records.get(0).getBackendDetail("rowNos"));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testFieldNameToIndexMapping() throws QException
+   {
+      TestFileToRows fileToRows = new TestFileToRows(List.of(
+         new Serializable[] { 1, "Homer", "Simpson", true },
+         new Serializable[] { 2, "Marge", "Simpson", false },
+         new Serializable[] { 3, "Bart", "Simpson", "A" },
+         new Serializable[] { 4, "Ned", "Flanders", 3.1 }
+      ));
+
+      BulkLoadFileRow header = null;
+
+      FlatRowsToRecord rowsToRecord = new FlatRowsToRecord();
+
+      BulkInsertMapping mapping = new BulkInsertMapping()
+         .withFieldNameToIndexMap(Map.of(
+            "firstName", 1,
+            "lastName", 2
+         ))
+         .withFieldNameToDefaultValueMap(Map.of(
+            "noOfShoes", 2
+         ))
+         .withTableName(TestUtils.TABLE_NAME_PERSON)
+         .withHasHeaderRow(false);
+
+      List<QRecord> records = rowsToRecord.nextPage(fileToRows, header, mapping, 1);
+      assertEquals(List.of("Homer"), getValues(records, "firstName"));
+      assertEquals(List.of("Simpson"), getValues(records, "lastName"));
+      assertEquals(List.of(2), getValues(records, "noOfShoes"));
+      assertEquals(3, records.get(0).getValues().size()); // make sure no additional values were set
+
+      records = rowsToRecord.nextPage(fileToRows, header, mapping, 2);
+      assertEquals(List.of("Marge", "Bart"), getValues(records, "firstName"));
+      assertEquals(List.of(2, 2), getValues(records, "noOfShoes"));
+
+      records = rowsToRecord.nextPage(fileToRows, header, mapping, Integer.MAX_VALUE);
+      assertEquals(List.of("Ned"), getValues(records, "firstName"));
+      assertEquals(List.of(2), getValues(records, "noOfShoes"));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testPossibleValueMappings() throws QException
+   {
+      TestFileToRows fileToRows = new TestFileToRows(List.of(
+         new Serializable[] { "id", "firstName", "Last Name", "Home State" },
+         new Serializable[] { 1, "Homer", "Simpson", 1 },
+         new Serializable[] { 2, "Marge", "Simpson", "MO" },
+         new Serializable[] { 3, "Bart", "Simpson", null },
+         new Serializable[] { 4, "Ned", "Flanders", "Not a state" },
+         new Serializable[] { 5, "Mr.", "Burns", 5 }
+      ));
+
+      BulkLoadFileRow header = fileToRows.next();
+
+      FlatRowsToRecord rowsToRecord = new FlatRowsToRecord();
+
+      BulkInsertMapping mapping = new BulkInsertMapping()
+         .withFieldNameToHeaderNameMap(Map.of(
+            "firstName", "firstName",
+            "lastName", "Last Name",
+            "homeStateId", "Home State"
+         ))
+         .withTableName(TestUtils.TABLE_NAME_PERSON)
+         .withHasHeaderRow(true);
+
+      List<QRecord> records = rowsToRecord.nextPage(fileToRows, header, mapping, Integer.MAX_VALUE);
+      assertEquals(5, records.size());
+      assertEquals(List.of("Homer", "Marge", "Bart", "Ned", "Mr."), getValues(records, "firstName"));
+      assertEquals(ListBuilder.of(1, 2, null, "Not a state", 5), getValues(records, "homeStateId"));
+
+      assertThat(records.get(0).getErrors()).isNullOrEmpty();
+      assertThat(records.get(1).getErrors()).isNullOrEmpty();
+      assertThat(records.get(2).getErrors()).isNullOrEmpty();
+      assertThat(records.get(3).getErrors()).hasSize(1).element(0).matches(e -> e.getMessage().contains("not a valid option"));
+      assertThat(records.get(4).getErrors()).hasSize(1).element(0).matches(e -> e.getMessage().contains("not a valid option"));
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   private List<Serializable> getValues(List<QRecord> records, String fieldName)
+   {
+      return (records.stream().map(r -> r.getValue(fieldName)).toList());
+   }
+
+}

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/TallRowsToRecordTest.java

@@ -0,0 +1,599 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping;
+
+
+import java.io.Serializable;
+import java.util.List;
+import java.util.Map;
+import com.kingsrook.qqq.backend.core.BaseTest;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.CsvFileToRows;
+import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkInsertMapping;
+import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow;
+import com.kingsrook.qqq.backend.core.utils.TestUtils;
+import org.junit.jupiter.api.Test;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+
+/*******************************************************************************
+ ** Unit test for TallRowsToRecord 
+ *******************************************************************************/
+class TallRowsToRecordTest extends BaseTest
+{
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testOrderAndLines() throws QException
+   {
+      CsvFileToRows fileToRows = CsvFileToRows.forString("""
+         orderNo, Ship To, lastName, SKU,       Quantity
+         1,       Homer,   Simpson,  DONUT,     12
+         ,        Homer,   Simpson,  BEER,      500
+         ,        Homer,   Simpson,  COUCH,     1
+         2,       Ned,     Flanders, BIBLE,     7
+         ,        Ned,     Flanders, LAWNMOWER, 1
+         """);
+
+      BulkLoadFileRow header = fileToRows.next();
+
+      TallRowsToRecord rowsToRecord = new TallRowsToRecord();
+
+      BulkInsertMapping mapping = new BulkInsertMapping()
+         .withFieldNameToHeaderNameMap(Map.of(
+            "orderNo", "orderNo",
+            "shipToName", "Ship To",
+            "orderLine.sku", "SKU",
+            "orderLine.quantity", "Quantity"
+         ))
+         .withTallLayoutGroupByIndexMap(Map.of(
+            TestUtils.TABLE_NAME_ORDER, List.of(1, 2),
+            "orderLine", List.of(3)
+         ))
+         .withMappedAssociations(List.of("orderLine"))
+         .withTableName(TestUtils.TABLE_NAME_ORDER)
+         .withLayout(BulkInsertMapping.Layout.TALL)
+         .withHasHeaderRow(true);
+
+      List<QRecord> records = rowsToRecord.nextPage(fileToRows, header, mapping, Integer.MAX_VALUE);
+      assertEquals(2, records.size());
+
+      QRecord order = records.get(0);
+      assertEquals(1, order.getValueInteger("orderNo"));
+      assertEquals("Homer", order.getValueString("shipToName"));
+      assertEquals(3, order.getAssociatedRecords().get("orderLine").size());
+      assertEquals(List.of("DONUT", "BEER", "COUCH"), getValues(order.getAssociatedRecords().get("orderLine"), "sku"));
+      assertEquals(List.of(12, 500, 1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity"));
+      assertEquals("Rows 2-4", order.getBackendDetail("rowNos"));
+      assertEquals(3, ((List<?>) order.getBackendDetail("fileRows")).size());
+      assertEquals("Row 2", order.getAssociatedRecords().get("orderLine").get(0).getBackendDetail("rowNos"));
+      assertEquals("Row 3", order.getAssociatedRecords().get("orderLine").get(1).getBackendDetail("rowNos"));
+      assertEquals("Row 4", order.getAssociatedRecords().get("orderLine").get(2).getBackendDetail("rowNos"));
+
+      order = records.get(1);
+      assertEquals(2, order.getValueInteger("orderNo"));
+      assertEquals("Ned", order.getValueString("shipToName"));
+      assertEquals(2, order.getAssociatedRecords().get("orderLine").size());
+      assertEquals(List.of("BIBLE", "LAWNMOWER"), getValues(order.getAssociatedRecords().get("orderLine"), "sku"));
+      assertEquals(List.of(7, 1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity"));
+      assertEquals("Rows 5-6", order.getBackendDetail("rowNos"));
+      assertEquals(2, ((List<?>) order.getBackendDetail("fileRows")).size());
+   }
+
+
+
+   /*******************************************************************************
+    ** test to show that we can do 1 default line item (child record) for each
+    ** header record.
+    *******************************************************************************/
+   @Test
+   void testOrderAndLinesWithLineValuesFromDefaults() throws QException
+   {
+      CsvFileToRows fileToRows = CsvFileToRows.forString("""
+         orderNo, Ship To, lastName
+         1,       Homer,   Simpson
+         2,       Ned,     Flanders
+         """);
+
+      BulkLoadFileRow header = fileToRows.next();
+
+      TallRowsToRecord rowsToRecord = new TallRowsToRecord();
+
+      BulkInsertMapping mapping = new BulkInsertMapping()
+         .withFieldNameToHeaderNameMap(Map.of(
+            "orderNo", "orderNo",
+            "shipToName", "Ship To"
+         ))
+         .withFieldNameToDefaultValueMap(Map.of(
+            "orderLine.sku", "NUCLEAR-ROD",
+            "orderLine.quantity", 1
+         ))
+         .withMappedAssociations(List.of("orderLine"))
+         .withTableName(TestUtils.TABLE_NAME_ORDER)
+         .withLayout(BulkInsertMapping.Layout.TALL)
+         .withHasHeaderRow(true);
+
+      List<QRecord> records = rowsToRecord.nextPage(fileToRows, header, mapping, Integer.MAX_VALUE);
+      assertEquals(2, records.size());
+
+      QRecord order = records.get(0);
+      assertEquals(1, order.getValueInteger("orderNo"));
+      assertEquals("Homer", order.getValueString("shipToName"));
+      assertEquals(1, order.getAssociatedRecords().get("orderLine").size());
+      assertEquals(List.of("NUCLEAR-ROD"), getValues(order.getAssociatedRecords().get("orderLine"), "sku"));
+      assertEquals(List.of(1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity"));
+      assertEquals("Row 2", order.getBackendDetail("rowNos"));
+      assertEquals(1, ((List<?>) order.getBackendDetail("fileRows")).size());
+
+      order = records.get(1);
+      assertEquals(2, order.getValueInteger("orderNo"));
+      assertEquals("Ned", order.getValueString("shipToName"));
+      assertEquals(1, order.getAssociatedRecords().get("orderLine").size());
+      assertEquals(List.of("NUCLEAR-ROD"), getValues(order.getAssociatedRecords().get("orderLine"), "sku"));
+      assertEquals(List.of(1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity"));
+      assertEquals("Row 3", order.getBackendDetail("rowNos"));
+      assertEquals(1, ((List<?>) order.getBackendDetail("fileRows")).size());
+   }
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testOrderAndLinesWithoutHeader() throws QException
+   {
+      // 0, 1,     2,        3,         4
+      CsvFileToRows fileToRows = CsvFileToRows.forString("""
+         1, Homer, Simpson,  DONUT,     12
+         ,  Homer, Simpson,  BEER,      500
+         ,  Homer, Simpson,  COUCH,     1
+         2, Ned,   Flanders, BIBLE,     7
+         ,  Ned,   Flanders, LAWNMOWER, 1
+         """);
+
+      BulkLoadFileRow header = null;
+
+      TallRowsToRecord rowsToRecord = new TallRowsToRecord();
+
+      BulkInsertMapping mapping = new BulkInsertMapping()
+         .withFieldNameToIndexMap(Map.of(
+            "orderNo", 0,
+            "shipToName", 1,
+            "orderLine.sku", 3,
+            "orderLine.quantity", 4
+         ))
+         .withTallLayoutGroupByIndexMap(Map.of(
+            TestUtils.TABLE_NAME_ORDER, List.of(1, 2),
+            "orderLine", List.of(3)
+         ))
+         .withMappedAssociations(List.of("orderLine"))
+         .withTableName(TestUtils.TABLE_NAME_ORDER)
+         .withLayout(BulkInsertMapping.Layout.TALL)
+         .withHasHeaderRow(false);
+
+      List<QRecord> records = rowsToRecord.nextPage(fileToRows, header, mapping, Integer.MAX_VALUE);
+      assertEquals(2, records.size());
+
+      QRecord order = records.get(0);
+      assertEquals(1, order.getValueInteger("orderNo"));
+      assertEquals("Homer", order.getValueString("shipToName"));
+      assertEquals(3, order.getAssociatedRecords().get("orderLine").size());
+      assertEquals(List.of("DONUT", "BEER", "COUCH"), getValues(order.getAssociatedRecords().get("orderLine"), "sku"));
+      assertEquals(List.of(12, 500, 1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity"));
+      assertEquals("Rows 1-3", order.getBackendDetail("rowNos"));
+      assertEquals(3, ((List<?>) order.getBackendDetail("fileRows")).size());
+      assertEquals("Row 1", order.getAssociatedRecords().get("orderLine").get(0).getBackendDetail("rowNos"));
+      assertEquals("Row 2", order.getAssociatedRecords().get("orderLine").get(1).getBackendDetail("rowNos"));
+      assertEquals("Row 3", order.getAssociatedRecords().get("orderLine").get(2).getBackendDetail("rowNos"));
+
+      order = records.get(1);
+      assertEquals(2, order.getValueInteger("orderNo"));
+      assertEquals("Ned", order.getValueString("shipToName"));
+      assertEquals(2, order.getAssociatedRecords().get("orderLine").size());
+      assertEquals(List.of("BIBLE", "LAWNMOWER"), getValues(order.getAssociatedRecords().get("orderLine"), "sku"));
+      assertEquals(List.of(7, 1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity"));
+      assertEquals("Rows 4-5", order.getBackendDetail("rowNos"));
+      assertEquals(2, ((List<?>) order.getBackendDetail("fileRows")).size());
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testOrderLinesAndOrderExtrinsic() throws QException
+   {
+      CsvFileToRows fileToRows = CsvFileToRows.forString("""
+         orderNo, Ship To, lastName, SKU,       Quantity, Extrinsic Key, Extrinsic Value
+         1,       Homer,   Simpson,  DONUT,     12,       Store Name,    QQQ Mart
+         1,       ,        ,         BEER,      500,      Coupon Code,   10QOff
+         1,       ,        ,         COUCH,     1
+         2,       Ned,     Flanders, BIBLE,     7
+         2,       ,        ,         LAWNMOWER, 1
+         """);
+
+      BulkLoadFileRow header = fileToRows.next();
+
+      TallRowsToRecord rowsToRecord = new TallRowsToRecord();
+
+      BulkInsertMapping mapping = new BulkInsertMapping()
+         .withFieldNameToHeaderNameMap(Map.of(
+            "orderNo", "orderNo",
+            "shipToName", "Ship To",
+            "orderLine.sku", "SKU",
+            "orderLine.quantity", "Quantity",
+            "extrinsics.key", "Extrinsic Key",
+            "extrinsics.value", "Extrinsic Value"
+         ))
+         .withTallLayoutGroupByIndexMap(Map.of(
+            TestUtils.TABLE_NAME_ORDER, List.of(0),
+            "orderLine", List.of(3),
+            "extrinsics", List.of(5)
+         ))
+         .withMappedAssociations(List.of("orderLine", "extrinsics"))
+         .withTableName(TestUtils.TABLE_NAME_ORDER)
+         .withLayout(BulkInsertMapping.Layout.TALL)
+         .withHasHeaderRow(true);
+
+      List<QRecord> records = rowsToRecord.nextPage(fileToRows, header, mapping, Integer.MAX_VALUE);
+      assertEquals(2, records.size());
+
+      QRecord order = records.get(0);
+      assertEquals(1, order.getValueInteger("orderNo"));
+      assertEquals("Homer", order.getValueString("shipToName"));
+      assertEquals(3, order.getAssociatedRecords().get("orderLine").size());
+      assertEquals(List.of("DONUT", "BEER", "COUCH"), getValues(order.getAssociatedRecords().get("orderLine"), "sku"));
+      assertEquals(List.of(12, 500, 1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity"));
+      assertEquals(2, order.getAssociatedRecords().get("extrinsics").size());
+      assertEquals(List.of("Store Name", "Coupon Code"), getValues(order.getAssociatedRecords().get("extrinsics"), "key"));
+      assertEquals(List.of("QQQ Mart", "10QOff"), getValues(order.getAssociatedRecords().get("extrinsics"), "value"));
+
+      order = records.get(1);
+      assertEquals(2, order.getValueInteger("orderNo"));
+      assertEquals("Ned", order.getValueString("shipToName"));
+      assertEquals(2, order.getAssociatedRecords().get("orderLine").size());
+      assertEquals(List.of("BIBLE", "LAWNMOWER"), getValues(order.getAssociatedRecords().get("orderLine"), "sku"));
+      assertEquals(List.of(7, 1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity"));
+      assertThat(order.getAssociatedRecords().get("extrinsics")).isNullOrEmpty();
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testOrderLinesWithLineExtrinsicsAndOrderExtrinsic() throws QException
+   {
+      Integer defaultStoreId              = 101;
+      String  defaultLineNo               = "102";
+      String  defaultOrderLineExtraSource = "file";
+
+      CsvFileToRows fileToRows = CsvFileToRows.forString("""
+         orderNo, Ship To, lastName, SKU,       Quantity, Extrinsic Key, Extrinsic Value, Line Extrinsic Key, Line Extrinsic Value
+         1,       Homer,   Simpson,  DONUT,     12,       Store Name,    QQQ Mart,        Flavor,             Chocolate
+         1,       ,        ,         DONUT,     ,         Coupon Code,   10QOff,          Size,               Large
+         1,       ,        ,         BEER,      500,      ,              ,                Flavor,             Hops
+         1,       ,        ,         COUCH,     1
+         2,       Ned,     Flanders, BIBLE,     7,        ,              ,                Flavor,             King James
+         2,       ,        ,         LAWNMOWER, 1
+         """);
+
+      BulkLoadFileRow header = fileToRows.next();
+
+      TallRowsToRecord rowsToRecord = new TallRowsToRecord();
+
+      BulkInsertMapping mapping = new BulkInsertMapping()
+         .withFieldNameToHeaderNameMap(Map.of(
+            "orderNo", "orderNo",
+            "shipToName", "Ship To",
+            "orderLine.sku", "SKU",
+            "orderLine.quantity", "Quantity",
+            "extrinsics.key", "Extrinsic Key",
+            "extrinsics.value", "Extrinsic Value",
+            "orderLine.extrinsics.key", "Line Extrinsic Key",
+            "orderLine.extrinsics.value", "Line Extrinsic Value"
+         ))
+         .withFieldNameToDefaultValueMap(Map.of(
+            "storeId", defaultStoreId,
+            "orderLine.lineNumber", defaultLineNo,
+            "orderLine.extrinsics.source", defaultOrderLineExtraSource
+         ))
+         .withFieldNameToValueMapping(Map.of("orderLine.sku", Map.of("DONUT", "D'OH-NUT")))
+         .withTallLayoutGroupByIndexMap(Map.of(
+            TestUtils.TABLE_NAME_ORDER, List.of(0),
+            "orderLine", List.of(3),
+            "extrinsics", List.of(5),
+            "orderLine.extrinsics", List.of(7)
+         ))
+         .withMappedAssociations(List.of("orderLine", "extrinsics", "orderLine.extrinsics"))
+         .withTableName(TestUtils.TABLE_NAME_ORDER)
+         .withLayout(BulkInsertMapping.Layout.TALL)
+         .withHasHeaderRow(true);
+
+      List<QRecord> records = rowsToRecord.nextPage(fileToRows, header, mapping, Integer.MAX_VALUE);
+      assertEquals(2, records.size());
+
+      QRecord order = records.get(0);
+      assertEquals(1, order.getValueInteger("orderNo"));
+      assertEquals("Homer", order.getValueString("shipToName"));
+      assertEquals(defaultStoreId, order.getValue("storeId"));
+      assertEquals(List.of("D'OH-NUT", "BEER", "COUCH"), getValues(order.getAssociatedRecords().get("orderLine"), "sku"));
+      assertEquals(List.of(12, 500, 1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity"));
+      assertEquals(List.of(defaultLineNo, defaultLineNo, defaultLineNo), getValues(order.getAssociatedRecords().get("orderLine"), "lineNumber"));
+      assertEquals(List.of("Store Name", "Coupon Code"), getValues(order.getAssociatedRecords().get("extrinsics"), "key"));
+      assertEquals(List.of("QQQ Mart", "10QOff"), getValues(order.getAssociatedRecords().get("extrinsics"), "value"));
+
+      QRecord lineItem = order.getAssociatedRecords().get("orderLine").get(0);
+      assertEquals(List.of("Flavor", "Size"), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "key"));
+      assertEquals(List.of("Chocolate", "Large"), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "value"));
+      assertEquals(List.of(defaultOrderLineExtraSource, defaultOrderLineExtraSource), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "source"));
+
+      lineItem = order.getAssociatedRecords().get("orderLine").get(1);
+      assertEquals(List.of("Flavor"), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "key"));
+      assertEquals(List.of("Hops"), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "value"));
+
+      order = records.get(1);
+      assertEquals(2, order.getValueInteger("orderNo"));
+      assertEquals("Ned", order.getValueString("shipToName"));
+      assertEquals(defaultStoreId, order.getValue("storeId"));
+      assertEquals(List.of("BIBLE", "LAWNMOWER"), getValues(order.getAssociatedRecords().get("orderLine"), "sku"));
+      assertEquals(List.of(7, 1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity"));
+      assertEquals(List.of(defaultLineNo, defaultLineNo), getValues(order.getAssociatedRecords().get("orderLine"), "lineNumber"));
+      assertThat(order.getAssociatedRecords().get("extrinsics")).isNullOrEmpty();
+
+      lineItem = order.getAssociatedRecords().get("orderLine").get(0);
+      assertEquals(List.of("Flavor"), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "key"));
+      assertEquals(List.of("King James"), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "value"));
+      assertEquals(List.of(defaultOrderLineExtraSource), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "source"));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testAutomaticGroupByAllIndexes() throws QException
+   {
+      Integer defaultStoreId              = 101;
+      String  defaultLineNo               = "102";
+      String  defaultOrderLineExtraSource = "file";
+
+      CsvFileToRows fileToRows = CsvFileToRows.forString("""
+         orderNo, Ship To, lastName, SKU,       Quantity, Extrinsic Key, Extrinsic Value, Line Extrinsic Key, Line Extrinsic Value
+         1,       Homer,   Simpson,  DONUT,     12,       Store Name,    QQQ Mart,        Flavor,             Chocolate
+         1,       Homer,   Simpson,  DONUT,     12,       Coupon Code,   10QOff,          Size,               Large
+         1,       Homer,   Simpson,  BEER,      500,      ,              ,                Flavor,             Hops
+         1,       Homer,   Simpson,  COUCH,     1
+         2,       Ned,     Flanders, BIBLE,     7,        ,              ,                Flavor,             King James
+         2,       Ned,     Flanders, LAWNMOWER, 1
+         """);
+
+      BulkLoadFileRow header = fileToRows.next();
+
+      TallRowsToRecord rowsToRecord = new TallRowsToRecord();
+
+      BulkInsertMapping mapping = new BulkInsertMapping()
+         .withFieldNameToHeaderNameMap(Map.of(
+            "orderNo", "orderNo",
+            "shipToName", "Ship To",
+            "orderLine.sku", "SKU",
+            "orderLine.quantity", "Quantity",
+            "extrinsics.key", "Extrinsic Key",
+            "extrinsics.value", "Extrinsic Value",
+            "orderLine.extrinsics.key", "Line Extrinsic Key",
+            "orderLine.extrinsics.value", "Line Extrinsic Value"
+         ))
+         .withFieldNameToDefaultValueMap(Map.of(
+            "storeId", defaultStoreId,
+            "orderLine.lineNumber", defaultLineNo,
+            "orderLine.extrinsics.source", defaultOrderLineExtraSource
+         ))
+         .withFieldNameToValueMapping(Map.of("orderLine.sku", Map.of("DONUT", "D'OH-NUT")))
+         .withMappedAssociations(List.of("orderLine", "extrinsics", "orderLine.extrinsics"))
+         .withTableName(TestUtils.TABLE_NAME_ORDER)
+         .withLayout(BulkInsertMapping.Layout.TALL)
+         .withHasHeaderRow(true);
+
+      List<QRecord> records = rowsToRecord.nextPage(fileToRows, header, mapping, Integer.MAX_VALUE);
+      assertEquals(2, records.size());
+
+      QRecord order = records.get(0);
+      assertEquals(1, order.getValueInteger("orderNo"));
+      assertEquals("Homer", order.getValueString("shipToName"));
+      assertEquals(defaultStoreId, order.getValue("storeId"));
+      assertEquals(List.of("D'OH-NUT", "BEER", "COUCH"), getValues(order.getAssociatedRecords().get("orderLine"), "sku"));
+      assertEquals(List.of(12, 500, 1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity"));
+      assertEquals(List.of(defaultLineNo, defaultLineNo, defaultLineNo), getValues(order.getAssociatedRecords().get("orderLine"), "lineNumber"));
+      assertEquals(List.of("Store Name", "Coupon Code"), getValues(order.getAssociatedRecords().get("extrinsics"), "key"));
+      assertEquals(List.of("QQQ Mart", "10QOff"), getValues(order.getAssociatedRecords().get("extrinsics"), "value"));
+
+      QRecord lineItem = order.getAssociatedRecords().get("orderLine").get(0);
+      assertEquals(List.of("Flavor", "Size"), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "key"));
+      assertEquals(List.of("Chocolate", "Large"), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "value"));
+      assertEquals(List.of(defaultOrderLineExtraSource, defaultOrderLineExtraSource), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "source"));
+
+      lineItem = order.getAssociatedRecords().get("orderLine").get(1);
+      assertEquals(List.of("Flavor"), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "key"));
+      assertEquals(List.of("Hops"), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "value"));
+
+      order = records.get(1);
+      assertEquals(2, order.getValueInteger("orderNo"));
+      assertEquals("Ned", order.getValueString("shipToName"));
+      assertEquals(defaultStoreId, order.getValue("storeId"));
+      assertEquals(List.of("BIBLE", "LAWNMOWER"), getValues(order.getAssociatedRecords().get("orderLine"), "sku"));
+      assertEquals(List.of(7, 1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity"));
+      assertEquals(List.of(defaultLineNo, defaultLineNo), getValues(order.getAssociatedRecords().get("orderLine"), "lineNumber"));
+      assertThat(order.getAssociatedRecords().get("extrinsics")).isNullOrEmpty();
+
+      lineItem = order.getAssociatedRecords().get("orderLine").get(0);
+      assertEquals(List.of("Flavor"), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "key"));
+      assertEquals(List.of("King James"), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "value"));
+      assertEquals(List.of(defaultOrderLineExtraSource), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "source"));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testSingleLine() throws QException
+   {
+      Integer defaultStoreId = 101;
+
+      CsvFileToRows fileToRows = CsvFileToRows.forString("""
+         orderNo, Ship To, lastName
+         1,       Homer,   Simpson
+         """);
+
+      BulkLoadFileRow header = fileToRows.next();
+
+      TallRowsToRecord rowsToRecord = new TallRowsToRecord();
+
+      BulkInsertMapping mapping = new BulkInsertMapping()
+         .withFieldNameToHeaderNameMap(Map.of(
+            "orderNo", "orderNo",
+            "shipToName", "Ship To"
+         ))
+         .withFieldNameToDefaultValueMap(Map.of(
+            "storeId", defaultStoreId
+         ))
+         .withTableName(TestUtils.TABLE_NAME_ORDER)
+         .withLayout(BulkInsertMapping.Layout.TALL)
+         .withHasHeaderRow(true);
+
+      List<QRecord> records = rowsToRecord.nextPage(fileToRows, header, mapping, Integer.MAX_VALUE);
+      assertEquals(1, records.size());
+
+      QRecord order = records.get(0);
+      assertEquals(1, order.getValueInteger("orderNo"));
+      assertEquals("Homer", order.getValueString("shipToName"));
+      assertEquals(defaultStoreId, order.getValue("storeId"));
+      assertEquals("Row 2", order.getBackendDetail("rowNos"));
+      assertEquals(1, ((List<?>) order.getBackendDetail("fileRows")).size());
+   }
+
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testPagination() throws QException
+   {
+      CsvFileToRows fileToRows = CsvFileToRows.forString("""
+         orderNo, Ship To, lastName, SKU,       Quantity
+         1,       Homer,   Simpson,  DONUT,     12
+         2,       Ned,     Flanders, BIBLE,     7
+         2,       Ned,     Flanders, LAWNMOWER, 1
+         3,       Bart,    Simpson,  SKATEBOARD,1
+         3,       Bart,    Simpson,  SLINGSHOT, 1
+         """);
+
+      BulkLoadFileRow header = fileToRows.next();
+
+      TallRowsToRecord rowsToRecord = new TallRowsToRecord();
+
+      BulkInsertMapping mapping = new BulkInsertMapping()
+         .withFieldNameToHeaderNameMap(Map.of(
+            "orderNo", "orderNo",
+            "shipToName", "Ship To",
+            "orderLine.sku", "SKU",
+            "orderLine.quantity", "Quantity"
+         ))
+         .withTallLayoutGroupByIndexMap(Map.of(
+            TestUtils.TABLE_NAME_ORDER, List.of(1, 2),
+            "orderLine", List.of(3)
+         ))
+         .withMappedAssociations(List.of("orderLine"))
+         .withTableName(TestUtils.TABLE_NAME_ORDER)
+         .withLayout(BulkInsertMapping.Layout.TALL)
+         .withHasHeaderRow(true);
+
+      List<QRecord> records = rowsToRecord.nextPage(fileToRows, header, mapping, 2);
+      assertEquals(2, records.size());
+
+      QRecord order = records.get(0);
+      assertEquals(1, order.getValueInteger("orderNo"));
+      assertEquals("Homer", order.getValueString("shipToName"));
+      assertEquals(List.of("DONUT"), getValues(order.getAssociatedRecords().get("orderLine"), "sku"));
+      assertEquals("Row 2", order.getBackendDetail("rowNos"));
+      assertEquals(1, ((List<?>) order.getBackendDetail("fileRows")).size());
+
+      order = records.get(1);
+      assertEquals(2, order.getValueInteger("orderNo"));
+      assertEquals("Ned", order.getValueString("shipToName"));
+      assertEquals(List.of("BIBLE", "LAWNMOWER"), getValues(order.getAssociatedRecords().get("orderLine"), "sku"));
+      assertEquals("Rows 3-4", order.getBackendDetail("rowNos"));
+      assertEquals(2, ((List<?>) order.getBackendDetail("fileRows")).size());
+
+      records = rowsToRecord.nextPage(fileToRows, header, mapping, 2);
+      assertEquals(1, records.size());
+      order = records.get(0);
+      assertEquals(3, order.getValueInteger("orderNo"));
+      assertEquals("Bart", order.getValueString("shipToName"));
+      assertEquals(List.of("SKATEBOARD", "SLINGSHOT"), getValues(order.getAssociatedRecords().get("orderLine"), "sku"));
+      assertEquals("Rows 5-6", order.getBackendDetail("rowNos"));
+      assertEquals(2, ((List<?>) order.getBackendDetail("fileRows")).size());
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testShouldProcessAssociation()
+   {
+      TallRowsToRecord tallRowsToRecord = new TallRowsToRecord();
+      assertTrue(tallRowsToRecord.shouldProcessAssociation(null, "foo"));
+      assertTrue(tallRowsToRecord.shouldProcessAssociation("", "foo"));
+      assertTrue(tallRowsToRecord.shouldProcessAssociation("foo", "foo.bar"));
+      assertTrue(tallRowsToRecord.shouldProcessAssociation("foo.bar", "foo.bar.baz"));
+
+      assertFalse(tallRowsToRecord.shouldProcessAssociation(null, "foo.bar"));
+      assertFalse(tallRowsToRecord.shouldProcessAssociation("", "foo.bar"));
+      assertFalse(tallRowsToRecord.shouldProcessAssociation("fiz", "foo.bar"));
+      assertFalse(tallRowsToRecord.shouldProcessAssociation("fiz.biz", "foo.bar"));
+      assertFalse(tallRowsToRecord.shouldProcessAssociation("foo", "foo.bar.baz"));
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   private List<Serializable> getValues(List<QRecord> records, String fieldName)
+   {
+      return (records.stream().map(r -> r.getValue(fieldName)).toList());
+   }
+
+}

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMappingTest.java

@@ -0,0 +1,291 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping;
+
+
+import java.io.Serializable;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import com.kingsrook.qqq.backend.core.BaseTest;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.CsvFileToRows;
+import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkInsertMapping;
+import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow;
+import com.kingsrook.qqq.backend.core.utils.TestUtils;
+import com.kingsrook.qqq.backend.core.utils.collections.MapBuilder;
+import org.junit.jupiter.api.Test;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+
+/*******************************************************************************
+ ** Unit test for WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMapping 
+ *******************************************************************************/
+class WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMappingTest extends BaseTest
+{
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testOrderAndLinesWithoutDupes() throws QException
+   {
+      String csv = """
+         orderNo, Ship To, lastName, SKU 1,     Quantity 1, SKU 2,     Quantity 2, SKU 3, Quantity 3
+         1,       Homer,   Simpson,  DONUT,     12,         BEER,      500,        COUCH,  two
+         2,       Ned,     Flanders, BIBLE,     7,          LAWNMOWER, 1
+         """;
+
+      CsvFileToRows   fileToRows = CsvFileToRows.forString(csv);
+      BulkLoadFileRow header     = fileToRows.next();
+
+      WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMapping rowsToRecord = new WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMapping();
+
+      BulkInsertMapping mapping = new BulkInsertMapping()
+         .withFieldNameToHeaderNameMap(Map.of(
+            "orderNo", "orderNo",
+            "shipToName", "Ship To",
+            "orderLine.sku,0", "SKU 1",
+            "orderLine.quantity,0", "Quantity 1",
+            "orderLine.sku,1", "SKU 2",
+            "orderLine.quantity,1", "Quantity 2",
+            "orderLine.sku,2", "SKU 3",
+            "orderLine.quantity,2", "Quantity 3"
+         ))
+         .withMappedAssociations(List.of("orderLine"))
+         .withFieldNameToValueMapping(Map.of(
+            "orderLine.quantity,2", Map.of("two", 2)
+         ))
+         .withTableName(TestUtils.TABLE_NAME_ORDER)
+         .withLayout(BulkInsertMapping.Layout.WIDE)
+         .withHasHeaderRow(true);
+
+      List<QRecord> records = rowsToRecord.nextPage(fileToRows, header, mapping, Integer.MAX_VALUE);
+      assertEquals(2, records.size());
+
+      QRecord order = records.get(0);
+      assertEquals(1, order.getValueInteger("orderNo"));
+      assertEquals("Homer", order.getValueString("shipToName"));
+      assertEquals(List.of("DONUT", "BEER", "COUCH"), getValues(order.getAssociatedRecords().get("orderLine"), "sku"));
+      assertEquals(List.of(12, 500, 2), getValues(order.getAssociatedRecords().get("orderLine"), "quantity"));
+      assertEquals(1, ((List<?>) order.getBackendDetail("fileRows")).size());
+      assertEquals("Row 2", order.getAssociatedRecords().get("orderLine").get(0).getBackendDetail("rowNos"));
+
+      order = records.get(1);
+      assertEquals(2, order.getValueInteger("orderNo"));
+      assertEquals("Ned", order.getValueString("shipToName"));
+      assertEquals(List.of("BIBLE", "LAWNMOWER"), getValues(order.getAssociatedRecords().get("orderLine"), "sku"));
+      assertEquals(List.of(7, 1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity"));
+      assertEquals(1, ((List<?>) order.getBackendDetail("fileRows")).size());
+      assertEquals("Row 3", order.getAssociatedRecords().get("orderLine").get(0).getBackendDetail("rowNos"));
+   }
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testOrderAndLinesWithLineValuesFromDefaults() throws QException
+   {
+      String csv = """
+         orderNo, Ship To, lastName, SKU 1,     Quantity 1
+         1,       Homer,   Simpson,  DONUT,     12,
+         2,       Ned,     Flanders, 
+         """;
+
+      CsvFileToRows   fileToRows = CsvFileToRows.forString(csv);
+      BulkLoadFileRow header     = fileToRows.next();
+
+      WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMapping rowsToRecord = new WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMapping();
+
+      BulkInsertMapping mapping = new BulkInsertMapping()
+         .withFieldNameToHeaderNameMap(Map.of(
+            "orderNo", "orderNo",
+            "shipToName", "Ship To",
+            "orderLine.sku,0", "SKU 1",
+            "orderLine.quantity,0", "Quantity 1"
+         ))
+         .withFieldNameToDefaultValueMap(Map.of(
+            "orderLine.sku,1", "NUCLEAR-ROD",
+            "orderLine.quantity,1", 1
+         ))
+         .withMappedAssociations(List.of("orderLine"))
+         .withTableName(TestUtils.TABLE_NAME_ORDER)
+         .withLayout(BulkInsertMapping.Layout.WIDE)
+         .withHasHeaderRow(true);
+
+      List<QRecord> records = rowsToRecord.nextPage(fileToRows, header, mapping, Integer.MAX_VALUE);
+      assertEquals(2, records.size());
+
+      QRecord order = records.get(0);
+      assertEquals(1, order.getValueInteger("orderNo"));
+      assertEquals("Homer", order.getValueString("shipToName"));
+      assertEquals(List.of("DONUT", "NUCLEAR-ROD"), getValues(order.getAssociatedRecords().get("orderLine"), "sku"));
+      assertEquals(List.of(12, 1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity"));
+      assertEquals(1, ((List<?>) order.getBackendDetail("fileRows")).size());
+      assertEquals("Row 2", order.getAssociatedRecords().get("orderLine").get(0).getBackendDetail("rowNos"));
+
+      order = records.get(1);
+      assertEquals(2, order.getValueInteger("orderNo"));
+      assertEquals("Ned", order.getValueString("shipToName"));
+      assertEquals(List.of("NUCLEAR-ROD"), getValues(order.getAssociatedRecords().get("orderLine"), "sku"));
+      assertEquals(List.of(1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity"));
+      assertEquals(1, ((List<?>) order.getBackendDetail("fileRows")).size());
+      assertEquals("Row 3", order.getAssociatedRecords().get("orderLine").get(0).getBackendDetail("rowNos"));
+   }
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testOrderAndLinesWithoutHeader() throws QException
+   {
+      // 0, 1,     2,        3,     4,  5,         6,   7,     8
+      String csv = """
+         1, Homer, Simpson,  DONUT, 12, BEER,      500, COUCH, 1
+         2, Ned,   Flanders, BIBLE, 7,  LAWNMOWER, 1
+         """;
+
+      CsvFileToRows   fileToRows = CsvFileToRows.forString(csv);
+      BulkLoadFileRow header     = null;
+
+      WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMapping rowsToRecord = new WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMapping();
+
+      BulkInsertMapping mapping = new BulkInsertMapping()
+         .withFieldNameToIndexMap(Map.of(
+            "orderNo", 0,
+            "shipToName", 1,
+            "orderLine.sku,0", 3,
+            "orderLine.quantity,0", 4,
+            "orderLine.sku,1", 5,
+            "orderLine.quantity,1", 6,
+            "orderLine.sku,2", 7,
+            "orderLine.quantity,2", 8
+         ))
+         .withMappedAssociations(List.of("orderLine"))
+         .withTableName(TestUtils.TABLE_NAME_ORDER)
+         .withLayout(BulkInsertMapping.Layout.WIDE)
+         .withHasHeaderRow(false);
+
+      List<QRecord> records = rowsToRecord.nextPage(fileToRows, header, mapping, Integer.MAX_VALUE);
+      assertEquals(2, records.size());
+
+      QRecord order = records.get(0);
+      assertEquals(1, order.getValueInteger("orderNo"));
+      assertEquals("Homer", order.getValueString("shipToName"));
+      assertEquals(List.of("DONUT", "BEER", "COUCH"), getValues(order.getAssociatedRecords().get("orderLine"), "sku"));
+      assertEquals(List.of(12, 500, 1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity"));
+      assertEquals(1, ((List<?>) order.getBackendDetail("fileRows")).size());
+      assertEquals("Row 1", order.getAssociatedRecords().get("orderLine").get(0).getBackendDetail("rowNos"));
+      assertEquals("Row 1", order.getAssociatedRecords().get("orderLine").get(0).getBackendDetail("rowNos"));
+      assertEquals("Row 1", order.getAssociatedRecords().get("orderLine").get(1).getBackendDetail("rowNos"));
+      assertEquals("Row 1", order.getAssociatedRecords().get("orderLine").get(2).getBackendDetail("rowNos"));
+
+      order = records.get(1);
+      assertEquals(2, order.getValueInteger("orderNo"));
+      assertEquals("Ned", order.getValueString("shipToName"));
+      assertEquals(List.of("BIBLE", "LAWNMOWER"), getValues(order.getAssociatedRecords().get("orderLine"), "sku"));
+      assertEquals(List.of(7, 1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity"));
+      assertEquals(1, ((List<?>) order.getBackendDetail("fileRows")).size());
+      assertEquals("Row 2", order.getAssociatedRecords().get("orderLine").get(0).getBackendDetail("rowNos"));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testOrderLinesAndOrderExtrinsicWithoutDupes() throws QException
+   {
+      String csv = """
+         orderNo, Ship To, lastName, SKU 1, Quantity 1, SKU 2,     Quantity 2, SKU 3, Quantity 3, Extrinsic Key 1, Extrinsic Value 1, Extrinsic Key 2, Extrinsic Value 2
+         1,       Homer,   Simpson,  DONUT, 12,         BEER,      500,        COUCH, 1,          Store Name,      QQQ Mart,          Coupon Code,     10QOff
+         2,       Ned,     Flanders, BIBLE, 7,          LAWNMOWER, 1
+         """;
+
+      CsvFileToRows   fileToRows = CsvFileToRows.forString(csv);
+      BulkLoadFileRow header     = fileToRows.next();
+
+      WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMapping rowsToRecord = new WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMapping();
+
+      BulkInsertMapping mapping = new BulkInsertMapping()
+         .withFieldNameToHeaderNameMap(MapBuilder.of(() -> new HashMap<String, String>())
+            .with("orderNo", "orderNo")
+            .with("shipToName", "Ship To")
+
+            .with("orderLine.sku,0", "SKU 1")
+            .with("orderLine.quantity,0", "Quantity 1")
+            .with("orderLine.sku,1", "SKU 2")
+            .with("orderLine.quantity,1", "Quantity 2")
+            .with("orderLine.sku,2", "SKU 3")
+            .with("orderLine.quantity,2", "Quantity 3")
+
+            .with("extrinsics.key,0", "Extrinsic Key 1")
+            .with("extrinsics.value,0", "Extrinsic Value 1")
+            .with("extrinsics.key,1", "Extrinsic Key 2")
+            .with("extrinsics.value,1", "Extrinsic Value 2")
+            .build()
+         )
+         .withFieldNameToDefaultValueMap(Map.of(
+            "orderLine.lineNumber,0", "1",
+            "orderLine.lineNumber,1", "2",
+            "orderLine.lineNumber,2", "3"
+         ))
+         .withMappedAssociations(List.of("orderLine", "extrinsics"))
+         .withTableName(TestUtils.TABLE_NAME_ORDER)
+         .withLayout(BulkInsertMapping.Layout.WIDE)
+         .withHasHeaderRow(true);
+
+      List<QRecord> records = rowsToRecord.nextPage(fileToRows, header, mapping, Integer.MAX_VALUE);
+      assertEquals(2, records.size());
+
+      QRecord order = records.get(0);
+      assertEquals(1, order.getValueInteger("orderNo"));
+      assertEquals("Homer", order.getValueString("shipToName"));
+      assertEquals(List.of("DONUT", "BEER", "COUCH"), getValues(order.getAssociatedRecords().get("orderLine"), "sku"));
+      assertEquals(List.of(12, 500, 1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity"));
+      assertEquals(List.of("1", "2", "3"), getValues(order.getAssociatedRecords().get("orderLine"), "lineNumber"));
+      assertEquals(List.of("Store Name", "Coupon Code"), getValues(order.getAssociatedRecords().get("extrinsics"), "key"));
+      assertEquals(List.of("QQQ Mart", "10QOff"), getValues(order.getAssociatedRecords().get("extrinsics"), "value"));
+
+      order = records.get(1);
+      assertEquals(2, order.getValueInteger("orderNo"));
+      assertEquals("Ned", order.getValueString("shipToName"));
+      assertEquals(List.of("BIBLE", "LAWNMOWER"), getValues(order.getAssociatedRecords().get("orderLine"), "sku"));
+      assertEquals(List.of(7, 1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity"));
+      assertEquals(List.of("1", "2"), getValues(order.getAssociatedRecords().get("orderLine"), "lineNumber"));
+      assertThat(order.getAssociatedRecords().get("extrinsics")).isNullOrEmpty();
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   private List<Serializable> getValues(List<QRecord> records, String fieldName)
+   {
+      return (records.stream().map(r -> r.getValue(fieldName)).toList());
+   }
+
+}

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithSpreadMappingTest.java

@@ -0,0 +1,306 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping;
+
+
+import java.io.Serializable;
+import java.util.List;
+import java.util.Map;
+import com.kingsrook.qqq.backend.core.BaseTest;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.CsvFileToRows;
+import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkInsertMapping;
+import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow;
+import com.kingsrook.qqq.backend.core.utils.TestUtils;
+import org.junit.jupiter.api.Test;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+
+/*******************************************************************************
+ ** Unit test for WideRowsToRecord
+ *******************************************************************************/
+class WideRowsToRecordWithSpreadMappingTest extends BaseTest
+{
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testOrderAndLinesWithoutDupes() throws QException
+   {
+      testOrderAndLines("""
+         orderNo, Ship To, lastName, SKU 1,     Quantity 1, SKU 2,     Quantity 2, SKU 3, Quantity 3
+         1,       Homer,   Simpson,  DONUT,     12,         BEER,      500,        COUCH,  1
+         2,       Ned,     Flanders, BIBLE,     7,          LAWNMOWER, 1
+         """);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testOrderAndLinesWithDupes() throws QException
+   {
+      testOrderAndLines("""
+         orderNo, Ship To, lastName, SKU,   Quantity, SKU,       Quantity, SKU,   Quantity
+         1,       Homer,   Simpson,  DONUT, 12,       BEER,      500,      COUCH, 1
+         2,       Ned,     Flanders, BIBLE, 7,        LAWNMOWER, 1
+         """);
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   private void testOrderAndLines(String csv) throws QException
+   {
+      CsvFileToRows   fileToRows = CsvFileToRows.forString(csv);
+      BulkLoadFileRow header     = fileToRows.next();
+
+      WideRowsToRecordWithSpreadMapping rowsToRecord = new WideRowsToRecordWithSpreadMapping();
+
+      BulkInsertMapping mapping = new BulkInsertMapping()
+         .withFieldNameToHeaderNameMap(Map.of(
+            "orderNo", "orderNo",
+            "shipToName", "Ship To",
+            "orderLine.sku", "SKU",
+            "orderLine.quantity", "Quantity"
+         ))
+         .withMappedAssociations(List.of("orderLine"))
+         .withTableName(TestUtils.TABLE_NAME_ORDER)
+         .withLayout(BulkInsertMapping.Layout.WIDE)
+         .withHasHeaderRow(true);
+
+      List<QRecord> records = rowsToRecord.nextPage(fileToRows, header, mapping, Integer.MAX_VALUE);
+      assertEquals(2, records.size());
+
+      QRecord order = records.get(0);
+      assertEquals(1, order.getValueInteger("orderNo"));
+      assertEquals("Homer", order.getValueString("shipToName"));
+      assertEquals(List.of("DONUT", "BEER", "COUCH"), getValues(order.getAssociatedRecords().get("orderLine"), "sku"));
+      assertEquals(List.of(12, 500, 1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity"));
+
+      order = records.get(1);
+      assertEquals(2, order.getValueInteger("orderNo"));
+      assertEquals("Ned", order.getValueString("shipToName"));
+      assertEquals(List.of("BIBLE", "LAWNMOWER"), getValues(order.getAssociatedRecords().get("orderLine"), "sku"));
+      assertEquals(List.of(7, 1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity"));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testOrderLinesAndOrderExtrinsicWithoutDupes() throws QException
+   {
+      testOrderLinesAndOrderExtrinsic("""
+         orderNo, Ship To, lastName, SKU 1, Quantity 1, SKU 2,     Quantity 2, SKU 3, Quantity 3, Extrinsic Key 1, Extrinsic Value 1, Extrinsic Key 2, Extrinsic Value 2
+         1,       Homer,   Simpson,  DONUT, 12,         BEER,      500,        COUCH, 1,          Store Name,      QQQ Mart,          Coupon Code,     10QOff
+         2,       Ned,     Flanders, BIBLE, 7,          LAWNMOWER, 1
+         """);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testOrderLinesAndOrderExtrinsicWithDupes() throws QException
+   {
+      testOrderLinesAndOrderExtrinsic("""
+         orderNo, Ship To, lastName, SKU,   Quantity, SKU,       Quantity, SKU,   Quantity, Extrinsic Key, Extrinsic Value, Extrinsic Key, Extrinsic Value
+         1,       Homer,   Simpson,  DONUT, 12,       BEER,      500,      COUCH, 1,        Store Name,    QQQ Mart,        Coupon Code,   10QOff
+         2,       Ned,     Flanders, BIBLE, 7,        LAWNMOWER, 1
+         """);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private void testOrderLinesAndOrderExtrinsic(String csv) throws QException
+   {
+      CsvFileToRows   fileToRows = CsvFileToRows.forString(csv);
+      BulkLoadFileRow header     = fileToRows.next();
+
+      WideRowsToRecordWithSpreadMapping rowsToRecord = new WideRowsToRecordWithSpreadMapping();
+
+      BulkInsertMapping mapping = new BulkInsertMapping()
+         .withFieldNameToHeaderNameMap(Map.of(
+            "orderNo", "orderNo",
+            "shipToName", "Ship To",
+            "orderLine.sku", "SKU",
+            "orderLine.quantity", "Quantity",
+            "extrinsics.key", "Extrinsic Key",
+            "extrinsics.value", "Extrinsic Value"
+         ))
+         .withMappedAssociations(List.of("orderLine", "extrinsics"))
+         .withTableName(TestUtils.TABLE_NAME_ORDER)
+         .withLayout(BulkInsertMapping.Layout.WIDE)
+         .withHasHeaderRow(true);
+
+      List<QRecord> records = rowsToRecord.nextPage(fileToRows, header, mapping, Integer.MAX_VALUE);
+      assertEquals(2, records.size());
+
+      QRecord order = records.get(0);
+      assertEquals(1, order.getValueInteger("orderNo"));
+      assertEquals("Homer", order.getValueString("shipToName"));
+      assertEquals(List.of("DONUT", "BEER", "COUCH"), getValues(order.getAssociatedRecords().get("orderLine"), "sku"));
+      assertEquals(List.of(12, 500, 1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity"));
+      assertEquals(List.of("Store Name", "Coupon Code"), getValues(order.getAssociatedRecords().get("extrinsics"), "key"));
+      assertEquals(List.of("QQQ Mart", "10QOff"), getValues(order.getAssociatedRecords().get("extrinsics"), "value"));
+
+      order = records.get(1);
+      assertEquals(2, order.getValueInteger("orderNo"));
+      assertEquals("Ned", order.getValueString("shipToName"));
+      assertEquals(List.of("BIBLE", "LAWNMOWER"), getValues(order.getAssociatedRecords().get("orderLine"), "sku"));
+      assertEquals(List.of(7, 1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity"));
+      assertThat(order.getAssociatedRecords().get("extrinsics")).isNullOrEmpty();
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testOrderLinesWithLineExtrinsicsAndOrderExtrinsicWithoutDupes() throws QException
+   {
+      testOrderLinesWithLineExtrinsicsAndOrderExtrinsic("""
+         orderNo, Ship To, lastName,  Extrinsic Key 1, Extrinsic Value 1, Extrinsic Key 2, Extrinsic Value 2, SKU 1, Quantity 1, Line Extrinsic Key 1, Line Extrinsic Value 1, Line Extrinsic Key 2, Line Extrinsic Value 2, SKU 2,     Quantity 2, Line Extrinsic Key 1, Line Extrinsic Value 1, SKU 3, Quantity 3, Line Extrinsic Key 1, Line Extrinsic Value 1, Line Extrinsic Key 2
+         1,       Homer,   Simpson,   Store Name,      QQQ Mart,          Coupon Code,     10QOff,            DONUT, 12,         Flavor,               Chocolate,              Size,                 Large,                  BEER,      500,        Flavor,               Hops,                   COUCH, 1,          Color,                Brown,                  foo,
+         2,       Ned,     Flanders,  ,                ,                  ,                ,                  BIBLE, 7,          Flavor,               King James,             Size,                 X-Large,                LAWNMOWER, 1
+         """);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testOrderLinesWithLineExtrinsicsAndOrderExtrinsicWithDupes() throws QException
+   {
+      testOrderLinesWithLineExtrinsicsAndOrderExtrinsic("""
+         orderNo, Ship To, lastName,  Extrinsic Key, Extrinsic Value, Extrinsic Key, Extrinsic Value, SKU,   Quantity, Line Extrinsic Key, Line Extrinsic Value, Line Extrinsic Key, Line Extrinsic Value, SKU,       Quantity, Line Extrinsic Key, Line Extrinsic Value, SKU,   Quantity, Line Extrinsic Key, Line Extrinsic Value, Line Extrinsic Key
+         1,       Homer,   Simpson,   Store Name,    QQQ Mart,        Coupon Code,   10QOff,          DONUT, 12,       Flavor,             Chocolate,            Size,               Large,                BEER,      500,      Flavor,             Hops,                 COUCH, 1,        Color,              Brown,                foo
+         2,       Ned,     Flanders,  ,              ,                ,              ,                BIBLE, 7,        Flavor,             King James,           Size,               X-Large,              LAWNMOWER, 1
+         """);
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   private void testOrderLinesWithLineExtrinsicsAndOrderExtrinsic(String csv) throws QException
+   {
+      Integer defaultStoreId        = 42;
+      String  defaultLineNo         = "47";
+      String  defaultLineExtraValue = "bar";
+
+      CsvFileToRows   fileToRows = CsvFileToRows.forString(csv);
+      BulkLoadFileRow header     = fileToRows.next();
+
+      WideRowsToRecordWithSpreadMapping rowsToRecord = new WideRowsToRecordWithSpreadMapping();
+
+      BulkInsertMapping mapping = new BulkInsertMapping()
+         .withFieldNameToHeaderNameMap(Map.of(
+            "orderNo", "orderNo",
+            "shipToName", "Ship To",
+            "orderLine.sku", "SKU",
+            "orderLine.quantity", "Quantity",
+            "extrinsics.key", "Extrinsic Key",
+            "extrinsics.value", "Extrinsic Value",
+            "orderLine.extrinsics.key", "Line Extrinsic Key",
+            "orderLine.extrinsics.value", "Line Extrinsic Value"
+         ))
+         .withMappedAssociations(List.of("orderLine", "extrinsics", "orderLine.extrinsics"))
+         .withFieldNameToValueMapping(Map.of("orderLine.extrinsics.value", Map.of("Large", "L", "X-Large", "XL")))
+         .withFieldNameToDefaultValueMap(Map.of(
+            "storeId", defaultStoreId,
+            "orderLine.lineNumber", defaultLineNo,
+            "orderLine.extrinsics.value", defaultLineExtraValue
+         ))
+         .withTableName(TestUtils.TABLE_NAME_ORDER)
+         .withLayout(BulkInsertMapping.Layout.WIDE)
+         .withHasHeaderRow(true);
+
+      List<QRecord> records = rowsToRecord.nextPage(fileToRows, header, mapping, Integer.MAX_VALUE);
+      assertEquals(2, records.size());
+
+      QRecord order = records.get(0);
+      assertEquals(1, order.getValueInteger("orderNo"));
+      assertEquals("Homer", order.getValueString("shipToName"));
+      assertEquals(defaultStoreId, order.getValue("storeId"));
+      assertEquals(List.of("DONUT", "BEER", "COUCH"), getValues(order.getAssociatedRecords().get("orderLine"), "sku"));
+      assertEquals(List.of(12, 500, 1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity"));
+      assertEquals(List.of(defaultLineNo, defaultLineNo, defaultLineNo), getValues(order.getAssociatedRecords().get("orderLine"), "lineNumber"));
+      assertEquals(List.of("Store Name", "Coupon Code"), getValues(order.getAssociatedRecords().get("extrinsics"), "key"));
+      assertEquals(List.of("QQQ Mart", "10QOff"), getValues(order.getAssociatedRecords().get("extrinsics"), "value"));
+
+      QRecord lineItem = order.getAssociatedRecords().get("orderLine").get(0);
+      assertEquals(List.of("Flavor", "Size"), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "key"));
+      assertEquals(List.of("Chocolate", "L"), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "value"));
+
+      lineItem = order.getAssociatedRecords().get("orderLine").get(1);
+      assertEquals(List.of("Flavor"), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "key"));
+      assertEquals(List.of("Hops"), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "value"));
+
+      lineItem = order.getAssociatedRecords().get("orderLine").get(2);
+      assertEquals(List.of("Color", "foo"), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "key"));
+      assertEquals(List.of("Brown", defaultLineExtraValue), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "value"));
+
+      order = records.get(1);
+      assertEquals(2, order.getValueInteger("orderNo"));
+      assertEquals("Ned", order.getValueString("shipToName"));
+      assertEquals(defaultStoreId, order.getValue("storeId"));
+      assertEquals(List.of("BIBLE", "LAWNMOWER"), getValues(order.getAssociatedRecords().get("orderLine"), "sku"));
+      assertEquals(List.of(7, 1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity"));
+      assertEquals(List.of(defaultLineNo, defaultLineNo), getValues(order.getAssociatedRecords().get("orderLine"), "lineNumber"));
+      assertThat(order.getAssociatedRecords().get("extrinsics")).isNullOrEmpty();
+
+      lineItem = order.getAssociatedRecords().get("orderLine").get(0);
+      assertEquals(List.of("Flavor", "Size"), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "key"));
+      assertEquals(List.of("King James", "XL"), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "value"));
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   private List<Serializable> getValues(List<QRecord> records, String fieldName)
+   {
+      return (records.stream().map(r -> r.getValue(fieldName)).toList());
+   }
+
+}

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/NoopLoadStepTest.java

@@ -0,0 +1,50 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend;
+
+
+import com.kingsrook.qqq.backend.core.BaseTest;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
+import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
+import org.junit.jupiter.api.Test;
+
+
+/*******************************************************************************
+ ** Unit test for NoopLoadStep 
+ *******************************************************************************/
+class NoopLoadStepTest extends BaseTest
+{
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void test() throws QException
+   {
+      //////////////////////////////////////
+      // sorry, just here for coverage... //
+      //////////////////////////////////////
+      new NoopLoadStep().runOnePage(new RunBackendStepInput(), new RunBackendStepOutput());
+   }
+
+}

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/savedbulkloadprofiles/SavedBulkLoadProfileProcessTests.java

@@ -0,0 +1,186 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.processes.implementations.savedbulkloadprofiles;
+
+
+import java.util.List;
+import com.kingsrook.qqq.backend.core.BaseTest;
+import com.kingsrook.qqq.backend.core.actions.processes.RunProcessAction;
+import com.kingsrook.qqq.backend.core.context.QContext;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.exceptions.QUserFacingException;
+import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput;
+import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessOutput;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.savedbulkloadprofiles.SavedBulkLoadProfileMetaDataProvider;
+import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadProfile;
+import com.kingsrook.qqq.backend.core.utils.JsonUtils;
+import com.kingsrook.qqq.backend.core.utils.TestUtils;
+import org.junit.jupiter.api.Test;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+
+
+/*******************************************************************************
+ ** Unit tests for all saved-bulk-load-profile processes
+ *******************************************************************************/
+class SavedBulkLoadProfileProcessTests extends BaseTest
+{
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void test() throws QException
+   {
+      QInstance qInstance = QContext.getQInstance();
+      new SavedBulkLoadProfileMetaDataProvider().defineAll(qInstance, TestUtils.MEMORY_BACKEND_NAME, null);
+      String tableName = TestUtils.TABLE_NAME_PERSON_MEMORY;
+
+      {
+         ////////////////////////////////////////////
+         // query - should be no profiles to start //
+         ////////////////////////////////////////////
+         RunProcessInput runProcessInput = new RunProcessInput();
+         runProcessInput.setProcessName(QuerySavedBulkLoadProfileProcess.getProcessMetaData().getName());
+         runProcessInput.addValue("tableName", tableName);
+         RunProcessOutput runProcessOutput = new RunProcessAction().execute(runProcessInput);
+         assertEquals(0, ((List<?>) runProcessOutput.getValues().get("savedBulkLoadProfileList")).size());
+      }
+
+      Integer savedBulkLoadProfileId;
+      {
+         /////////////////////////
+         // store a new profile //
+         /////////////////////////
+         RunProcessInput runProcessInput = new RunProcessInput();
+         runProcessInput.setProcessName(StoreSavedBulkLoadProfileProcess.getProcessMetaData().getName());
+         runProcessInput.addValue("label", "My Profile");
+         runProcessInput.addValue("tableName", tableName);
+         runProcessInput.addValue("mappingJson", JsonUtils.toJson(new BulkLoadProfile()));
+         RunProcessOutput runProcessOutput = new RunProcessAction().execute(runProcessInput);
+         List<QRecord>    savedBulkLoadProfileList    = (List<QRecord>) runProcessOutput.getValues().get("savedBulkLoadProfileList");
+         assertEquals(1, savedBulkLoadProfileList.size());
+         savedBulkLoadProfileId = savedBulkLoadProfileList.get(0).getValueInteger("id");
+         assertNotNull(savedBulkLoadProfileId);
+
+         //////////////////////////////////////////////////////////////////
+         // try to store it again - should throw a "duplicate" exception //
+         //////////////////////////////////////////////////////////////////
+         assertThatThrownBy(() -> new RunProcessAction().execute(runProcessInput))
+            .isInstanceOf(QUserFacingException.class)
+            .hasMessageContaining("already have a saved Bulk Load Profile");
+      }
+
+      {
+         //////////////////////////////////////
+         // query - should find our profiles //
+         //////////////////////////////////////
+         RunProcessInput runProcessInput = new RunProcessInput();
+         runProcessInput.setProcessName(QuerySavedBulkLoadProfileProcess.getProcessMetaData().getName());
+         runProcessInput.addValue("tableName", tableName);
+         RunProcessOutput runProcessOutput = new RunProcessAction().execute(runProcessInput);
+         List<QRecord>    savedBulkLoadProfileList    = (List<QRecord>) runProcessOutput.getValues().get("savedBulkLoadProfileList");
+         assertEquals(1, savedBulkLoadProfileList.size());
+         assertEquals(1, savedBulkLoadProfileList.get(0).getValueInteger("id"));
+         assertEquals("My Profile", savedBulkLoadProfileList.get(0).getValueString("label"));
+      }
+
+      {
+         ////////////////////////
+         // update our Profile //
+         ////////////////////////
+         RunProcessInput runProcessInput = new RunProcessInput();
+         runProcessInput.setProcessName(StoreSavedBulkLoadProfileProcess.getProcessMetaData().getName());
+         runProcessInput.addValue("id", savedBulkLoadProfileId);
+         runProcessInput.addValue("label", "My Updated Profile");
+         runProcessInput.addValue("tableName", tableName);
+         runProcessInput.addValue("mappingJson", JsonUtils.toJson(new BulkLoadProfile()));
+         RunProcessOutput runProcessOutput = new RunProcessAction().execute(runProcessInput);
+         List<QRecord>    savedBulkLoadProfileList    = (List<QRecord>) runProcessOutput.getValues().get("savedBulkLoadProfileList");
+         assertEquals(1, savedBulkLoadProfileList.size());
+         assertEquals(1, savedBulkLoadProfileList.get(0).getValueInteger("id"));
+         assertEquals("My Updated Profile", savedBulkLoadProfileList.get(0).getValueString("label"));
+      }
+
+      Integer anotherSavedProfileId;
+      {
+         /////////////////////////////////////////////////////////////////////////////////////////////
+         // store a second one w/ different name (will be used below in update-dupe-check use-case) //
+         /////////////////////////////////////////////////////////////////////////////////////////////
+         RunProcessInput runProcessInput = new RunProcessInput();
+         runProcessInput.setProcessName(StoreSavedBulkLoadProfileProcess.getProcessMetaData().getName());
+         runProcessInput.addValue("label", "My Second Profile");
+         runProcessInput.addValue("tableName", tableName);
+         runProcessInput.addValue("mappingJson", JsonUtils.toJson(new BulkLoadProfile()));
+         RunProcessOutput runProcessOutput = new RunProcessAction().execute(runProcessInput);
+         List<QRecord>    savedBulkLoadProfileList    = (List<QRecord>) runProcessOutput.getValues().get("savedBulkLoadProfileList");
+         anotherSavedProfileId = savedBulkLoadProfileList.get(0).getValueInteger("id");
+      }
+
+      {
+         /////////////////////////////////////////////////
+         // try to rename the second to match the first //
+         /////////////////////////////////////////////////
+         RunProcessInput runProcessInput = new RunProcessInput();
+         runProcessInput.setProcessName(StoreSavedBulkLoadProfileProcess.getProcessMetaData().getName());
+         runProcessInput.addValue("id", anotherSavedProfileId);
+         runProcessInput.addValue("label", "My Updated Profile");
+         runProcessInput.addValue("tableName", tableName);
+         runProcessInput.addValue("mappingJson", JsonUtils.toJson(new BulkLoadProfile()));
+
+         //////////////////////////////////////////
+         // should throw a "duplicate" exception //
+         //////////////////////////////////////////
+         assertThatThrownBy(() -> new RunProcessAction().execute(runProcessInput))
+            .isInstanceOf(QUserFacingException.class)
+            .hasMessageContaining("already have a saved Bulk Load Profile");
+      }
+
+      {
+         /////////////////////////
+         // delete our profiles //
+         /////////////////////////
+         RunProcessInput runProcessInput = new RunProcessInput();
+         runProcessInput.setProcessName(DeleteSavedBulkLoadProfileProcess.getProcessMetaData().getName());
+         runProcessInput.addValue("id", savedBulkLoadProfileId);
+         RunProcessOutput runProcessOutput = new RunProcessAction().execute(runProcessInput);
+
+         runProcessInput.addValue("id", anotherSavedProfileId);
+         runProcessOutput = new RunProcessAction().execute(runProcessInput);
+      }
+
+      {
+         /////////////////////////////////////////
+         // query - should be no profiles again //
+         /////////////////////////////////////////
+         RunProcessInput runProcessInput = new RunProcessInput();
+         runProcessInput.setProcessName(QuerySavedBulkLoadProfileProcess.getProcessMetaData().getName());
+         runProcessInput.addValue("tableName", tableName);
+         RunProcessOutput runProcessOutput = new RunProcessAction().execute(runProcessInput);
+         assertEquals(0, ((List<?>) runProcessOutput.getValues().get("savedBulkLoadProfileList")).size());
+      }
+   }
+
+}

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/locks/ProcessLockUtilsTest.java

@@ -25,13 +25,17 @@ package com.kingsrook.qqq.backend.core.processes.locks;
 import java.time.Duration;
 import java.time.Instant;
 import java.time.temporal.ChronoUnit;
+import java.util.ArrayList;
 import java.util.List;
+import java.util.Map;
 import java.util.concurrent.TimeUnit;
 import com.kingsrook.qqq.backend.core.BaseTest;
 import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
 import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.instances.QInstanceValidator;
+import com.kingsrook.qqq.backend.core.logging.QCollectingLogger;
+import com.kingsrook.qqq.backend.core.logging.QLogger;
 import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
 import com.kingsrook.qqq.backend.core.model.metadata.MetaDataProducerMultiOutput;
 import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
@@ -39,6 +43,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 import com.kingsrook.qqq.backend.core.model.session.QUser;
 import com.kingsrook.qqq.backend.core.utils.SleepUtils;
 import com.kingsrook.qqq.backend.core.utils.TestUtils;
+import com.kingsrook.qqq.backend.core.utils.collections.ListBuilder;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
 import static org.assertj.core.api.Assertions.assertThat;
@@ -227,6 +232,28 @@ class ProcessLockUtilsTest extends BaseTest
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testReleaseBadInputs()
+   {
+      //////////////////////////////////////////////////////////
+      // make sure we don't blow up, just noop in these cases //
+      //////////////////////////////////////////////////////////
+      QCollectingLogger qCollectingLogger = QLogger.activateCollectingLoggerForClass(ProcessLockUtils.class);
+      ProcessLockUtils.releaseById(null);
+      ProcessLockUtils.release(null);
+      ProcessLockUtils.releaseMany(null);
+      ProcessLockUtils.releaseByIds(null);
+      ProcessLockUtils.releaseMany(ListBuilder.of(null));
+      ProcessLockUtils.releaseByIds(ListBuilder.of(null));
+      QLogger.deactivateCollectingLoggerForClass(ProcessLockUtils.class);
+      assertEquals(6, qCollectingLogger.getCollectedMessages().stream().filter(m -> m.getMessage().contains("noop")).count());
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/
@@ -304,7 +331,7 @@ class ProcessLockUtilsTest extends BaseTest
       //////////////////////////////////////////////
       // checkin w/ a time - sets it to that time //
       //////////////////////////////////////////////
-      Instant specifiedTime = Instant.now();
+      Instant specifiedTime = Instant.now().plusSeconds(47);
       ProcessLockUtils.checkIn(processLock, specifiedTime);
       processLock = ProcessLockUtils.getById(processLock.getId());
       assertEquals(specifiedTime, processLock.getExpiresAtTimestamp());
@@ -380,4 +407,122 @@ class ProcessLockUtilsTest extends BaseTest
       assertNull(processLock.getExpiresAtTimestamp());
    }
 
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testMany() throws QException
+   {
+      /////////////////////////////////////////////////
+      // make sure that we can create multiple locks //
+      /////////////////////////////////////////////////
+      List<String>                                         keys         = List.of("1", "2", "3");
+      List<ProcessLock>                   processLocks = new ArrayList<>();
+      Map<String, ProcessLockOrException> results      = ProcessLockUtils.createMany(keys, "typeA", "me");
+      for(String key : keys)
+      {
+         ProcessLock processLock = results.get(key).processLock();
+         assertNotNull(processLock.getId());
+         assertNotNull(processLock.getCheckInTimestamp());
+         assertNull(processLock.getExpiresAtTimestamp());
+         processLocks.add(processLock);
+      }
+
+      /////////////////////////////////////////////////////////
+      // make sure we can't create a second for the same key //
+      /////////////////////////////////////////////////////////
+      assertThatThrownBy(() -> ProcessLockUtils.create("1", "typeA", "you"))
+         .isInstanceOf(UnableToObtainProcessLockException.class)
+         .hasMessageContaining("Held by: " + QContext.getQSession().getUser().getIdReference())
+         .hasMessageContaining("with details: me")
+         .hasMessageNotContaining("expiring at: 20")
+         .matches(e -> ((UnableToObtainProcessLockException) e).getExistingLock() != null);
+
+      /////////////////////////////////////////////////////////
+      // make sure we can create another for a different key //
+      /////////////////////////////////////////////////////////
+      ProcessLockUtils.create("4", "typeA", "him");
+
+      /////////////////////////////////////////////////////////////////////
+      // make sure we can create another for a different type (same key) //
+      /////////////////////////////////////////////////////////////////////
+      ProcessLockUtils.create("1", "typeB", "her");
+
+      ////////////////////////////////////////////////////////////////////
+      // now try to create some that will overlap, but one that'll work //
+      ////////////////////////////////////////////////////////////////////
+      keys = List.of("3", "4", "5");
+      results = ProcessLockUtils.createMany(keys, "typeA", "me");
+      for(String key : List.of("3", "4"))
+      {
+         UnableToObtainProcessLockException exception = results.get(key).unableToObtainProcessLockException();
+         assertNotNull(exception);
+      }
+
+      ProcessLock processLock = results.get("5").processLock();
+      assertNotNull(processLock.getId());
+      assertNotNull(processLock.getCheckInTimestamp());
+      assertNull(processLock.getExpiresAtTimestamp());
+      processLocks.add(processLock);
+
+      //////////////////////////////
+      // make sure we can release //
+      //////////////////////////////
+      ProcessLockUtils.releaseMany(processLocks);
+
+      ///////////////////////////////////////////////////////
+      // make sure can re-lock 1 now after it was released //
+      ///////////////////////////////////////////////////////
+      processLock = ProcessLockUtils.create("1", "typeA", "you");
+      assertNotNull(processLock.getId());
+      assertEquals("you", processLock.getDetails());
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testManyWithSleep() throws QException
+   {
+      /////////////////////////////////////////////////
+      // make sure that we can create multiple locks //
+      /////////////////////////////////////////////////
+      List<String>                        keys     = List.of("1", "2", "3");
+      Map<String, ProcessLockOrException> results0 = ProcessLockUtils.createMany(keys, "typeB", "me");
+      for(String key : keys)
+      {
+         assertNotNull(results0.get(key).processLock());
+      }
+
+      ////////////////////////////////////////////////////////////
+      // try again - and 2 and 3 should fail, if we don't sleep //
+      ////////////////////////////////////////////////////////////
+      keys = List.of("2", "3", "4");
+      Map<String, ProcessLockOrException> results1 = ProcessLockUtils.createMany(keys, "typeB", "you");
+      assertNull(results1.get("2").processLock());
+      assertNull(results1.get("3").processLock());
+      assertNotNull(results1.get("4").processLock());
+
+      ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      // try another insert, which should initially succeed for #5, then sleep, and eventually succeed on 3 & 4 as well //
+      ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      keys = List.of("3", "4", "5");
+      Map<String, ProcessLockOrException> results2 = ProcessLockUtils.createMany(keys, "typeB", "them", Duration.of(1, ChronoUnit.SECONDS), Duration.of(3, ChronoUnit.SECONDS));
+      for(String key : keys)
+      {
+         assertNotNull(results2.get(key).processLock());
+      }
+
+      ////////////////////////////////////////////////////////////////////////////////////////////////
+      // make sure that we have a different ids for some that expired and then succeeded post-sleep //
+      ////////////////////////////////////////////////////////////////////////////////////////////////
+      assertNotEquals(results0.get("3").processLock().getId(), results2.get("3").processLock().getId());
+      assertNotEquals(results1.get("4").processLock().getId(), results2.get("4").processLock().getId());
+
+   }
+
 }

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/StringUtilsTest.java

@@ -105,6 +105,7 @@ class StringUtilsTest extends BaseTest
       assertEquals("Foo bar", StringUtils.allCapsToMixedCase("FOo bar"));
       assertEquals("Foo Bar", StringUtils.allCapsToMixedCase("FOo BAr"));
       assertEquals("foo bar", StringUtils.allCapsToMixedCase("foo bar"));
+      assertEquals("Foo Bar", StringUtils.allCapsToMixedCase("FOO_BAR"));
    }
 
 

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java

@@ -70,6 +70,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.dashboard.QWidgetMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.DisplayFormat;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.ValueTooLongBehavior;
 import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinOn;
 import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinType;
 import com.kingsrook.qqq.backend.core.model.metadata.joins.QJoinMetaData;
@@ -142,6 +143,7 @@ public class TestUtils
    public static final String APP_NAME_MISCELLANEOUS = "miscellaneous";
 
    public static final String TABLE_NAME_TWO_KEYS            = "twoKeys";
+   public static final String TABLE_NAME_MEMORY_STORAGE      = "memoryStorage";
    public static final String TABLE_NAME_PERSON              = "person";
    public static final String TABLE_NAME_SHAPE               = "shape";
    public static final String TABLE_NAME_SHAPE_CACHE         = "shapeCache";
@@ -204,6 +206,7 @@ public class TestUtils
 
       qInstance.addTable(defineTablePerson());
       qInstance.addTable(defineTableTwoKeys());
+      qInstance.addTable(defineTableMemoryStorage());
       qInstance.addTable(definePersonFileTable());
       qInstance.addTable(definePersonMemoryTable());
       qInstance.addTable(definePersonMemoryCacheTable());
@@ -594,6 +597,22 @@ public class TestUtils
 
 
 
+   /*******************************************************************************
+    ** Define a table in the memory store that can be used for the StorageAction
+    *******************************************************************************/
+   public static QTableMetaData defineTableMemoryStorage()
+   {
+      return new QTableMetaData()
+         .withName(TABLE_NAME_MEMORY_STORAGE)
+         .withLabel("Memory Storage")
+         .withBackendName(MEMORY_BACKEND_NAME)
+         .withPrimaryKeyField("reference")
+         .withField(new QFieldMetaData("reference", QFieldType.STRING).withIsEditable(false))
+         .withField(new QFieldMetaData("contents", QFieldType.BLOB));
+   }
+
+
+
    /*******************************************************************************
     ** Define the 'person' table used in standard tests.
     *******************************************************************************/
@@ -644,6 +663,7 @@ public class TestUtils
          .withField(new QFieldMetaData("createDate", QFieldType.DATE_TIME).withIsEditable(false))
          .withField(new QFieldMetaData("modifyDate", QFieldType.DATE_TIME).withIsEditable(false))
          .withField(new QFieldMetaData("orderNo", QFieldType.STRING))
+         .withField(new QFieldMetaData("shipToName", QFieldType.STRING).withMaxLength(200).withBehavior(ValueTooLongBehavior.ERROR))
          .withField(new QFieldMetaData("orderDate", QFieldType.DATE))
          .withField(new QFieldMetaData("storeId", QFieldType.INTEGER))
          .withField(new QFieldMetaData("total", QFieldType.DECIMAL).withDisplayFormat(DisplayFormat.CURRENCY).withFieldSecurityLock(new FieldSecurityLock()
@@ -700,7 +720,8 @@ public class TestUtils
          .withField(new QFieldMetaData("modifyDate", QFieldType.DATE_TIME).withIsEditable(false))
          .withField(new QFieldMetaData("lineItemId", QFieldType.INTEGER))
          .withField(new QFieldMetaData("key", QFieldType.STRING))
-         .withField(new QFieldMetaData("value", QFieldType.STRING));
+         .withField(new QFieldMetaData("value", QFieldType.STRING))
+         .withField(new QFieldMetaData("source", QFieldType.STRING)); // doesn't really make sense, but useful to have an extra field here in some bulk-load tests
    }
 
 

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/ValueUtilsTest.java

@@ -251,6 +251,9 @@ class ValueUtilsTest extends BaseTest
       assertThrows(QValueException.class, () -> ValueUtils.getValueAsInstant("a,b"));
       assertThrows(QValueException.class, () -> ValueUtils.getValueAsInstant("1980/05/31"));
       assertThat(assertThrows(QValueException.class, () -> ValueUtils.getValueAsInstant(new Object())).getMessage()).contains("Unsupported class");
+
+      expected = Instant.parse("1980-05-31T01:30:00Z");
+      assertEquals(expected, ValueUtils.getValueAsInstant("1980-05-31 1:30:00"));
    }
 
 

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/aggregates/AggregatesTest.java

@@ -91,6 +91,57 @@ class AggregatesTest extends BaseTest
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testLong()
+   {
+      LongAggregates aggregates = new LongAggregates();
+
+      assertEquals(0, aggregates.getCount());
+      assertNull(aggregates.getMin());
+      assertNull(aggregates.getMax());
+      assertNull(aggregates.getSum());
+      assertNull(aggregates.getAverage());
+
+      aggregates.add(5L);
+      assertEquals(1, aggregates.getCount());
+      assertEquals(5, aggregates.getMin());
+      assertEquals(5, aggregates.getMax());
+      assertEquals(5, aggregates.getSum());
+      assertThat(aggregates.getAverage()).isCloseTo(new BigDecimal("5"), Offset.offset(BigDecimal.ZERO));
+
+      aggregates.add(10L);
+      assertEquals(2, aggregates.getCount());
+      assertEquals(5, aggregates.getMin());
+      assertEquals(10, aggregates.getMax());
+      assertEquals(15, aggregates.getSum());
+      assertThat(aggregates.getAverage()).isCloseTo(new BigDecimal("7.5"), Offset.offset(BigDecimal.ZERO));
+
+      aggregates.add(15L);
+      assertEquals(3, aggregates.getCount());
+      assertEquals(5, aggregates.getMin());
+      assertEquals(15, aggregates.getMax());
+      assertEquals(30, aggregates.getSum());
+      assertThat(aggregates.getAverage()).isCloseTo(new BigDecimal("10"), Offset.offset(BigDecimal.ZERO));
+
+      aggregates.add(null);
+      assertEquals(3, aggregates.getCount());
+      assertEquals(5, aggregates.getMin());
+      assertEquals(15, aggregates.getMax());
+      assertEquals(30, aggregates.getSum());
+      assertThat(aggregates.getAverage()).isCloseTo(new BigDecimal("10"), Offset.offset(BigDecimal.ZERO));
+
+      assertEquals(new BigDecimal("750"), aggregates.getProduct());
+      assertEquals(new BigDecimal("25.0000"), aggregates.getVariance());
+      assertEquals(new BigDecimal("5.0000"), aggregates.getStandardDeviation());
+      assertThat(aggregates.getVarP()).isCloseTo(new BigDecimal("16.6667"), Offset.offset(new BigDecimal(".0001")));
+      assertThat(aggregates.getStdDevP()).isCloseTo(new BigDecimal("4.0824"), Offset.offset(new BigDecimal(".0001")));
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/collections/CaseInsensitiveKeyMapTest.java

@@ -0,0 +1,52 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.utils.collections;
+
+
+import com.kingsrook.qqq.backend.core.BaseTest;
+import org.junit.jupiter.api.Test;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+
+/*******************************************************************************
+ ** Unit test for CaseInsensitiveKeyMap 
+ *******************************************************************************/
+class CaseInsensitiveKeyMapTest extends BaseTest
+{
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void test()
+   {
+      CaseInsensitiveKeyMap<Integer> map = new CaseInsensitiveKeyMap<>();
+      map.put("One", 1);
+      map.put("one", 1);
+      map.put("ONE", 1);
+      assertEquals(1, map.get("one"));
+      assertEquals(1, map.get("One"));
+      assertEquals(1, map.get("oNe"));
+      assertEquals(1, map.size());
+   }
+
+}

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/collections/TransformedKeyMapTest.java

@@ -0,0 +1,201 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.utils.collections;
+
+
+import java.math.BigDecimal;
+import java.util.Map;
+import com.kingsrook.qqq.backend.core.BaseTest;
+import org.junit.jupiter.api.Test;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNull;
+
+
+/*******************************************************************************
+ ** Unit test for TransformedKeyMap 
+ *******************************************************************************/
+@SuppressWarnings({ "RedundantCollectionOperation", "RedundantOperationOnEmptyContainer" })
+class TransformedKeyMapTest extends BaseTest
+{
+   private static final BigDecimal BIG_DECIMAL_TWO   = BigDecimal.valueOf(2);
+   private static final BigDecimal BIG_DECIMAL_THREE = BigDecimal.valueOf(3);
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testCaseInsensitiveKeyMap()
+   {
+      TransformedKeyMap<String, String, Integer> caseInsensitiveKeys = new TransformedKeyMap<>(key -> key.toLowerCase());
+      caseInsensitiveKeys.put("One", 1);
+      caseInsensitiveKeys.put("one", 1);
+      caseInsensitiveKeys.put("ONE", 1);
+      assertEquals(1, caseInsensitiveKeys.get("one"));
+      assertEquals(1, caseInsensitiveKeys.get("One"));
+      assertEquals(1, caseInsensitiveKeys.get("oNe"));
+      assertEquals(1, caseInsensitiveKeys.size());
+
+      //////////////////////////////////////////////////
+      // get back the first way it was put in the map //
+      //////////////////////////////////////////////////
+      assertEquals("One", caseInsensitiveKeys.entrySet().iterator().next().getKey());
+      assertEquals("One", caseInsensitiveKeys.keySet().iterator().next());
+
+      assertEquals(1, caseInsensitiveKeys.entrySet().size());
+      assertEquals(1, caseInsensitiveKeys.keySet().size());
+
+      for(String key : caseInsensitiveKeys.keySet())
+      {
+         assertEquals(1, caseInsensitiveKeys.get(key));
+      }
+
+      for(Map.Entry<String, Integer> entry : caseInsensitiveKeys.entrySet())
+      {
+         assertEquals("One", entry.getKey());
+         assertEquals(1, entry.getValue());
+      }
+
+      /////////////////////////////
+      // add a second unique key //
+      /////////////////////////////
+      caseInsensitiveKeys.put("Two", 2);
+      assertEquals(2, caseInsensitiveKeys.size());
+      assertEquals(2, caseInsensitiveKeys.entrySet().size());
+      assertEquals(2, caseInsensitiveKeys.keySet().size());
+
+      ////////////////////////////////////////
+      // make sure remove works as expected //
+      ////////////////////////////////////////
+      caseInsensitiveKeys.remove("TWO");
+      assertNull(caseInsensitiveKeys.get("Two"));
+      assertNull(caseInsensitiveKeys.get("two"));
+      assertEquals(1, caseInsensitiveKeys.size());
+      assertEquals(1, caseInsensitiveKeys.keySet().size());
+      assertEquals(1, caseInsensitiveKeys.entrySet().size());
+
+      ///////////////////////////////////////
+      // make sure clear works as expected //
+      ///////////////////////////////////////
+      caseInsensitiveKeys.clear();
+      assertNull(caseInsensitiveKeys.get("one"));
+      assertEquals(0, caseInsensitiveKeys.size());
+      assertEquals(0, caseInsensitiveKeys.keySet().size());
+      assertEquals(0, caseInsensitiveKeys.entrySet().size());
+
+      /////////////////////////////////////////
+      // make sure put-all works as expected //
+      /////////////////////////////////////////
+      caseInsensitiveKeys.putAll(Map.of("One", 1, "one", 1, "ONE", 1, "TwO", 2, "tWo", 2, "three", 3));
+      assertEquals(1, caseInsensitiveKeys.get("oNe"));
+      assertEquals(2, caseInsensitiveKeys.get("two"));
+      assertEquals(3, caseInsensitiveKeys.get("Three"));
+      assertEquals(3, caseInsensitiveKeys.size());
+      assertEquals(3, caseInsensitiveKeys.entrySet().size());
+      assertEquals(3, caseInsensitiveKeys.keySet().size());
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testStringToNumberMap()
+   {
+      TransformedKeyMap<String, Integer, BigDecimal> multiLingualWordToNumber = new TransformedKeyMap<>(key -> switch(key.toLowerCase())
+      {
+         case "one", "uno", "eins" -> 1;
+         case "two", "dos", "zwei" -> 2;
+         case "three", "tres", "drei" -> 3;
+         default -> null;
+      });
+      multiLingualWordToNumber.put("One", BigDecimal.ONE);
+      multiLingualWordToNumber.put("uno", BigDecimal.ONE);
+      assertEquals(BigDecimal.ONE, multiLingualWordToNumber.get("one"));
+      assertEquals(BigDecimal.ONE, multiLingualWordToNumber.get("uno"));
+      assertEquals(BigDecimal.ONE, multiLingualWordToNumber.get("eins"));
+      assertEquals(1, multiLingualWordToNumber.size());
+
+      //////////////////////////////////////////////////
+      // get back the first way it was put in the map //
+      //////////////////////////////////////////////////
+      assertEquals("One", multiLingualWordToNumber.entrySet().iterator().next().getKey());
+      assertEquals("One", multiLingualWordToNumber.keySet().iterator().next());
+
+      assertEquals(1, multiLingualWordToNumber.entrySet().size());
+      assertEquals(1, multiLingualWordToNumber.keySet().size());
+
+      for(String key : multiLingualWordToNumber.keySet())
+      {
+         assertEquals(BigDecimal.ONE, multiLingualWordToNumber.get(key));
+      }
+
+      for(Map.Entry<String, BigDecimal> entry : multiLingualWordToNumber.entrySet())
+      {
+         assertEquals("One", entry.getKey());
+         assertEquals(BigDecimal.ONE, entry.getValue());
+      }
+
+      /////////////////////////////
+      // add a second unique key //
+      /////////////////////////////
+      multiLingualWordToNumber.put("Two", BIG_DECIMAL_TWO);
+      assertEquals(BIG_DECIMAL_TWO, multiLingualWordToNumber.get("Dos"));
+      assertEquals(2, multiLingualWordToNumber.size());
+      assertEquals(2, multiLingualWordToNumber.entrySet().size());
+      assertEquals(2, multiLingualWordToNumber.keySet().size());
+
+      ////////////////////////////////////////
+      // make sure remove works as expected //
+      ////////////////////////////////////////
+      multiLingualWordToNumber.remove("ZWEI");
+      assertNull(multiLingualWordToNumber.get("Two"));
+      assertNull(multiLingualWordToNumber.get("Dos"));
+      assertEquals(1, multiLingualWordToNumber.size());
+      assertEquals(1, multiLingualWordToNumber.keySet().size());
+      assertEquals(1, multiLingualWordToNumber.entrySet().size());
+
+      ///////////////////////////////////////
+      // make sure clear works as expected //
+      ///////////////////////////////////////
+      multiLingualWordToNumber.clear();
+      assertNull(multiLingualWordToNumber.get("eins"));
+      assertNull(multiLingualWordToNumber.get("One"));
+      assertEquals(0, multiLingualWordToNumber.size());
+      assertEquals(0, multiLingualWordToNumber.keySet().size());
+      assertEquals(0, multiLingualWordToNumber.entrySet().size());
+
+      /////////////////////////////////////////
+      // make sure put-all works as expected //
+      /////////////////////////////////////////
+      multiLingualWordToNumber.putAll(Map.of("One", BigDecimal.ONE, "Uno", BigDecimal.ONE, "EINS", BigDecimal.ONE, "dos", BIG_DECIMAL_TWO, "zwei", BIG_DECIMAL_TWO, "tres", BIG_DECIMAL_THREE));
+      assertEquals(BigDecimal.ONE, multiLingualWordToNumber.get("oNe"));
+      assertEquals(BIG_DECIMAL_TWO, multiLingualWordToNumber.get("dos"));
+      assertEquals(BIG_DECIMAL_THREE, multiLingualWordToNumber.get("drei"));
+      assertEquals(3, multiLingualWordToNumber.size());
+      assertEquals(3, multiLingualWordToNumber.entrySet().size());
+      assertEquals(3, multiLingualWordToNumber.keySet().size());
+   }
+
+}