diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertV2Test.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertV2Test.java
new file mode 100644
index 00000000..8beaa891
--- /dev/null
+++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertV2Test.java
@@ -0,0 +1,236 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022. Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+
+package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert;
+
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
+import java.util.function.Consumer;
+import com.kingsrook.qqq.backend.core.BaseTest;
+import com.kingsrook.qqq.backend.core.actions.processes.RunProcessAction;
+import com.kingsrook.qqq.backend.core.actions.tables.StorageAction;
+import com.kingsrook.qqq.backend.core.context.QContext;
+import com.kingsrook.qqq.backend.core.instances.QInstanceEnricher;
+import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput;
+import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessOutput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.storage.StorageInput;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.frontend.QFrontendFieldMetaData;
+import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryRecordStore;
+import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadProfileField;
+import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess;
+import com.kingsrook.qqq.backend.core.utils.JsonUtils;
+import com.kingsrook.qqq.backend.core.utils.TestUtils;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertNull;
+
+
+/*******************************************************************************
+ ** Unit test for full bulk insert process
+ *******************************************************************************/
+class BulkInsertV2Test extends BaseTest
+{
+
+ /*******************************************************************************
+ **
+ *******************************************************************************/
+ @BeforeEach
+ @AfterEach
+ void beforeAndAfterEach()
+ {
+ MemoryRecordStore.getInstance().reset();
+ }
+
+
+
+ /*******************************************************************************
+ **
+ *******************************************************************************/
+ public static String getPersonCsvRow1()
+ {
+ return ("""
+ "0","2021-10-26 14:39:37","2021-10-26 14:39:37","John","Doe","1980-01-01","john@doe.com","Missouri",42
+ """);
+ }
+
+
+
+ /*******************************************************************************
+ **
+ *******************************************************************************/
+ public static String getPersonCsvRow2()
+ {
+ return ("""
+ "0","2021-10-26 14:39:37","2021-10-26 14:39:37","Jane","Doe","1981-01-01","john@doe.com","Illinois",
+ """);
+ }
+
+
+
+ /*******************************************************************************
+ **
+ *******************************************************************************/
+ public static String getPersonCsvHeaderUsingLabels()
+ {
+ return ("""
+ "Id","Create Date","Modify Date","First Name","Last Name","Birth Date","Email","Home State",noOfShoes
+ """);
+ }
+
+
+
+ /*******************************************************************************
+ **
+ *******************************************************************************/
+ @Test
+ void test() throws Exception
+ {
+ String defaultEmail = "noone@kingsrook.com";
+
+ ///////////////////////////////////////
+ // make sure table is empty to start //
+ ///////////////////////////////////////
+ assertThat(TestUtils.queryTable(TestUtils.TABLE_NAME_PERSON_MEMORY)).isEmpty();
+
+ QInstance qInstance = QContext.getQInstance();
+ String processName = "PersonBulkInsertV2";
+ new QInstanceEnricher(qInstance).defineTableBulkInsertV2(qInstance, qInstance.getTable(TestUtils.TABLE_NAME_PERSON_MEMORY), processName);
+
+ /////////////////////////////////////////////////////////
+ // start the process - expect to go to the upload step //
+ /////////////////////////////////////////////////////////
+ RunProcessInput runProcessInput = new RunProcessInput();
+ runProcessInput.setProcessName(processName);
+ RunProcessOutput runProcessOutput = new RunProcessAction().execute(runProcessInput);
+ String processUUID = runProcessOutput.getProcessUUID();
+ assertThat(runProcessOutput.getProcessState().getNextStepName()).isPresent().get().isEqualTo("upload");
+
+ //////////////////////////////
+ // simulate the file upload //
+ //////////////////////////////
+ String storageReference = UUID.randomUUID() + ".csv";
+ StorageInput storageInput = new StorageInput(TestUtils.TABLE_NAME_MEMORY_STORAGE).withReference(storageReference);
+ try(OutputStream outputStream = new StorageAction().createOutputStream(storageInput))
+ {
+ outputStream.write((getPersonCsvHeaderUsingLabels() + getPersonCsvRow1() + getPersonCsvRow2()).getBytes());
+ }
+ catch(IOException e)
+ {
+ throw (e);
+ }
+
+ //////////////////////////
+ // continue post-upload //
+ //////////////////////////
+ runProcessInput.setProcessUUID(processUUID);
+ runProcessInput.setStartAfterStep("upload");
+ runProcessInput.addValue("tableName", TestUtils.TABLE_NAME_PERSON_MEMORY);
+ runProcessInput.addValue("theFile", new ArrayList<>(List.of(storageInput)));
+ runProcessOutput = new RunProcessAction().execute(runProcessInput);
+ assertEquals(List.of("Id", "Create Date", "Modify Date", "First Name", "Last Name", "Birth Date", "Email", "Home State", "noOfShoes"), runProcessOutput.getValue("headerValues"));
+ assertEquals(List.of("A", "B", "C", "D", "E", "F", "G", "H", "I"), runProcessOutput.getValue("headerLetters"));
+ assertThat(runProcessOutput.getProcessState().getNextStepName()).isPresent().get().isEqualTo("fileMapping");
+
+ ////////////////////////////////////////////////////////////////////////////////
+ // all subsequent steps will want these data - so set up a lambda to set them //
+ ////////////////////////////////////////////////////////////////////////////////
+ Consumer addProfileToRunProcessInput = (RunProcessInput input) ->
+ {
+ input.addValue("version", "v1");
+ input.addValue("layout", "FLAT");
+ input.addValue("hasHeaderRow", "true");
+ input.addValue("fieldListJSON", JsonUtils.toJson(List.of(
+ new BulkLoadProfileField().withFieldName("firstName").withColumnIndex(3),
+ new BulkLoadProfileField().withFieldName("lastName").withColumnIndex(4),
+ new BulkLoadProfileField().withFieldName("email").withDefaultValue(defaultEmail),
+ new BulkLoadProfileField().withFieldName("homeStateId").withColumnIndex(7).withDoValueMapping(true),
+ new BulkLoadProfileField().withFieldName("noOfShoes").withColumnIndex(8)
+ )));
+ };
+
+ ////////////////////////////////
+ // continue post file-mapping //
+ ////////////////////////////////
+ runProcessInput.setStartAfterStep("fileMapping");
+ addProfileToRunProcessInput.accept(runProcessInput);
+ runProcessOutput = new RunProcessAction().execute(runProcessInput);
+ Serializable valueMappingField = runProcessOutput.getValue("valueMappingField");
+ assertThat(valueMappingField).isInstanceOf(QFrontendFieldMetaData.class);
+ assertEquals("homeStateId", ((QFrontendFieldMetaData) valueMappingField).getName());
+ assertEquals(List.of("Missouri", "Illinois"), runProcessOutput.getValue("fileValues"));
+ assertEquals(List.of("homeStateId"), runProcessOutput.getValue("fieldNamesToDoValueMapping"));
+ assertEquals(0, runProcessOutput.getValue("valueMappingFieldIndex"));
+ assertThat(runProcessOutput.getProcessState().getNextStepName()).isPresent().get().isEqualTo("valueMapping");
+
+ /////////////////////////////////
+ // continue post value-mapping //
+ /////////////////////////////////
+ runProcessInput.setStartAfterStep("valueMapping");
+ runProcessInput.addValue("mappedValuesJSON", JsonUtils.toJson(Map.of("Illinois", 1, "Missouri", 2)));
+ addProfileToRunProcessInput.accept(runProcessInput);
+ runProcessOutput = new RunProcessAction().execute(runProcessInput);
+ assertThat(runProcessOutput.getProcessState().getNextStepName()).isPresent().get().isEqualTo("review");
+
+ /////////////////////////////////
+ // continue post review screen //
+ /////////////////////////////////
+ runProcessInput.setStartAfterStep("review");
+ addProfileToRunProcessInput.accept(runProcessInput);
+ runProcessOutput = new RunProcessAction().execute(runProcessInput);
+ assertThat(runProcessOutput.getRecords()).hasSize(2);
+ assertThat(runProcessOutput.getProcessState().getNextStepName()).isPresent().get().isEqualTo("result");
+ assertThat(runProcessOutput.getValues().get(StreamedETLWithFrontendProcess.FIELD_PROCESS_SUMMARY)).isNotNull().isInstanceOf(List.class);
+ assertThat(runProcessOutput.getException()).isEmpty();
+
+ ////////////////////////////////////
+ // query for the inserted records //
+ ////////////////////////////////////
+ List records = TestUtils.queryTable(TestUtils.TABLE_NAME_PERSON_MEMORY);
+
+ assertEquals("John", records.get(0).getValueString("firstName"));
+ assertEquals("Jane", records.get(1).getValueString("firstName"));
+
+ assertNotNull(records.get(0).getValue("id"));
+ assertNotNull(records.get(1).getValue("id"));
+
+ assertEquals(2, records.get(0).getValueInteger("homeStateId"));
+ assertEquals(1, records.get(1).getValueInteger("homeStateId"));
+
+ assertEquals(defaultEmail, records.get(0).getValueString("email"));
+ assertEquals(defaultEmail, records.get(1).getValueString("email"));
+
+ assertEquals(42, records.get(0).getValueInteger("noOfShoes"));
+ assertNull(records.get(1).getValue("noOfShoes"));
+ }
+
+}
\ No newline at end of file
diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java
index 8ca6da8f..89adac13 100644
--- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java
+++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java
@@ -142,6 +142,7 @@ public class TestUtils
public static final String APP_NAME_MISCELLANEOUS = "miscellaneous";
public static final String TABLE_NAME_TWO_KEYS = "twoKeys";
+ public static final String TABLE_NAME_MEMORY_STORAGE = "memoryStorage";
public static final String TABLE_NAME_PERSON = "person";
public static final String TABLE_NAME_SHAPE = "shape";
public static final String TABLE_NAME_SHAPE_CACHE = "shapeCache";
@@ -204,6 +205,7 @@ public class TestUtils
qInstance.addTable(defineTablePerson());
qInstance.addTable(defineTableTwoKeys());
+ qInstance.addTable(defineTableMemoryStorage());
qInstance.addTable(definePersonFileTable());
qInstance.addTable(definePersonMemoryTable());
qInstance.addTable(definePersonMemoryCacheTable());
@@ -594,6 +596,22 @@ public class TestUtils
+ /*******************************************************************************
+ ** Define a table in the memory store that can be used for the StorageAction
+ *******************************************************************************/
+ public static QTableMetaData defineTableMemoryStorage()
+ {
+ return new QTableMetaData()
+ .withName(TABLE_NAME_MEMORY_STORAGE)
+ .withLabel("Memory Storage")
+ .withBackendName(MEMORY_BACKEND_NAME)
+ .withPrimaryKeyField("reference")
+ .withField(new QFieldMetaData("reference", QFieldType.STRING).withIsEditable(false))
+ .withField(new QFieldMetaData("contents", QFieldType.BLOB));
+ }
+
+
+
/*******************************************************************************
** Define the 'person' table used in standard tests.
*******************************************************************************/