diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertLoadStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertLoadStep.java index b35bb24b..30f98c91 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertLoadStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertLoadStep.java @@ -25,6 +25,7 @@ package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert; import java.io.Serializable; import java.util.ArrayList; import java.util.List; +import java.util.Objects; import com.kingsrook.qqq.backend.core.context.QContext; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.logging.QLogger; @@ -110,6 +111,10 @@ public class BulkInsertLoadStep extends LoadViaInsertStep implements ProcessSumm if(field.getType().isNumeric()) { ProcessSummaryLine idsLine = new ProcessSummaryLine(Status.INFO, "Inserted " + field.getLabel() + " values between " + firstInsertedPrimaryKey + " and " + lastInsertedPrimaryKey); + if(Objects.equals(firstInsertedPrimaryKey, lastInsertedPrimaryKey)) + { + idsLine.setMessage("Inserted " + field.getLabel() + " " + firstInsertedPrimaryKey); + } idsLine.setCount(null); processSummary.add(idsLine); } diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertFullProcessTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertFullProcessTest.java index 2ab31159..91972a91 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertFullProcessTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertFullProcessTest.java @@ -22,24 +22,21 @@ package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert; -import java.io.IOException; import java.io.OutputStream; import java.io.Serializable; import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.UUID; -import java.util.function.Consumer; import com.kingsrook.qqq.backend.core.BaseTest; import com.kingsrook.qqq.backend.core.actions.processes.RunProcessAction; import com.kingsrook.qqq.backend.core.actions.tables.StorageAction; -import com.kingsrook.qqq.backend.core.context.QContext; -import com.kingsrook.qqq.backend.core.instances.QInstanceEnricher; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryAssert; import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput; import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessOutput; import com.kingsrook.qqq.backend.core.model.actions.tables.storage.StorageInput; import com.kingsrook.qqq.backend.core.model.data.QRecord; -import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.frontend.QFrontendFieldMetaData; import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryRecordStore; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadProfile; @@ -61,6 +58,9 @@ import static org.junit.jupiter.api.Assertions.assertNull; *******************************************************************************/ class BulkInsertFullProcessTest extends BaseTest { + private static final String defaultEmail = "noone@kingsrook.com"; + + /******************************************************************************* ** @@ -116,48 +116,20 @@ class BulkInsertFullProcessTest extends BaseTest @Test void test() throws Exception { - String defaultEmail = "noone@kingsrook.com"; - - /////////////////////////////////////// - // make sure table is empty to start // - /////////////////////////////////////// assertThat(TestUtils.queryTable(TestUtils.TABLE_NAME_PERSON_MEMORY)).isEmpty(); - QInstance qInstance = QContext.getQInstance(); - String processName = "PersonBulkInsertV2"; - new QInstanceEnricher(qInstance).defineTableBulkInsert(qInstance, qInstance.getTable(TestUtils.TABLE_NAME_PERSON_MEMORY), processName); - ///////////////////////////////////////////////////////// // start the process - expect to go to the upload step // ///////////////////////////////////////////////////////// - RunProcessInput runProcessInput = new RunProcessInput(); - runProcessInput.setProcessName(processName); - runProcessInput.addValue("tableName", TestUtils.TABLE_NAME_PERSON_MEMORY); - RunProcessOutput runProcessOutput = new RunProcessAction().execute(runProcessInput); + RunProcessInput runProcessInput = new RunProcessInput(); + RunProcessOutput runProcessOutput = startProcess(runProcessInput); String processUUID = runProcessOutput.getProcessUUID(); assertThat(runProcessOutput.getProcessState().getNextStepName()).isPresent().get().isEqualTo("upload"); - ////////////////////////////// - // simulate the file upload // - ////////////////////////////// - String storageReference = UUID.randomUUID() + ".csv"; - StorageInput storageInput = new StorageInput(TestUtils.TABLE_NAME_MEMORY_STORAGE).withReference(storageReference); - try(OutputStream outputStream = new StorageAction().createOutputStream(storageInput)) - { - outputStream.write((getPersonCsvHeaderUsingLabels() + getPersonCsvRow1() + getPersonCsvRow2()).getBytes()); - } - catch(IOException e) - { - throw (e); - } - ////////////////////////// // continue post-upload // ////////////////////////// - runProcessInput.setProcessUUID(processUUID); - runProcessInput.setStartAfterStep("upload"); - runProcessInput.addValue("theFile", new ArrayList<>(List.of(storageInput))); - runProcessOutput = new RunProcessAction().execute(runProcessInput); + runProcessOutput = continueProcessPostUpload(runProcessInput, processUUID, simulateFileUpload(2)); assertEquals(List.of("Id", "Create Date", "Modify Date", "First Name", "Last Name", "Birth Date", "Email", "Home State", "noOfShoes"), runProcessOutput.getValue("headerValues")); assertEquals(List.of("A", "B", "C", "D", "E", "F", "G", "H", "I"), runProcessOutput.getValue("headerLetters")); @@ -176,29 +148,10 @@ class BulkInsertFullProcessTest extends BaseTest assertThat(runProcessOutput.getProcessState().getNextStepName()).isPresent().get().isEqualTo("fileMapping"); - //////////////////////////////////////////////////////////////////////////////// - // all subsequent steps will want these data - so set up a lambda to set them // - //////////////////////////////////////////////////////////////////////////////// - Consumer addProfileToRunProcessInput = (RunProcessInput input) -> - { - input.addValue("version", "v1"); - input.addValue("layout", "FLAT"); - input.addValue("hasHeaderRow", "true"); - input.addValue("fieldListJSON", JsonUtils.toJson(List.of( - new BulkLoadProfileField().withFieldName("firstName").withColumnIndex(3), - new BulkLoadProfileField().withFieldName("lastName").withColumnIndex(4), - new BulkLoadProfileField().withFieldName("email").withDefaultValue(defaultEmail), - new BulkLoadProfileField().withFieldName("homeStateId").withColumnIndex(7).withDoValueMapping(true).withValueMappings(Map.of("Illinois", 1)), - new BulkLoadProfileField().withFieldName("noOfShoes").withColumnIndex(8) - ))); - }; - //////////////////////////////// // continue post file-mapping // //////////////////////////////// - runProcessInput.setStartAfterStep("fileMapping"); - addProfileToRunProcessInput.accept(runProcessInput); - runProcessOutput = new RunProcessAction().execute(runProcessInput); + runProcessOutput = continueProcessPostFileMapping(runProcessInput); Serializable valueMappingField = runProcessOutput.getValue("valueMappingField"); assertThat(valueMappingField).isInstanceOf(QFrontendFieldMetaData.class); assertEquals("homeStateId", ((QFrontendFieldMetaData) valueMappingField).getName()); @@ -211,23 +164,20 @@ class BulkInsertFullProcessTest extends BaseTest ///////////////////////////////// // continue post value-mapping // ///////////////////////////////// - runProcessInput.setStartAfterStep("valueMapping"); - runProcessInput.addValue("mappedValuesJSON", JsonUtils.toJson(Map.of("Illinois", 1, "Missouri", 2))); - addProfileToRunProcessInput.accept(runProcessInput); - runProcessOutput = new RunProcessAction().execute(runProcessInput); + runProcessOutput = continueProcessPostValueMapping(runProcessInput); assertThat(runProcessOutput.getProcessState().getNextStepName()).isPresent().get().isEqualTo("review"); ///////////////////////////////// // continue post review screen // ///////////////////////////////// - runProcessInput.setStartAfterStep("review"); - addProfileToRunProcessInput.accept(runProcessInput); - runProcessOutput = new RunProcessAction().execute(runProcessInput); + runProcessOutput = continueProcessPostReviewScreen(runProcessInput); assertThat(runProcessOutput.getRecords()).hasSize(2); assertThat(runProcessOutput.getProcessState().getNextStepName()).isPresent().get().isEqualTo("result"); assertThat(runProcessOutput.getValues().get(StreamedETLWithFrontendProcess.FIELD_PROCESS_SUMMARY)).isNotNull().isInstanceOf(List.class); assertThat(runProcessOutput.getException()).isEmpty(); + ProcessSummaryAssert.assertThat(runProcessOutput).hasLineWithMessageContaining("Inserted Id values between 1 and 2"); + //////////////////////////////////// // query for the inserted records // //////////////////////////////////// @@ -249,4 +199,133 @@ class BulkInsertFullProcessTest extends BaseTest assertNull(records.get(1).getValue("noOfShoes")); } + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testOneRow() throws Exception + { + /////////////////////////////////////// + // make sure table is empty to start // + /////////////////////////////////////// + assertThat(TestUtils.queryTable(TestUtils.TABLE_NAME_PERSON_MEMORY)).isEmpty(); + + RunProcessInput runProcessInput = new RunProcessInput(); + RunProcessOutput runProcessOutput = startProcess(runProcessInput); + String processUUID = runProcessOutput.getProcessUUID(); + + continueProcessPostUpload(runProcessInput, processUUID, simulateFileUpload(1)); + continueProcessPostFileMapping(runProcessInput); + continueProcessPostValueMapping(runProcessInput); + runProcessOutput = continueProcessPostReviewScreen(runProcessInput); + + //////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // all that just so we can make sure this message is right (because it was wrong when we first wrote it, lol) // + //////////////////////////////////////////////////////////////////////////////////////////////////////////////// + ProcessSummaryAssert.assertThat(runProcessOutput).hasLineWithMessageContaining("Inserted Id 1"); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private static RunProcessOutput continueProcessPostReviewScreen(RunProcessInput runProcessInput) throws QException + { + RunProcessOutput runProcessOutput; + runProcessInput.setStartAfterStep("review"); + addProfileToRunProcessInput(runProcessInput); + runProcessOutput = new RunProcessAction().execute(runProcessInput); + return runProcessOutput; + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private static RunProcessOutput continueProcessPostValueMapping(RunProcessInput runProcessInput) throws QException + { + runProcessInput.setStartAfterStep("valueMapping"); + runProcessInput.addValue("mappedValuesJSON", JsonUtils.toJson(Map.of("Illinois", 1, "Missouri", 2))); + addProfileToRunProcessInput(runProcessInput); + RunProcessOutput runProcessOutput = new RunProcessAction().execute(runProcessInput); + return (runProcessOutput); + } + + + + private static RunProcessOutput continueProcessPostFileMapping(RunProcessInput runProcessInput) throws QException + { + RunProcessOutput runProcessOutput; + runProcessInput.setStartAfterStep("fileMapping"); + addProfileToRunProcessInput(runProcessInput); + runProcessOutput = new RunProcessAction().execute(runProcessInput); + return runProcessOutput; + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private static RunProcessOutput continueProcessPostUpload(RunProcessInput runProcessInput, String processUUID, StorageInput storageInput) throws QException + { + runProcessInput.setProcessUUID(processUUID); + runProcessInput.setStartAfterStep("upload"); + runProcessInput.addValue("theFile", new ArrayList<>(List.of(storageInput))); + RunProcessOutput runProcessOutput = new RunProcessAction().execute(runProcessInput); + return (runProcessOutput); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private static StorageInput simulateFileUpload(int noOfRows) throws Exception + { + String storageReference = UUID.randomUUID() + ".csv"; + StorageInput storageInput = new StorageInput(TestUtils.TABLE_NAME_MEMORY_STORAGE).withReference(storageReference); + try(OutputStream outputStream = new StorageAction().createOutputStream(storageInput)) + { + outputStream.write((getPersonCsvHeaderUsingLabels() + getPersonCsvRow1() + (noOfRows == 2 ? getPersonCsvRow2() : "")).getBytes()); + } + return storageInput; + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private static RunProcessOutput startProcess(RunProcessInput runProcessInput) throws QException + { + runProcessInput.setProcessName(TestUtils.TABLE_NAME_PERSON_MEMORY + ".bulkInsert"); + runProcessInput.addValue("tableName", TestUtils.TABLE_NAME_PERSON_MEMORY); + RunProcessOutput runProcessOutput = new RunProcessAction().execute(runProcessInput); + return runProcessOutput; + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private static void addProfileToRunProcessInput(RunProcessInput input) + { + input.addValue("version", "v1"); + input.addValue("layout", "FLAT"); + input.addValue("hasHeaderRow", "true"); + input.addValue("fieldListJSON", JsonUtils.toJson(List.of( + new BulkLoadProfileField().withFieldName("firstName").withColumnIndex(3), + new BulkLoadProfileField().withFieldName("lastName").withColumnIndex(4), + new BulkLoadProfileField().withFieldName("email").withDefaultValue(defaultEmail), + new BulkLoadProfileField().withFieldName("homeStateId").withColumnIndex(7).withDoValueMapping(true).withValueMappings(Map.of("Illinois", 1)), + new BulkLoadProfileField().withFieldName("noOfShoes").withColumnIndex(8) + ))); + } + } \ No newline at end of file