diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertPrepareFileMappingStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertPrepareFileMappingStep.java index c72df5fd..19e2dc79 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertPrepareFileMappingStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertPrepareFileMappingStep.java @@ -24,8 +24,12 @@ package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert; import java.io.File; import java.io.InputStream; +import java.io.Serializable; import java.util.ArrayList; +import java.util.Collections; +import java.util.LinkedHashMap; import java.util.List; +import java.util.Map; import com.kingsrook.qqq.backend.core.actions.processes.BackendStep; import com.kingsrook.qqq.backend.core.actions.tables.StorageAction; import com.kingsrook.qqq.backend.core.exceptions.QException; @@ -37,9 +41,14 @@ import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapp import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping.BulkLoadTableStructureBuilder; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadProfile; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadProfileField; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadTableStructure; import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess; +import com.kingsrook.qqq.backend.core.utils.CollectionUtils; +import com.kingsrook.qqq.backend.core.utils.JsonUtils; +import com.kingsrook.qqq.backend.core.utils.StringUtils; import com.kingsrook.qqq.backend.core.utils.ValueUtils; +import org.json.JSONObject; /******************************************************************************* @@ -72,7 +81,7 @@ public class BulkInsertPrepareFileMappingStep implements BackendStep { @SuppressWarnings("unchecked") List headerValues = (List) runBackendStepOutput.getValue("headerValues"); - buildSuggestedMapping(headerValues, tableStructure, runBackendStepOutput); + buildSuggestedMapping(headerValues, getPrepopulatedValues(runBackendStepInput), tableStructure, runBackendStepOutput); } } @@ -81,10 +90,62 @@ public class BulkInsertPrepareFileMappingStep implements BackendStep /*************************************************************************** ** ***************************************************************************/ - private void buildSuggestedMapping(List headerValues, BulkLoadTableStructure tableStructure, RunBackendStepOutput runBackendStepOutput) + private Map getPrepopulatedValues(RunBackendStepInput runBackendStepInput) + { + String prepopulatedValuesJson = runBackendStepInput.getValueString("prepopulatedValues"); + if(StringUtils.hasContent(prepopulatedValuesJson)) + { + Map rs = new LinkedHashMap<>(); + JSONObject jsonObject = JsonUtils.toJSONObject(prepopulatedValuesJson); + for(String key : jsonObject.keySet()) + { + rs.put(key, jsonObject.optString(key, null)); + } + return (rs); + } + + return (Collections.emptyMap()); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private void buildSuggestedMapping(List headerValues, Map prepopulatedValues, BulkLoadTableStructure tableStructure, RunBackendStepOutput runBackendStepOutput) { BulkLoadMappingSuggester bulkLoadMappingSuggester = new BulkLoadMappingSuggester(); BulkLoadProfile bulkLoadProfile = bulkLoadMappingSuggester.suggestBulkLoadMappingProfile(tableStructure, headerValues); + + if(CollectionUtils.nullSafeHasContents(prepopulatedValues)) + { + for(Map.Entry entry : prepopulatedValues.entrySet()) + { + String fieldName = entry.getKey(); + boolean foundFieldInProfile = false; + + for(BulkLoadProfileField bulkLoadProfileField : bulkLoadProfile.getFieldList()) + { + if(bulkLoadProfileField.getFieldName().equals(fieldName)) + { + foundFieldInProfile = true; + bulkLoadProfileField.setColumnIndex(null); + bulkLoadProfileField.setHeaderName(null); + bulkLoadProfileField.setDefaultValue(entry.getValue()); + break; + } + } + + if(!foundFieldInProfile) + { + BulkLoadProfileField bulkLoadProfileField = new BulkLoadProfileField(); + bulkLoadProfileField.setFieldName(fieldName); + bulkLoadProfileField.setDefaultValue(entry.getValue()); + bulkLoadProfile.getFieldList().add(bulkLoadProfileField); + } + } + } + runBackendStepOutput.addValue("bulkLoadProfile", bulkLoadProfile); runBackendStepOutput.addValue("suggestedBulkLoadProfile", bulkLoadProfile); } diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertPrepareFileMappingStepTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertPrepareFileMappingStepTest.java index ea6e0e8f..c633c920 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertPrepareFileMappingStepTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertPrepareFileMappingStepTest.java @@ -22,8 +22,22 @@ package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert; +import java.io.OutputStream; +import java.nio.charset.StandardCharsets; +import java.util.Map; +import java.util.Optional; import com.kingsrook.qqq.backend.core.BaseTest; +import com.kingsrook.qqq.backend.core.actions.tables.StorageAction; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.storage.StorageInput; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadProfile; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadProfileField; +import com.kingsrook.qqq.backend.core.utils.JsonUtils; +import com.kingsrook.qqq.backend.core.utils.TestUtils; import org.junit.jupiter.api.Test; +import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -66,4 +80,39 @@ class BulkInsertPrepareFileMappingStepTest extends BaseTest assertEquals("BAA", BulkInsertPrepareFileMappingStep.toHeaderLetter(2 * 26 * 26 + 26 + 0)); } + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void test() throws Exception + { + String fileName = "personFile.csv"; + + StorageInput storageInput = new StorageInput(TestUtils.TABLE_NAME_MEMORY_STORAGE).withReference(fileName); + OutputStream outputStream = new StorageAction().createOutputStream(storageInput); + outputStream.write(""" + name,noOfShoes + John,2 + Jane,4 + """.getBytes(StandardCharsets.UTF_8)); + outputStream.close(); + + RunProcessInput runProcessInput = new RunProcessInput(); + BulkInsertStepUtils.setStorageInputForTheFile(runProcessInput, storageInput); + runProcessInput.addValue("tableName", TestUtils.TABLE_NAME_PERSON_MEMORY); + runProcessInput.addValue("prepopulatedValues", JsonUtils.toJson(Map.of("homeStateId", 1))); + + RunBackendStepInput runBackendStepInput = new RunBackendStepInput(runProcessInput.getProcessState()); + RunBackendStepOutput runBackendStepOutput = new RunBackendStepOutput(); + + new BulkInsertPrepareFileMappingStep().run(runBackendStepInput, runBackendStepOutput); + + BulkLoadProfile bulkLoadProfile = (BulkLoadProfile) runBackendStepOutput.getValue("suggestedBulkLoadProfile"); + Optional homeStateId = bulkLoadProfile.getFieldList().stream().filter(f -> f.getFieldName().equals("homeStateId")).findFirst(); + assertThat(homeStateId).isPresent(); + assertEquals("1", homeStateId.get().getDefaultValue()); + } + } \ No newline at end of file