QRunIO/qqq
2
0
Fork 0
mirror of https://github.com/Kingsrook/qqq.git synced 2025-07-18 05:01:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

Allow a map of prepopulatedValues to be provided as an input value, to set defaultValues for fields

Browse Source
This commit is contained in:
darin.kelkhoff
2025-03-18 10:04:16 -05:00
parent 491fcd6d25
commit 0f8ad2fb78
2 changed files with 112 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

65
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertPrepareFileMappingStep.java
View File

@ -24,8 +24,12 @@ package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert;
import java.io.File;
import java.io.InputStream;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.Collections;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import com.kingsrook.qqq.backend.core.actions.processes.BackendStep;
import com.kingsrook.qqq.backend.core.actions.tables.StorageAction;
import com.kingsrook.qqq.backend.core.exceptions.QException;
@ -37,9 +41,14 @@ import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapp
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping.BulkLoadTableStructureBuilder;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadProfile;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadProfileField;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadTableStructure;
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess;
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
import com.kingsrook.qqq.backend.core.utils.JsonUtils;
import com.kingsrook.qqq.backend.core.utils.StringUtils;
import com.kingsrook.qqq.backend.core.utils.ValueUtils;
import org.json.JSONObject;
/*******************************************************************************
@ -72,7 +81,7 @@ public class BulkInsertPrepareFileMappingStep implements BackendStep
{
@SuppressWarnings("unchecked")
List<String> headerValues = (List<String>) runBackendStepOutput.getValue("headerValues");
buildSuggestedMapping(headerValues, tableStructure, runBackendStepOutput);
buildSuggestedMapping(headerValues, getPrepopulatedValues(runBackendStepInput), tableStructure, runBackendStepOutput);
}
}
@ -81,10 +90,62 @@ public class BulkInsertPrepareFileMappingStep implements BackendStep
/***************************************************************************
**
***************************************************************************/
private void buildSuggestedMapping(List<String> headerValues, BulkLoadTableStructure tableStructure, RunBackendStepOutput runBackendStepOutput)
private Map<String, Serializable> getPrepopulatedValues(RunBackendStepInput runBackendStepInput)
{
String prepopulatedValuesJson = runBackendStepInput.getValueString("prepopulatedValues");
if(StringUtils.hasContent(prepopulatedValuesJson))
{
Map<String, Serializable> rs = new LinkedHashMap<>();
JSONObject jsonObject = JsonUtils.toJSONObject(prepopulatedValuesJson);
for(String key : jsonObject.keySet())
{
rs.put(key, jsonObject.optString(key, null));
}
return (rs);
}
return (Collections.emptyMap());
}
/***************************************************************************
**
***************************************************************************/
private void buildSuggestedMapping(List<String> headerValues, Map<String, Serializable> prepopulatedValues, BulkLoadTableStructure tableStructure, RunBackendStepOutput runBackendStepOutput)
{
BulkLoadMappingSuggester bulkLoadMappingSuggester = new BulkLoadMappingSuggester();
BulkLoadProfile bulkLoadProfile = bulkLoadMappingSuggester.suggestBulkLoadMappingProfile(tableStructure, headerValues);
if(CollectionUtils.nullSafeHasContents(prepopulatedValues))
{
for(Map.Entry<String, Serializable> entry : prepopulatedValues.entrySet())
{
String fieldName = entry.getKey();
boolean foundFieldInProfile = false;
for(BulkLoadProfileField bulkLoadProfileField : bulkLoadProfile.getFieldList())
{
if(bulkLoadProfileField.getFieldName().equals(fieldName))
{
foundFieldInProfile = true;
bulkLoadProfileField.setColumnIndex(null);
bulkLoadProfileField.setHeaderName(null);
bulkLoadProfileField.setDefaultValue(entry.getValue());
break;
}
}
if(!foundFieldInProfile)
{
BulkLoadProfileField bulkLoadProfileField = new BulkLoadProfileField();
bulkLoadProfileField.setFieldName(fieldName);
bulkLoadProfileField.setDefaultValue(entry.getValue());
bulkLoadProfile.getFieldList().add(bulkLoadProfileField);
}
}
}
runBackendStepOutput.addValue("bulkLoadProfile", bulkLoadProfile);
runBackendStepOutput.addValue("suggestedBulkLoadProfile", bulkLoadProfile);
}

49
qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertPrepareFileMappingStepTest.java
View File

@ -22,8 +22,22 @@
package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert;
import java.io.OutputStream;
import java.nio.charset.StandardCharsets;
import java.util.Map;
import java.util.Optional;
import com.kingsrook.qqq.backend.core.BaseTest;
import com.kingsrook.qqq.backend.core.actions.tables.StorageAction;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.storage.StorageInput;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadProfile;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadProfileField;
import com.kingsrook.qqq.backend.core.utils.JsonUtils;
import com.kingsrook.qqq.backend.core.utils.TestUtils;
import org.junit.jupiter.api.Test;
import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.jupiter.api.Assertions.assertEquals;
@ -66,4 +80,39 @@ class BulkInsertPrepareFileMappingStepTest extends BaseTest
assertEquals("BAA", BulkInsertPrepareFileMappingStep.toHeaderLetter(2 * 26 * 26 + 26 + 0));
}
/*******************************************************************************
**
*******************************************************************************/
@Test
void test() throws Exception
{
String fileName = "personFile.csv";
StorageInput storageInput = new StorageInput(TestUtils.TABLE_NAME_MEMORY_STORAGE).withReference(fileName);
OutputStream outputStream = new StorageAction().createOutputStream(storageInput);
outputStream.write("""
name,noOfShoes
John,2
Jane,4
""".getBytes(StandardCharsets.UTF_8));
outputStream.close();
RunProcessInput runProcessInput = new RunProcessInput();
BulkInsertStepUtils.setStorageInputForTheFile(runProcessInput, storageInput);
runProcessInput.addValue("tableName", TestUtils.TABLE_NAME_PERSON_MEMORY);
runProcessInput.addValue("prepopulatedValues", JsonUtils.toJson(Map.of("homeStateId", 1)));
RunBackendStepInput runBackendStepInput = new RunBackendStepInput(runProcessInput.getProcessState());
RunBackendStepOutput runBackendStepOutput = new RunBackendStepOutput();
new BulkInsertPrepareFileMappingStep().run(runBackendStepInput, runBackendStepOutput);
BulkLoadProfile bulkLoadProfile = (BulkLoadProfile) runBackendStepOutput.getValue("suggestedBulkLoadProfile");
Optional<BulkLoadProfileField> homeStateId = bulkLoadProfile.getFieldList().stream().filter(f -> f.getFieldName().equals("homeStateId")).findFirst();
assertThat(homeStateId).isPresent();
assertEquals("1", homeStateId.get().getDefaultValue());
}
}