QRunIO/qqq
2
0
Fork 0
mirror of https://github.com/Kingsrook/qqq.git synced 2025-07-18 13:10:44 +00:00
Code Issues Packages Projects Releases Wiki Activity

CE-1955 - Bulk load checkpoint:

Browse Source 
- Switch wide format to identify associations via comma-number-indexes...
- Add suggested mappings
- use header name instead of column index for mappings
- add counts of children process summary lines
- excel value/type handling
This commit is contained in:
darin.kelkhoff
2024-11-25 10:07:26 -06:00
parent 9ad9d52634
commit 58ae17bbac
29 changed files with 1167 additions and 988 deletions
Download Patch File Download Diff File Expand all files Collapse all files

68
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java
View File

@ -73,7 +73,6 @@ import com.kingsrook.qqq.backend.core.processes.implementations.bulk.delete.Bulk
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.delete.BulkDeleteTransformStep; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.delete.BulkDeleteTransformStep;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.edit.BulkEditLoadStep; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.edit.BulkEditLoadStep;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.edit.BulkEditTransformStep; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.edit.BulkEditTransformStep;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkInsertExtractStep;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkInsertLoadStep; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkInsertLoadStep;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkInsertPrepareFileMappingStep; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkInsertPrepareFileMappingStep;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkInsertPrepareValueMappingStep; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkInsertPrepareValueMappingStep;
@ -81,7 +80,6 @@ import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.Bulk
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkInsertReceiveValueMappingStep; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkInsertReceiveValueMappingStep;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkInsertTransformStep; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkInsertTransformStep;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkInsertV2ExtractStep; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkInsertV2ExtractStep;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping.BulkInsertMapping;
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.ExtractViaQueryStep; import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.ExtractViaQueryStep;
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess; import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess;
import com.kingsrook.qqq.backend.core.scheduler.QScheduleManager; import com.kingsrook.qqq.backend.core.scheduler.QScheduleManager;
@ -815,75 +813,11 @@ public class QInstanceEnricher
/******************************************************************************* /*******************************************************************************
** **
*******************************************************************************/ *******************************************************************************/
private void defineTableBulkInsert(QInstance qInstance, QTableMetaData table, String processName) public void defineTableBulkInsert(QInstance qInstance, QTableMetaData table, String processName)
{ {
Map<String, Serializable> values = new HashMap<>(); Map<String, Serializable> values = new HashMap<>();
values.put(StreamedETLWithFrontendProcess.FIELD_DESTINATION_TABLE, table.getName()); values.put(StreamedETLWithFrontendProcess.FIELD_DESTINATION_TABLE, table.getName());
QProcessMetaData process = StreamedETLWithFrontendProcess.defineProcessMetaData(
BulkInsertExtractStep.class,
BulkInsertTransformStep.class,
BulkInsertLoadStep.class,
values
)
.withName(processName)
.withLabel(table.getLabel() + " Bulk Insert")
.withTableName(table.getName())
.withIsHidden(true)
.withPermissionRules(qInstance.getDefaultPermissionRules().clone()
.withCustomPermissionChecker(new QCodeReference(BulkTableActionProcessPermissionChecker.class)));
List<QFieldMetaData> editableFields = new ArrayList<>();
for(QFieldSection section : CollectionUtils.nonNullList(table.getSections()))
{
for(String fieldName : CollectionUtils.nonNullList(section.getFieldNames()))
{
try
{
QFieldMetaData field = table.getField(fieldName);
if(field.getIsEditable() && !field.getType().equals(QFieldType.BLOB))
{
editableFields.add(field);
}
}
catch(Exception e)
{
// shrug?
}
}
}
String fieldsForHelpText = editableFields.stream()
.map(QFieldMetaData::getLabel)
.collect(Collectors.joining(", "));
QFrontendStepMetaData uploadScreen = new QFrontendStepMetaData()
.withName("upload")
.withLabel("Upload File")
.withFormField(new QFieldMetaData("theFile", QFieldType.BLOB).withLabel(table.getLabel() + " File").withIsRequired(true))
.withComponent(new QFrontendComponentMetaData()
.withType(QComponentType.HELP_TEXT)
.withValue("previewText", "file upload instructions")
.withValue("text", "Upload a CSV file with the following columns:\n" + fieldsForHelpText))
.withComponent(new QFrontendComponentMetaData().withType(QComponentType.EDIT_FORM));
process.addStep(0, uploadScreen);
process.getFrontendStep("review").setRecordListFields(editableFields);
qInstance.addProcess(process);
}
/*******************************************************************************
**
*******************************************************************************/
public void defineTableBulkInsertV2(QInstance qInstance, QTableMetaData table, String processName)
{
qInstance.addPossibleValueSource(QPossibleValueSource.newForEnum("bulkInsertFileLayout", BulkInsertMapping.Layout.values()));
Map<String, Serializable> values = new HashMap<>();
values.put(StreamedETLWithFrontendProcess.FIELD_DESTINATION_TABLE, table.getName());
QProcessMetaData process = StreamedETLWithFrontendProcess.defineProcessMetaData( QProcessMetaData process = StreamedETLWithFrontendProcess.defineProcessMetaData(
BulkInsertV2ExtractStep.class, BulkInsertV2ExtractStep.class,
BulkInsertTransformStep.class, BulkInsertTransformStep.class,

128
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertPrepareFileMappingStep.java
View File

@ -25,27 +25,19 @@ package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert;
import java.io.File; import java.io.File;
import java.io.InputStream; import java.io.InputStream;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Comparator; import java.util.List;
import java.util.HashSet;
import java.util.Set;
import com.kingsrook.qqq.backend.core.actions.processes.BackendStep; import com.kingsrook.qqq.backend.core.actions.processes.BackendStep;
import com.kingsrook.qqq.backend.core.actions.tables.StorageAction; import com.kingsrook.qqq.backend.core.actions.tables.StorageAction;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
import com.kingsrook.qqq.backend.core.model.actions.tables.storage.StorageInput; import com.kingsrook.qqq.backend.core.model.actions.tables.storage.StorageInput;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinOn;
import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinType;
import com.kingsrook.qqq.backend.core.model.metadata.joins.QJoinMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.tables.Association;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.FileToRowsInterface; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.FileToRowsInterface;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping.BulkLoadMappingSuggester;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping.BulkLoadTableStructureBuilder;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadProfile;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadTableStructure; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadTableStructure;
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
import com.kingsrook.qqq.backend.core.utils.StringUtils;
import com.kingsrook.qqq.backend.core.utils.ValueUtils; import com.kingsrook.qqq.backend.core.utils.ValueUtils;
@ -62,7 +54,26 @@ public class BulkInsertPrepareFileMappingStep implements BackendStep
public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException
{ {
buildFileDetailsForMappingStep(runBackendStepInput, runBackendStepOutput); buildFileDetailsForMappingStep(runBackendStepInput, runBackendStepOutput);
buildFieldsForMappingStep(runBackendStepInput, runBackendStepOutput);
String tableName = runBackendStepInput.getValueString("tableName");
BulkLoadTableStructure tableStructure = BulkLoadTableStructureBuilder.buildTableStructure(tableName);
runBackendStepOutput.addValue("tableStructure", tableStructure);
@SuppressWarnings("unchecked")
List<String> headerValues = (List<String>) runBackendStepOutput.getValue("headerValues");
buildSuggestedMapping(headerValues, tableStructure, runBackendStepOutput);
}
/***************************************************************************
**
***************************************************************************/
private void buildSuggestedMapping(List<String> headerValues, BulkLoadTableStructure tableStructure, RunBackendStepOutput runBackendStepOutput)
{
BulkLoadMappingSuggester bulkLoadMappingSuggester = new BulkLoadMappingSuggester();
BulkLoadProfile bulkLoadProfile = bulkLoadMappingSuggester.suggestBulkLoadMappingProfile(tableStructure, headerValues);
runBackendStepOutput.addValue("bulkLoadProfile", bulkLoadProfile);
} }
@ -121,6 +132,7 @@ public class BulkInsertPrepareFileMappingStep implements BackendStep
} }
} }
runBackendStepOutput.addValue("bodyValuesPreview", bodyValues); runBackendStepOutput.addValue("bodyValuesPreview", bodyValues);
} }
catch(Exception e) catch(Exception e)
{ {
@ -147,94 +159,4 @@ public class BulkInsertPrepareFileMappingStep implements BackendStep
return (rs.toString()); return (rs.toString());
} }
/***************************************************************************
**
***************************************************************************/
private static void buildFieldsForMappingStep(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput)
{
String tableName = runBackendStepInput.getValueString("tableName");
BulkLoadTableStructure tableStructure = buildTableStructure(tableName, null, null);
runBackendStepOutput.addValue("tableStructure", tableStructure);
}
/***************************************************************************
**
***************************************************************************/
private static BulkLoadTableStructure buildTableStructure(String tableName, Association association, String parentAssociationPath)
{
QTableMetaData table = QContext.getQInstance().getTable(tableName);
BulkLoadTableStructure tableStructure = new BulkLoadTableStructure();
tableStructure.setTableName(tableName);
tableStructure.setLabel(table.getLabel());
Set<String> associationJoinFieldNamesToExclude = new HashSet<>();
if(association == null)
{
tableStructure.setIsMain(true);
tableStructure.setIsMany(false);
tableStructure.setAssociationPath(null);
}
else
{
tableStructure.setIsMain(false);
QJoinMetaData join = QContext.getQInstance().getJoin(association.getJoinName());
if(join.getType().equals(JoinType.ONE_TO_MANY) || join.getType().equals(JoinType.MANY_TO_ONE))
{
tableStructure.setIsMany(true);
}
for(JoinOn joinOn : join.getJoinOns())
{
////////////////////////////////////////////////////////////////////////////////////////////////
// don't allow the user to map the "join field" from a child up to its parent //
// (e.g., you can't map lineItem.orderId -- that'll happen automatically via the association) //
////////////////////////////////////////////////////////////////////////////////////////////////
if(join.getLeftTable().equals(tableName))
{
associationJoinFieldNamesToExclude.add(joinOn.getLeftField());
}
else if(join.getRightTable().equals(tableName))
{
associationJoinFieldNamesToExclude.add(joinOn.getRightField());
}
}
if(!StringUtils.hasContent(parentAssociationPath))
{
tableStructure.setAssociationPath(association.getName());
}
else
{
tableStructure.setAssociationPath(parentAssociationPath + "." + association.getName());
}
}
ArrayList<QFieldMetaData> fields = new ArrayList<>();
tableStructure.setFields(fields);
for(QFieldMetaData field : table.getFields().values())
{
if(field.getIsEditable() && !associationJoinFieldNamesToExclude.contains(field.getName()))
{
fields.add(field);
}
}
fields.sort(Comparator.comparing(f -> f.getLabel()));
for(Association childAssociation : CollectionUtils.nonNullList(table.getAssociations()))
{
BulkLoadTableStructure associatedStructure = buildTableStructure(childAssociation.getAssociatedTableName(), childAssociation, parentAssociationPath);
tableStructure.addAssociation(associatedStructure);
}
return (tableStructure);
}
} }

2
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertPrepareValueMappingStep.java
View File

@ -51,7 +51,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleVal
import com.kingsrook.qqq.backend.core.model.metadata.tables.Association; import com.kingsrook.qqq.backend.core.model.metadata.tables.Association;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.FileToRowsInterface; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.FileToRowsInterface;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping.BulkInsertMapping; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkInsertMapping;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow;
import com.kingsrook.qqq.backend.core.utils.CollectionUtils; import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
import com.kingsrook.qqq.backend.core.utils.StringUtils; import com.kingsrook.qqq.backend.core.utils.StringUtils;

15
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertReceiveFileMappingStep.java
View File

@ -37,7 +37,7 @@ import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInpu
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
import com.kingsrook.qqq.backend.core.model.actions.tables.storage.StorageInput; import com.kingsrook.qqq.backend.core.model.actions.tables.storage.StorageInput;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.FileToRowsInterface; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.FileToRowsInterface;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping.BulkInsertMapping; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkInsertMapping;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadProfile; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadProfile;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadProfileField; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadProfileField;
@ -102,7 +102,12 @@ public class BulkInsertReceiveFileMappingStep implements BackendStep
BulkLoadFileRow headerRow = fileToRowsInterface.next(); BulkLoadFileRow headerRow = fileToRowsInterface.next();
for(BulkLoadProfileField bulkLoadProfileField : bulkLoadProfile.getFieldList()) for(BulkLoadProfileField bulkLoadProfileField : bulkLoadProfile.getFieldList())
{ {
if(bulkLoadProfileField.getColumnIndex() != null) if(bulkLoadProfileField.getHeaderName() != null)
{
String headerName = bulkLoadProfileField.getHeaderName();
fieldNameToHeaderNameMap.put(bulkLoadProfileField.getFieldName(), headerName);
}
else if(bulkLoadProfileField.getColumnIndex() != null)
{ {
String headerName = ValueUtils.getValueAsString(headerRow.getValueElseNull(bulkLoadProfileField.getColumnIndex())); String headerName = ValueUtils.getValueAsString(headerRow.getValueElseNull(bulkLoadProfileField.getColumnIndex()));
fieldNameToHeaderNameMap.put(bulkLoadProfileField.getFieldName(), headerName); fieldNameToHeaderNameMap.put(bulkLoadProfileField.getFieldName(), headerName);
@ -164,7 +169,11 @@ public class BulkInsertReceiveFileMappingStep implements BackendStep
{ {
if(bulkLoadProfileField.getFieldName().contains(".")) if(bulkLoadProfileField.getFieldName().contains("."))
{ {
associationNameSet.add(bulkLoadProfileField.getFieldName().substring(0, bulkLoadProfileField.getFieldName().lastIndexOf('.'))); //////////////////////////////////////////////////////////////////////////////////////////////////////////
// handle parent.child.grandchild.fieldName,index.index.index if we do sub-indexes for grandchildren... //
//////////////////////////////////////////////////////////////////////////////////////////////////////////
String fieldNameBeforeIndex = bulkLoadProfileField.getFieldName().split(",")[0];
associationNameSet.add(fieldNameBeforeIndex.substring(0, fieldNameBeforeIndex.lastIndexOf('.')));
} }
} }
bulkInsertMapping.setMappedAssociations(new ArrayList<>(associationNameSet)); bulkInsertMapping.setMappedAssociations(new ArrayList<>(associationNameSet));

2
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertReceiveValueMappingStep.java
View File

@ -32,7 +32,7 @@ import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.logging.QLogger; import com.kingsrook.qqq.backend.core.logging.QLogger;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping.BulkInsertMapping; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkInsertMapping;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadProfile; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadProfile;
import com.kingsrook.qqq.backend.core.utils.JsonUtils; import com.kingsrook.qqq.backend.core.utils.JsonUtils;

1
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertStepUtils.java
View File

@ -109,6 +109,7 @@ public class BulkInsertStepUtils
BulkLoadProfileField bulkLoadProfileField = new BulkLoadProfileField(); BulkLoadProfileField bulkLoadProfileField = new BulkLoadProfileField();
fieldList.add(bulkLoadProfileField); fieldList.add(bulkLoadProfileField);
bulkLoadProfileField.setFieldName(jsonObject.optString("fieldName")); bulkLoadProfileField.setFieldName(jsonObject.optString("fieldName"));
bulkLoadProfileField.setHeaderName(jsonObject.has("headerName") ? jsonObject.getString("headerName") : null);
bulkLoadProfileField.setColumnIndex(jsonObject.has("columnIndex") ? jsonObject.getInt("columnIndex") : null); bulkLoadProfileField.setColumnIndex(jsonObject.has("columnIndex") ? jsonObject.getInt("columnIndex") : null);
bulkLoadProfileField.setDefaultValue((Serializable) jsonObject.opt("defaultValue")); bulkLoadProfileField.setDefaultValue((Serializable) jsonObject.opt("defaultValue"));
bulkLoadProfileField.setDoValueMapping(jsonObject.optBoolean("doValueMapping")); bulkLoadProfileField.setDoValueMapping(jsonObject.optBoolean("doValueMapping"));

29
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTransformStep.java
View File

@ -48,6 +48,7 @@ import com.kingsrook.qqq.backend.core.model.actions.processes.Status;
import com.kingsrook.qqq.backend.core.model.actions.tables.QInputSource; import com.kingsrook.qqq.backend.core.model.actions.tables.QInputSource;
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.tables.Association;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.tables.UniqueKey; import com.kingsrook.qqq.backend.core.model.metadata.tables.UniqueKey;
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.AbstractTransformStep; import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.AbstractTransformStep;
@ -67,7 +68,8 @@ public class BulkInsertTransformStep extends AbstractTransformStep
private ProcessSummaryWarningsAndErrorsRollup processSummaryWarningsAndErrorsRollup = ProcessSummaryWarningsAndErrorsRollup.build("inserted"); private ProcessSummaryWarningsAndErrorsRollup processSummaryWarningsAndErrorsRollup = ProcessSummaryWarningsAndErrorsRollup.build("inserted");
private Map<UniqueKey, ProcessSummaryLineWithUKSampleValues> ukErrorSummaries = new HashMap<>(); private Map<UniqueKey, ProcessSummaryLineWithUKSampleValues> ukErrorSummaries = new HashMap<>();
private Map<String, ProcessSummaryLine> associationsToInsertSummaries = new HashMap<>();
private QTableMetaData table; private QTableMetaData table;
@ -259,6 +261,13 @@ public class BulkInsertTransformStep extends AbstractTransformStep
{ {
okSummary.incrementCountAndAddPrimaryKey(null); okSummary.incrementCountAndAddPrimaryKey(null);
outputRecords.add(record); outputRecords.add(record);
for(Map.Entry<String, List<QRecord>> entry : CollectionUtils.nonNullMap(record.getAssociatedRecords()).entrySet())
{
String associationName = entry.getKey();
ProcessSummaryLine associationToInsertLine = associationsToInsertSummaries.computeIfAbsent(associationName, x -> new ProcessSummaryLine(Status.OK));
associationToInsertLine.incrementCount(CollectionUtils.nonNullList(entry.getValue()).size());
}
} }
} }
@ -366,6 +375,24 @@ public class BulkInsertTransformStep extends AbstractTransformStep
okSummary.pickMessage(isForResultScreen); okSummary.pickMessage(isForResultScreen);
okSummary.addSelfToListIfAnyCount(rs); okSummary.addSelfToListIfAnyCount(rs);
for(Map.Entry<String, ProcessSummaryLine> entry : associationsToInsertSummaries.entrySet())
{
Optional<Association> association = table.getAssociations().stream().filter(a -> a.getName().equals(entry.getKey())).findFirst();
if(association.isPresent())
{
QTableMetaData associationTable = QContext.getQInstance().getTable(association.get().getAssociatedTableName());
String associationLabel = associationTable.getLabel();
ProcessSummaryLine line = entry.getValue();
line.setSingularFutureMessage(associationLabel + " record will be inserted.");
line.setPluralFutureMessage(associationLabel + " records will be inserted.");
line.setSingularPastMessage(associationLabel + " record was inserted.");
line.setPluralPastMessage(associationLabel + " records were inserted.");
line.pickMessage(isForResultScreen);
line.addSelfToListIfAnyCount(rs);
}
}
for(Map.Entry<UniqueKey, ProcessSummaryLineWithUKSampleValues> entry : ukErrorSummaries.entrySet()) for(Map.Entry<UniqueKey, ProcessSummaryLineWithUKSampleValues> entry : ukErrorSummaries.entrySet())
{ {
UniqueKey uniqueKey = entry.getKey(); UniqueKey uniqueKey = entry.getKey();

2
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertV2ExtractStep.java
View File

@ -32,8 +32,8 @@ import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutp
import com.kingsrook.qqq.backend.core.model.actions.tables.storage.StorageInput; import com.kingsrook.qqq.backend.core.model.actions.tables.storage.StorageInput;
import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.FileToRowsInterface; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.FileToRowsInterface;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping.BulkInsertMapping;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping.RowsToRecordInterface; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping.RowsToRecordInterface;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkInsertMapping;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow;
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.AbstractExtractStep; import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.AbstractExtractStep;

41
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/XlsxFileToRows.java
View File

@ -25,9 +25,14 @@ package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.fil
import java.io.IOException; import java.io.IOException;
import java.io.InputStream; import java.io.InputStream;
import java.io.Serializable; import java.io.Serializable;
import java.math.BigDecimal;
import java.math.MathContext;
import java.time.LocalDateTime;
import java.util.Optional;
import java.util.stream.Stream; import java.util.stream.Stream;
import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow;
import org.dhatim.fastexcel.reader.Cell;
import org.dhatim.fastexcel.reader.ReadableWorkbook; import org.dhatim.fastexcel.reader.ReadableWorkbook;
import org.dhatim.fastexcel.reader.Sheet; import org.dhatim.fastexcel.reader.Sheet;
@ -74,7 +79,41 @@ public class XlsxFileToRows extends AbstractIteratorBasedFileToRows<org.dhatim.f
for(int i = 0; i < readerRow.getCellCount(); i++) for(int i = 0; i < readerRow.getCellCount(); i++)
{ {
values[i] = readerRow.getCell(i).getText(); Cell cell = readerRow.getCell(i);
if(cell.getType() != null)
{
values[i] = switch(cell.getType())
{
case NUMBER ->
{
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// ... with fastexcel reader, we don't get styles... so, we just know type = number, for dates and ints & decimals... //
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
Optional<LocalDateTime> dateTime = readerRow.getCellAsDate(i);
if(dateTime.isPresent() && dateTime.get().getYear() > 1915 && dateTime.get().getYear() < 2100)
{
yield dateTime.get();
}
Optional<BigDecimal> optionalBigDecimal = readerRow.getCellAsNumber(i);
if(optionalBigDecimal.isPresent())
{
BigDecimal bigDecimal = optionalBigDecimal.get();
if(bigDecimal.subtract(bigDecimal.round(new MathContext(0))).compareTo(BigDecimal.ZERO) == 0)
{
yield bigDecimal.intValue();
}
yield bigDecimal;
}
yield (null);
}
case BOOLEAN -> readerRow.getCellAsBoolean(i).orElse(null);
case STRING, FORMULA -> cell.getText();
case EMPTY, ERROR -> null;
};
}
} }
return new BulkLoadFileRow(values); return new BulkLoadFileRow(values);

216
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkInsertWideLayoutMapping.java
View File

@ -1,216 +0,0 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2024. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow;
import com.kingsrook.qqq.backend.core.utils.ValueUtils;
/*******************************************************************************
**
*******************************************************************************/
public class BulkInsertWideLayoutMapping
{
private List<ChildRecordMapping> childRecordMappings;
/*******************************************************************************
** Constructor
**
*******************************************************************************/
public BulkInsertWideLayoutMapping(List<ChildRecordMapping> childRecordMappings)
{
this.childRecordMappings = childRecordMappings;
}
/***************************************************************************
**
***************************************************************************/
public static class ChildRecordMapping
{
Map<String, String> fieldNameToHeaderNameMaps;
Map<String, BulkInsertWideLayoutMapping> associationNameToChildRecordMappingMap;
/*******************************************************************************
** Constructor
**
*******************************************************************************/
public ChildRecordMapping(Map<String, String> fieldNameToHeaderNameMaps)
{
this.fieldNameToHeaderNameMaps = fieldNameToHeaderNameMaps;
}
/*******************************************************************************
** Constructor
**
*******************************************************************************/
public ChildRecordMapping(Map<String, String> fieldNameToHeaderNameMaps, Map<String, BulkInsertWideLayoutMapping> associationNameToChildRecordMappingMap)
{
this.fieldNameToHeaderNameMaps = fieldNameToHeaderNameMaps;
this.associationNameToChildRecordMappingMap = associationNameToChildRecordMappingMap;
}
/*******************************************************************************
** Getter for fieldNameToHeaderNameMaps
*******************************************************************************/
public Map<String, String> getFieldNameToHeaderNameMaps()
{
return (this.fieldNameToHeaderNameMaps);
}
/*******************************************************************************
** Setter for fieldNameToHeaderNameMaps
*******************************************************************************/
public void setFieldNameToHeaderNameMaps(Map<String, String> fieldNameToHeaderNameMaps)
{
this.fieldNameToHeaderNameMaps = fieldNameToHeaderNameMaps;
}
/*******************************************************************************
** Fluent setter for fieldNameToHeaderNameMaps
*******************************************************************************/
public ChildRecordMapping withFieldNameToHeaderNameMaps(Map<String, String> fieldNameToHeaderNameMaps)
{
this.fieldNameToHeaderNameMaps = fieldNameToHeaderNameMaps;
return (this);
}
/*******************************************************************************
** Getter for associationNameToChildRecordMappingMap
*******************************************************************************/
public Map<String, BulkInsertWideLayoutMapping> getAssociationNameToChildRecordMappingMap()
{
return (this.associationNameToChildRecordMappingMap);
}
/*******************************************************************************
** Setter for associationNameToChildRecordMappingMap
*******************************************************************************/
public void setAssociationNameToChildRecordMappingMap(Map<String, BulkInsertWideLayoutMapping> associationNameToChildRecordMappingMap)
{
this.associationNameToChildRecordMappingMap = associationNameToChildRecordMappingMap;
}
/*******************************************************************************
** Fluent setter for associationNameToChildRecordMappingMap
*******************************************************************************/
public ChildRecordMapping withAssociationNameToChildRecordMappingMap(Map<String, BulkInsertWideLayoutMapping> associationNameToChildRecordMappingMap)
{
this.associationNameToChildRecordMappingMap = associationNameToChildRecordMappingMap;
return (this);
}
/***************************************************************************
**
***************************************************************************/
public Map<String, Integer> getFieldIndexes(BulkLoadFileRow headerRow)
{
// todo memoize or otherwise don't recompute
Map<String, Integer> rs = new HashMap<>();
////////////////////////////////////////////////////////
// for the current file, map header values to indexes //
////////////////////////////////////////////////////////
Map<String, Integer> headerToIndexMap = new HashMap<>();
for(int i = 0; i < headerRow.size(); i++)
{
String headerValue = ValueUtils.getValueAsString(headerRow.getValue(i));
headerToIndexMap.put(headerValue, i);
}
/////////////////////////////////////////////////////////////////////////////////////////////////////////
// loop over fields - finding what header name they are mapped to - then what index that header is at. //
/////////////////////////////////////////////////////////////////////////////////////////////////////////
for(Map.Entry<String, String> entry : fieldNameToHeaderNameMaps.entrySet())
{
String headerName = entry.getValue();
if(headerName != null)
{
Integer headerIndex = headerToIndexMap.get(headerName);
if(headerIndex != null)
{
rs.put(entry.getKey(), headerIndex);
}
}
}
return (rs);
}
}
/*******************************************************************************
** Getter for childRecordMappings
*******************************************************************************/
public List<ChildRecordMapping> getChildRecordMappings()
{
return (this.childRecordMappings);
}
/*******************************************************************************
** Setter for childRecordMappings
*******************************************************************************/
public void setChildRecordMappings(List<ChildRecordMapping> childRecordMappings)
{
this.childRecordMappings = childRecordMappings;
}
/*******************************************************************************
** Fluent setter for childRecordMappings
*******************************************************************************/
public BulkInsertWideLayoutMapping withChildRecordMappings(List<ChildRecordMapping> childRecordMappings)
{
this.childRecordMappings = childRecordMappings;
return (this);
}
}

232
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadMappingSuggester.java Normal file
View File

@ -0,0 +1,232 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2024. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadProfile;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadProfileField;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadTableStructure;
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
import com.kingsrook.qqq.backend.core.utils.StringUtils;
import com.kingsrook.qqq.backend.core.utils.collections.ListBuilder;
/*******************************************************************************
** Given a bulk-upload, create a suggested mapping
*******************************************************************************/
public class BulkLoadMappingSuggester
{
private Map<String, Integer> massagedHeadersWithoutNumbersToIndexMap;
private Map<String, Integer> massagedHeadersWithNumbersToIndexMap;
private String layout = "FLAT";
/***************************************************************************
**
***************************************************************************/
public BulkLoadProfile suggestBulkLoadMappingProfile(BulkLoadTableStructure tableStructure, List<String> headerRow)
{
massagedHeadersWithoutNumbersToIndexMap = new LinkedHashMap<>();
for(int i = 0; i < headerRow.size(); i++)
{
String headerValue = massageHeader(headerRow.get(i), true);
if(!massagedHeadersWithoutNumbersToIndexMap.containsKey(headerValue))
{
massagedHeadersWithoutNumbersToIndexMap.put(headerValue, i);
}
}
massagedHeadersWithNumbersToIndexMap = new LinkedHashMap<>();
for(int i = 0; i < headerRow.size(); i++)
{
String headerValue = massageHeader(headerRow.get(i), false);
if(!massagedHeadersWithNumbersToIndexMap.containsKey(headerValue))
{
massagedHeadersWithNumbersToIndexMap.put(headerValue, i);
}
}
ArrayList<BulkLoadProfileField> fieldList = new ArrayList<>();
processTable(tableStructure, fieldList, headerRow);
/////////////////////////////////////////////////
// sort the fields to match the column indexes //
/////////////////////////////////////////////////
fieldList.sort(Comparator.comparing(blpf -> blpf.getColumnIndex()));
BulkLoadProfile bulkLoadProfile = new BulkLoadProfile()
.withVersion("v1")
.withLayout(layout)
.withHasHeaderRow(true)
.withFieldList(fieldList);
return (bulkLoadProfile);
}
/***************************************************************************
**
***************************************************************************/
private void processTable(BulkLoadTableStructure tableStructure, ArrayList<BulkLoadProfileField> fieldList, List<String> headerRow)
{
Map<String, Integer> rs = new HashMap<>();
for(QFieldMetaData field : tableStructure.getFields())
{
String fieldName = massageHeader(field.getName(), false);
String fieldLabel = massageHeader(field.getLabel(), false);
String tablePlusFieldLabel = massageHeader(QContext.getQInstance().getTable(tableStructure.getTableName()).getLabel() + ": " + field.getLabel(), false);
String fullFieldName = (StringUtils.hasContent(tableStructure.getAssociationPath()) ? (tableStructure.getAssociationPath() + ".") : "") + field.getName();
////////////////////////////////////////////////////////////////////////////////////
// consider, if this is a many-table, if there are many matches, for wide mode... //
////////////////////////////////////////////////////////////////////////////////////
if(tableStructure.getIsMany())
{
List<Integer> matchingIndexes = new ArrayList<>();
for(Map.Entry<String, Integer> entry : massagedHeadersWithNumbersToIndexMap.entrySet())
{
String header = entry.getKey();
if(header.matches(fieldName + "\\d*$") || header.matches(fieldLabel + "\\d*$"))
{
matchingIndexes.add(entry.getValue());
}
}
if(CollectionUtils.nullSafeHasContents(matchingIndexes))
{
///////////////////////////////////////////////////////////////////////////////////////////////////////
// if we found more than 1 match - consider this a likely wide file, and build fields as wide-fields //
// else, if only 1, allow us to go down into the TALL block below //
// note - should we do a merger at the end, in case we found some wide, some tall? //
///////////////////////////////////////////////////////////////////////////////////////////////////////
if(matchingIndexes.size() > 1)
{
layout = "WIDE";
int i = 0;
for(Integer index : matchingIndexes)
{
fieldList.add(new BulkLoadProfileField()
.withFieldName(fullFieldName + "," + i)
.withHeaderName(headerRow.get(index))
.withColumnIndex(index)
);
i++;
}
continue;
}
}
}
////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// else - look for matches, first w/ headers with numbers, then headers w/o numbers checking labels and names //
////////////////////////////////////////////////////////////////////////////////////////////////////////////////
Integer index = null;
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// for each of these potential identities of the field: //
// 1) its label, massaged //
// 2) its name, massaged //
// 3) its label, massaged, with numbers stripped away //
// 4) its name, massaged, with numbers stripped away //
// check if that identity is in the massagedHeadersWithNumbersToIndexMap, or the massagedHeadersWithoutNumbersToIndexMap. //
// this is currently successful in the both versions of the address 1 / address 2 <=> address / address 2 use-case //
// that is, BulkLoadMappingSuggesterTest.testChallengingAddress1And2 //
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
for(String fieldIdentity : ListBuilder.of(fieldLabel, fieldName, tablePlusFieldLabel, massageHeader(fieldLabel, true), massageHeader(fieldName, true)))
{
if(massagedHeadersWithNumbersToIndexMap.containsKey(fieldIdentity))
{
index = massagedHeadersWithNumbersToIndexMap.get(fieldIdentity);
}
else if(massagedHeadersWithoutNumbersToIndexMap.containsKey(fieldIdentity))
{
index = massagedHeadersWithoutNumbersToIndexMap.get(fieldIdentity);
}
if(index != null)
{
break;
}
}
if(index != null)
{
fieldList.add(new BulkLoadProfileField()
.withFieldName(fullFieldName)
.withHeaderName(headerRow.get(index))
.withColumnIndex(index)
);
if(tableStructure.getIsMany() && layout.equals("FLAT"))
{
//////////////////////////////////////////////////////////////////////////////////////////
// the first time we find an is-many child, if we were still marked as flat, go to tall //
//////////////////////////////////////////////////////////////////////////////////////////
layout = "TALL";
}
}
}
////////////////////////////////////////////
// recursively process child associations //
////////////////////////////////////////////
for(BulkLoadTableStructure associationTableStructure : CollectionUtils.nonNullList(tableStructure.getAssociations()))
{
processTable(associationTableStructure, fieldList, headerRow);
}
}
/***************************************************************************
**
***************************************************************************/
private String massageHeader(String header, boolean stripNumbers)
{
if(header == null)
{
return (null);
}
String massagedWithNumbers = header.toLowerCase(Locale.ROOT).replaceAll("[^a-z0-9]", "");
return stripNumbers ? massagedWithNumbers.replaceAll("[0-9]", "") : massagedWithNumbers;
}
}

132
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadTableStructureBuilder.java Normal file
View File

@ -0,0 +1,132 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2024. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.HashSet;
import java.util.Set;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinOn;
import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinType;
import com.kingsrook.qqq.backend.core.model.metadata.joins.QJoinMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.tables.Association;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadTableStructure;
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
import com.kingsrook.qqq.backend.core.utils.ObjectUtils;
import com.kingsrook.qqq.backend.core.utils.StringUtils;
/*******************************************************************************
** utility to build BulkLoadTableStructure objects for a QQQ Table.
*******************************************************************************/
public class BulkLoadTableStructureBuilder
{
/***************************************************************************
**
***************************************************************************/
public static BulkLoadTableStructure buildTableStructure(String tableName)
{
return (buildTableStructure(tableName, null, null));
}
/***************************************************************************
**
***************************************************************************/
private static BulkLoadTableStructure buildTableStructure(String tableName, Association association, String parentAssociationPath)
{
QTableMetaData table = QContext.getQInstance().getTable(tableName);
BulkLoadTableStructure tableStructure = new BulkLoadTableStructure();
tableStructure.setTableName(tableName);
tableStructure.setLabel(table.getLabel());
Set<String> associationJoinFieldNamesToExclude = new HashSet<>();
if(association == null)
{
tableStructure.setIsMain(true);
tableStructure.setIsMany(false);
tableStructure.setAssociationPath(null);
}
else
{
tableStructure.setIsMain(false);
QJoinMetaData join = QContext.getQInstance().getJoin(association.getJoinName());
if(join.getType().equals(JoinType.ONE_TO_MANY) || join.getType().equals(JoinType.MANY_TO_ONE))
{
tableStructure.setIsMany(true);
}
for(JoinOn joinOn : join.getJoinOns())
{
////////////////////////////////////////////////////////////////////////////////////////////////
// don't allow the user to map the "join field" from a child up to its parent //
// (e.g., you can't map lineItem.orderId -- that'll happen automatically via the association) //
////////////////////////////////////////////////////////////////////////////////////////////////
if(join.getLeftTable().equals(tableName))
{
associationJoinFieldNamesToExclude.add(joinOn.getLeftField());
}
else if(join.getRightTable().equals(tableName))
{
associationJoinFieldNamesToExclude.add(joinOn.getRightField());
}
}
if(!StringUtils.hasContent(parentAssociationPath))
{
tableStructure.setAssociationPath(association.getName());
}
else
{
tableStructure.setAssociationPath(parentAssociationPath + "." + association.getName());
}
}
ArrayList<QFieldMetaData> fields = new ArrayList<>();
tableStructure.setFields(fields);
for(QFieldMetaData field : table.getFields().values())
{
if(field.getIsEditable() && !associationJoinFieldNamesToExclude.contains(field.getName()))
{
fields.add(field);
}
}
fields.sort(Comparator.comparing(f -> ObjectUtils.requireNonNullElse(f.getLabel(), f.getName(), "")));
for(Association childAssociation : CollectionUtils.nonNullList(table.getAssociations()))
{
BulkLoadTableStructure associatedStructure = buildTableStructure(childAssociation.getAssociatedTableName(), childAssociation, parentAssociationPath);
tableStructure.addAssociation(associatedStructure);
}
return (tableStructure);
}
}

1
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/FlatRowsToRecord.java
View File

@ -31,6 +31,7 @@ import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.FileToRowsInterface; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.FileToRowsInterface;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkInsertMapping;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow;

58
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/RowsToRecordInterface.java
View File

@ -27,9 +27,10 @@ import java.util.List;
import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.FileToRowsInterface; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.FileToRowsInterface;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkInsertMapping;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow;
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
import com.kingsrook.qqq.backend.core.utils.StringUtils; import com.kingsrook.qqq.backend.core.utils.StringUtils;
@ -48,42 +49,49 @@ public interface RowsToRecordInterface
/*************************************************************************** /***************************************************************************
** returns true if value from row was used, else false. ** returns true if value from row was used, else false.
***************************************************************************/ ***************************************************************************/
default boolean setValueOrDefault(QRecord record, QFieldMetaData field, String associationNameChain, BulkInsertMapping mapping, BulkLoadFileRow row, Integer index) default boolean setValueOrDefault(QRecord record, QFieldMetaData field, String associationNameChain, BulkInsertMapping mapping, BulkLoadFileRow row, Integer columnIndex)
{ {
String fieldName = field.getName(); return setValueOrDefault(record, field, associationNameChain, mapping, row, columnIndex, null);
QFieldType type = field.getType(); }
boolean valueFromRowWasUsed = false; /***************************************************************************
String fieldNameWithAssociationPrefix = StringUtils.hasContent(associationNameChain) ? associationNameChain + "." + fieldName : fieldName; ** returns true if value from row was used, else false.
***************************************************************************/
default boolean setValueOrDefault(QRecord record, QFieldMetaData field, String associationNameChain, BulkInsertMapping mapping, BulkLoadFileRow row, Integer columnIndex, List<Integer> wideAssociationIndexes)
{
boolean valueFromRowWasUsed = false;
Serializable value = null; /////////////////////////////////////////////////////////////////////////////////////////////////
if(index != null && row != null) // build full field-name -- possibly associations, then field name, then possibly index-suffix //
/////////////////////////////////////////////////////////////////////////////////////////////////
String fieldName = field.getName();
String fieldNameWithAssociationPrefix = StringUtils.hasContent(associationNameChain) ? associationNameChain + "." + fieldName : fieldName;
String wideAssociationSuffix = "";
if(CollectionUtils.nullSafeHasContents(wideAssociationIndexes))
{ {
value = row.getValueElseNull(index); wideAssociationSuffix = "," + StringUtils.join(".", wideAssociationIndexes);
}
String fullFieldName = fieldNameWithAssociationPrefix + wideAssociationSuffix;
//////////////////////////////////////////////
// ok - look in the row - then the defaults //
//////////////////////////////////////////////
Serializable value = null;
if(columnIndex != null && row != null)
{
value = row.getValueElseNull(columnIndex);
if(value != null && !"".equals(value)) if(value != null && !"".equals(value))
{ {
valueFromRowWasUsed = true; valueFromRowWasUsed = true;
} }
} }
else if(mapping.getFieldNameToDefaultValueMap().containsKey(fieldNameWithAssociationPrefix)) else if(mapping.getFieldNameToDefaultValueMap().containsKey(fullFieldName))
{ {
value = mapping.getFieldNameToDefaultValueMap().get(fieldNameWithAssociationPrefix); value = mapping.getFieldNameToDefaultValueMap().get(fullFieldName);
} }
/* note - moving this to ValueMapper...
if(value != null)
{
try
{
value = ValueUtils.getValueAsFieldType(type, value);
}
catch(Exception e)
{
record.addError(new BadInputStatusMessage("Value [" + value + "] for field [" + field.getLabel() + "] could not be converted to type [" + type + "]"));
}
}
*/
if(value != null) if(value != null)
{ {
record.setValue(fieldName, value); record.setValue(fieldName, value);

1
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/TallRowsToRecord.java
View File

@ -36,6 +36,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.tables.Association; import com.kingsrook.qqq.backend.core.model.metadata.tables.Association;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.FileToRowsInterface; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.FileToRowsInterface;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkInsertMapping;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow;
import com.kingsrook.qqq.backend.core.utils.CollectionUtils; import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
import com.kingsrook.qqq.backend.core.utils.Pair; import com.kingsrook.qqq.backend.core.utils.Pair;

1
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/ValueMapper.java
View File

@ -35,6 +35,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
import com.kingsrook.qqq.backend.core.model.metadata.tables.Association; import com.kingsrook.qqq.backend.core.model.metadata.tables.Association;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.model.statusmessages.BadInputStatusMessage; import com.kingsrook.qqq.backend.core.model.statusmessages.BadInputStatusMessage;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkInsertMapping;
import com.kingsrook.qqq.backend.core.utils.CollectionUtils; import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
import com.kingsrook.qqq.backend.core.utils.StringUtils; import com.kingsrook.qqq.backend.core.utils.StringUtils;
import com.kingsrook.qqq.backend.core.utils.ValueUtils; import com.kingsrook.qqq.backend.core.utils.ValueUtils;

196
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMapping.java Normal file
View File

@ -0,0 +1,196 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2024. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.tables.Association;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.FileToRowsInterface;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkInsertMapping;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow;
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
import com.kingsrook.qqq.backend.core.utils.Pair;
import com.kingsrook.qqq.backend.core.utils.StringUtils;
import com.kingsrook.qqq.backend.core.utils.memoization.Memoization;
/*******************************************************************************
** use a flatter mapping object, where field names look like:
** associationChain.fieldName,index.subIndex
*******************************************************************************/
public class WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMapping implements RowsToRecordInterface
{
private Memoization<Pair<String, String>, Boolean> shouldProcesssAssociationMemoization = new Memoization<>();
/***************************************************************************
**
***************************************************************************/
@Override
public List<QRecord> nextPage(FileToRowsInterface fileToRowsInterface, BulkLoadFileRow headerRow, BulkInsertMapping mapping, Integer limit) throws QException
{
QTableMetaData table = QContext.getQInstance().getTable(mapping.getTableName());
if(table == null)
{
throw (new QException("Table [" + mapping.getTableName() + "] was not found in the Instance"));
}
List<QRecord> rs = new ArrayList<>();
Map<String, Integer> fieldIndexes = mapping.getFieldIndexes(table, null, headerRow);
while(fileToRowsInterface.hasNext() && rs.size() < limit)
{
BulkLoadFileRow row = fileToRowsInterface.next();
QRecord record = makeRecordFromRow(mapping, table, "", row, fieldIndexes, headerRow, new ArrayList<>());
rs.add(record);
}
ValueMapper.valueMapping(rs, mapping, table);
return (rs);
}
/***************************************************************************
** may return null, if there were no values in the row for this (sub-wide) record.
***************************************************************************/
private QRecord makeRecordFromRow(BulkInsertMapping mapping, QTableMetaData table, String associationNameChain, BulkLoadFileRow row, Map<String, Integer> fieldIndexes, BulkLoadFileRow headerRow, List<Integer> wideAssociationIndexes) throws QException
{
//////////////////////////////////////////////////////
// start by building the record with its own fields //
//////////////////////////////////////////////////////
QRecord record = new QRecord();
boolean hadAnyValuesInRow = false;
for(QFieldMetaData field : table.getFields().values())
{
hadAnyValuesInRow = setValueOrDefault(record, field, associationNameChain, mapping, row, fieldIndexes.get(field.getName()), wideAssociationIndexes) || hadAnyValuesInRow;
}
if(!hadAnyValuesInRow)
{
return (null);
}
/////////////////////////////
// associations (children) //
/////////////////////////////
for(String associationName : CollectionUtils.nonNullList(mapping.getMappedAssociations()))
{
boolean processAssociation = shouldProcessAssociation(associationNameChain, associationName);
if(processAssociation)
{
String associationNameMinusChain = StringUtils.hasContent(associationNameChain)
? associationName.substring(associationNameChain.length() + 1)
: associationName;
Optional<Association> association = table.getAssociations().stream().filter(a -> a.getName().equals(associationNameMinusChain)).findFirst();
if(association.isEmpty())
{
throw (new QException("Couldn't find association: " + associationNameMinusChain + " under table: " + table.getName()));
}
QTableMetaData associatedTable = QContext.getQInstance().getTable(association.get().getAssociatedTableName());
List<QRecord> associatedRecords = processAssociation(associationNameMinusChain, associationNameChain, associatedTable, mapping, row, headerRow);
record.withAssociatedRecords(associationNameMinusChain, associatedRecords);
}
}
return record;
}
/***************************************************************************
**
***************************************************************************/
private List<QRecord> processAssociation(String associationName, String associationNameChain, QTableMetaData associatedTable, BulkInsertMapping mapping, BulkLoadFileRow row, BulkLoadFileRow headerRow) throws QException
{
List<QRecord> rs = new ArrayList<>();
String associationNameChainForRecursiveCalls = "".equals(associationNameChain) ? associationName : associationNameChain + "." + associationName;
for(int i = 0; true; i++)
{
// todo - doesn't support grand-children
List<Integer> wideAssociationIndexes = List.of(i);
Map<String, Integer> fieldIndexes = mapping.getFieldIndexes(associatedTable, associationNameChainForRecursiveCalls, headerRow, wideAssociationIndexes);
if(fieldIndexes.isEmpty())
{
break;
}
QRecord record = makeRecordFromRow(mapping, associatedTable, associationNameChainForRecursiveCalls, row, fieldIndexes, headerRow, wideAssociationIndexes);
if(record != null)
{
rs.add(record);
}
}
return (rs);
}
/***************************************************************************
**
***************************************************************************/
boolean shouldProcessAssociation(String associationNameChain, String associationName)
{
return shouldProcesssAssociationMemoization.getResult(Pair.of(associationNameChain, associationName), p ->
{
List<String> chainParts = new ArrayList<>();
List<String> nameParts = new ArrayList<>();
if(StringUtils.hasContent(associationNameChain))
{
chainParts.addAll(Arrays.asList(associationNameChain.split("\\.")));
}
if(StringUtils.hasContent(associationName))
{
nameParts.addAll(Arrays.asList(associationName.split("\\.")));
}
if(!nameParts.isEmpty())
{
nameParts.remove(nameParts.size() - 1);
}
return (chainParts.equals(nameParts));
}).orElse(false);
}
}

260
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithExplicitMapping.java
View File

@ -1,260 +0,0 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2024. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.tables.Association;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.FileToRowsInterface;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow;
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
import com.kingsrook.qqq.backend.core.utils.Pair;
import com.kingsrook.qqq.backend.core.utils.StringUtils;
import com.kingsrook.qqq.backend.core.utils.memoization.Memoization;
/*******************************************************************************
**
*******************************************************************************/
public class WideRowsToRecordWithExplicitMapping implements RowsToRecordInterface
{
private Memoization<Pair<String, String>, Boolean> shouldProcesssAssociationMemoization = new Memoization<>();
/***************************************************************************
**
***************************************************************************/
@Override
public List<QRecord> nextPage(FileToRowsInterface fileToRowsInterface, BulkLoadFileRow headerRow, BulkInsertMapping mapping, Integer limit) throws QException
{
QTableMetaData table = QContext.getQInstance().getTable(mapping.getTableName());
if(table == null)
{
throw (new QException("Table [" + mapping.getTableName() + "] was not found in the Instance"));
}
List<QRecord> rs = new ArrayList<>();
Map<String, Integer> fieldIndexes = mapping.getFieldIndexes(table, null, headerRow);
while(fileToRowsInterface.hasNext() && rs.size() < limit)
{
BulkLoadFileRow row = fileToRowsInterface.next();
QRecord record = new QRecord();
for(QFieldMetaData field : table.getFields().values())
{
setValueOrDefault(record, field, null, mapping, row, fieldIndexes.get(field.getName()));
}
processAssociations(mapping.getWideLayoutMapping(), "", headerRow, mapping, table, row, record);
rs.add(record);
}
ValueMapper.valueMapping(rs, mapping, table);
return (rs);
}
/***************************************************************************
**
***************************************************************************/
private void processAssociations(Map<String, BulkInsertWideLayoutMapping> mappingMap, String associationNameChain, BulkLoadFileRow headerRow, BulkInsertMapping mapping, QTableMetaData table, BulkLoadFileRow row, QRecord record) throws QException
{
for(Map.Entry<String, BulkInsertWideLayoutMapping> entry : CollectionUtils.nonNullMap(mappingMap).entrySet())
{
String associationName = entry.getKey();
BulkInsertWideLayoutMapping bulkInsertWideLayoutMapping = entry.getValue();
Optional<Association> association = table.getAssociations().stream().filter(a -> a.getName().equals(associationName)).findFirst();
if(association.isEmpty())
{
throw (new QException("Couldn't find association: " + associationName + " under table: " + table.getName()));
}
QTableMetaData associatedTable = QContext.getQInstance().getTable(association.get().getAssociatedTableName());
String subChain = StringUtils.hasContent(associationNameChain) ? associationNameChain + "." + associationName: associationName;
for(BulkInsertWideLayoutMapping.ChildRecordMapping childRecordMapping : bulkInsertWideLayoutMapping.getChildRecordMappings())
{
QRecord associatedRecord = processAssociation(associatedTable, subChain, childRecordMapping, mapping, row, headerRow);
if(associatedRecord != null)
{
record.withAssociatedRecord(associationName, associatedRecord);
}
}
}
}
/***************************************************************************
**
***************************************************************************/
private QRecord processAssociation(QTableMetaData table, String associationNameChain, BulkInsertWideLayoutMapping.ChildRecordMapping childRecordMapping, BulkInsertMapping mapping, BulkLoadFileRow row, BulkLoadFileRow headerRow) throws QException
{
Map<String, Integer> fieldIndexes = childRecordMapping.getFieldIndexes(headerRow);
QRecord associatedRecord = new QRecord();
boolean usedAnyValuesFromRow = false;
for(QFieldMetaData field : table.getFields().values())
{
boolean valueFromRowWasUsed = setValueOrDefault(associatedRecord, field, associationNameChain, mapping, row, fieldIndexes.get(field.getName()));
usedAnyValuesFromRow |= valueFromRowWasUsed;
}
if(usedAnyValuesFromRow)
{
processAssociations(childRecordMapping.getAssociationNameToChildRecordMappingMap(), associationNameChain, headerRow, mapping, table, row, associatedRecord);
return (associatedRecord);
}
else
{
return (null);
}
}
// /***************************************************************************
// **
// ***************************************************************************/
// private List<QRecord> processAssociationV2(String associationName, String associationNameChain, QTableMetaData table, BulkInsertMapping mapping, BulkLoadFileRow row, BulkLoadFileRow headerRow, QRecord record, int startIndex, int endIndex) throws QException
// {
// List<QRecord> rs = new ArrayList<>();
// Map<String, String> fieldNameToHeaderNameMapForThisAssociation = new HashMap<>();
// for(Map.Entry<String, String> entry : mapping.getFieldNameToHeaderNameMap().entrySet())
// {
// if(entry.getKey().startsWith(associationName + "."))
// {
// String fieldName = entry.getKey().substring(associationName.length() + 1);
// //////////////////////////////////////////////////////////////////////////
// // make sure the name here is for this table - not a sub-table under it //
// //////////////////////////////////////////////////////////////////////////
// if(!fieldName.contains("."))
// {
// fieldNameToHeaderNameMapForThisAssociation.put(fieldName, entry.getValue());
// }
// }
// }
// /////////////////////////////////////////////////////////////////////
// // loop over the length of the record, building associated records //
// /////////////////////////////////////////////////////////////////////
// QRecord associatedRecord = new QRecord();
// Set<String> processedFieldNames = new HashSet<>();
// boolean gotAnyValues = false;
// int subStartIndex = -1;
// for(int i = startIndex; i < endIndex; i++)
// {
// String headerValue = ValueUtils.getValueAsString(headerRow.getValue(i));
// for(Map.Entry<String, String> entry : fieldNameToHeaderNameMapForThisAssociation.entrySet())
// {
// if(headerValue.equals(entry.getValue()) || headerValue.matches(entry.getValue() + " ?\\d+"))
// {
// ///////////////////////////////////////////////
// // ok - this is a value for this association //
// ///////////////////////////////////////////////
// if(subStartIndex == -1)
// {
// subStartIndex = i;
// }
// String fieldName = entry.getKey();
// if(processedFieldNames.contains(fieldName))
// {
// /////////////////////////////////////////////////
// // this means we're starting a new sub-record! //
// /////////////////////////////////////////////////
// if(gotAnyValues)
// {
// addDefaultValuesToAssociatedRecord(processedFieldNames, table, associatedRecord, mapping, associationName);
// processAssociations(associationName, headerRow, mapping, table, row, associatedRecord, subStartIndex, i);
// rs.add(associatedRecord);
// }
// associatedRecord = new QRecord();
// processedFieldNames = new HashSet<>();
// gotAnyValues = false;
// subStartIndex = i + 1;
// }
// processedFieldNames.add(fieldName);
// Serializable value = row.getValueElseNull(i);
// if(value != null && !"".equals(value))
// {
// gotAnyValues = true;
// }
// setValueOrDefault(associatedRecord, fieldName, associationName, mapping, row, i);
// }
// }
// }
// ////////////////////////
// // handle final value //
// ////////////////////////
// if(gotAnyValues)
// {
// addDefaultValuesToAssociatedRecord(processedFieldNames, table, associatedRecord, mapping, associationName);
// processAssociations(associationName, headerRow, mapping, table, row, associatedRecord, subStartIndex, endIndex);
// rs.add(associatedRecord);
// }
// return (rs);
// }
/***************************************************************************
**
***************************************************************************/
private void addDefaultValuesToAssociatedRecord(Set<String> processedFieldNames, QTableMetaData table, QRecord associatedRecord, BulkInsertMapping mapping, String associationNameChain)
{
for(QFieldMetaData field : table.getFields().values())
{
if(!processedFieldNames.contains(field.getName()))
{
setValueOrDefault(associatedRecord, field, associationNameChain, mapping, null, null);
}
}
}
}

1
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithSpreadMapping.java
View File

@ -38,6 +38,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.tables.Association; import com.kingsrook.qqq.backend.core.model.metadata.tables.Association;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.FileToRowsInterface; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.FileToRowsInterface;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkInsertMapping;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow;
import com.kingsrook.qqq.backend.core.utils.Pair; import com.kingsrook.qqq.backend.core.utils.Pair;
import com.kingsrook.qqq.backend.core.utils.StringUtils; import com.kingsrook.qqq.backend.core.utils.StringUtils;

65
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkInsertMapping.java → qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/model/BulkInsertMapping.java
View File

@ -19,7 +19,7 @@
* along with this program. If not, see <https://www.gnu.org/licenses/>. * along with this program. If not, see <https://www.gnu.org/licenses/>.
*/ */
package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping; package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model;
import java.io.Serializable; import java.io.Serializable;
@ -34,7 +34,10 @@ import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.PossibleValueEnum; import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.PossibleValueEnum;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping.FlatRowsToRecord;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping.RowsToRecordInterface;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping.TallRowsToRecord;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping.WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMapping;
import com.kingsrook.qqq.backend.core.utils.CollectionUtils; import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
import com.kingsrook.qqq.backend.core.utils.Pair; import com.kingsrook.qqq.backend.core.utils.Pair;
import com.kingsrook.qqq.backend.core.utils.StringUtils; import com.kingsrook.qqq.backend.core.utils.StringUtils;
@ -63,8 +66,7 @@ public class BulkInsertMapping implements Serializable
private Map<String, Serializable> fieldNameToDefaultValueMap = new HashMap<>(); private Map<String, Serializable> fieldNameToDefaultValueMap = new HashMap<>();
private Map<String, Map<String, Serializable>> fieldNameToValueMapping = new HashMap<>(); private Map<String, Map<String, Serializable>> fieldNameToValueMapping = new HashMap<>();
private Map<String, List<Integer>> tallLayoutGroupByIndexMap = new HashMap<>(); private Map<String, List<Integer>> tallLayoutGroupByIndexMap = new HashMap<>();
private Map<String, BulkInsertWideLayoutMapping> wideLayoutMapping = new HashMap<>();
private List<String> mappedAssociations = new ArrayList<>(); private List<String> mappedAssociations = new ArrayList<>();
@ -79,7 +81,7 @@ public class BulkInsertMapping implements Serializable
{ {
FLAT(FlatRowsToRecord::new), FLAT(FlatRowsToRecord::new),
TALL(TallRowsToRecord::new), TALL(TallRowsToRecord::new),
WIDE(WideRowsToRecordWithExplicitMapping::new); WIDE(WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMapping::new);
/*************************************************************************** /***************************************************************************
@ -137,10 +139,21 @@ public class BulkInsertMapping implements Serializable
***************************************************************************/ ***************************************************************************/
@JsonIgnore @JsonIgnore
public Map<String, Integer> getFieldIndexes(QTableMetaData table, String associationNameChain, BulkLoadFileRow headerRow) throws QException public Map<String, Integer> getFieldIndexes(QTableMetaData table, String associationNameChain, BulkLoadFileRow headerRow) throws QException
{
return getFieldIndexes(table, associationNameChain, headerRow, null);
}
/***************************************************************************
**
***************************************************************************/
@JsonIgnore
public Map<String, Integer> getFieldIndexes(QTableMetaData table, String associationNameChain, BulkLoadFileRow headerRow, List<Integer> wideAssociationIndexes) throws QException
{ {
if(hasHeaderRow && fieldNameToHeaderNameMap != null) if(hasHeaderRow && fieldNameToHeaderNameMap != null)
{ {
return (getFieldIndexesForHeaderMappedUseCase(table, associationNameChain, headerRow)); return (getFieldIndexesForHeaderMappedUseCase(table, associationNameChain, headerRow, wideAssociationIndexes));
} }
else if(fieldNameToIndexMap != null) else if(fieldNameToIndexMap != null)
{ {
@ -208,7 +221,7 @@ public class BulkInsertMapping implements Serializable
/*************************************************************************** /***************************************************************************
** **
***************************************************************************/ ***************************************************************************/
private Map<String, Integer> getFieldIndexesForHeaderMappedUseCase(QTableMetaData table, String associationNameChain, BulkLoadFileRow headerRow) private Map<String, Integer> getFieldIndexesForHeaderMappedUseCase(QTableMetaData table, String associationNameChain, BulkLoadFileRow headerRow, List<Integer> wideAssociationIndexes)
{ {
Map<String, Integer> rs = new HashMap<>(); Map<String, Integer> rs = new HashMap<>();
@ -222,13 +235,19 @@ public class BulkInsertMapping implements Serializable
headerToIndexMap.put(headerValue, i); headerToIndexMap.put(headerValue, i);
} }
String wideAssociationSuffix = "";
if(CollectionUtils.nullSafeHasContents(wideAssociationIndexes))
{
wideAssociationSuffix = "," + StringUtils.join(".", wideAssociationIndexes);
}
///////////////////////////////////////////////////////////////////////////////////////////////////////// /////////////////////////////////////////////////////////////////////////////////////////////////////////
// loop over fields - finding what header name they are mapped to - then what index that header is at. // // loop over fields - finding what header name they are mapped to - then what index that header is at. //
///////////////////////////////////////////////////////////////////////////////////////////////////////// /////////////////////////////////////////////////////////////////////////////////////////////////////////
String fieldNamePrefix = !StringUtils.hasContent(associationNameChain) ? "" : associationNameChain + "."; String fieldNamePrefix = !StringUtils.hasContent(associationNameChain) ? "" : associationNameChain + ".";
for(QFieldMetaData field : table.getFields().values()) for(QFieldMetaData field : table.getFields().values())
{ {
String headerName = fieldNameToHeaderNameMap.get(fieldNamePrefix + field.getName()); String headerName = fieldNameToHeaderNameMap.get(fieldNamePrefix + field.getName() + wideAssociationSuffix);
if(headerName != null) if(headerName != null)
{ {
Integer headerIndex = headerToIndexMap.get(headerName); Integer headerIndex = headerToIndexMap.get(headerName);
@ -527,34 +546,4 @@ public class BulkInsertMapping implements Serializable
} }
/*******************************************************************************
** Getter for wideLayoutMapping
*******************************************************************************/
public Map<String, BulkInsertWideLayoutMapping> getWideLayoutMapping()
{
return (this.wideLayoutMapping);
}
/*******************************************************************************
** Setter for wideLayoutMapping
*******************************************************************************/
public void setWideLayoutMapping(Map<String, BulkInsertWideLayoutMapping> wideLayoutMapping)
{
this.wideLayoutMapping = wideLayoutMapping;
}
/*******************************************************************************
** Fluent setter for wideLayoutMapping
*******************************************************************************/
public BulkInsertMapping withWideLayoutMapping(Map<String, BulkInsertWideLayoutMapping> wideLayoutMapping)
{
this.wideLayoutMapping = wideLayoutMapping;
return (this);
}
} }

32
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/model/BulkLoadProfileField.java
View File

@ -33,6 +33,7 @@ public class BulkLoadProfileField
{ {
private String fieldName; private String fieldName;
private Integer columnIndex; private Integer columnIndex;
private String headerName;
private Serializable defaultValue; private Serializable defaultValue;
private Boolean doValueMapping; private Boolean doValueMapping;
private Map<String, Serializable> valueMappings; private Map<String, Serializable> valueMappings;
@ -192,4 +193,35 @@ public class BulkLoadProfileField
return (this); return (this);
} }
/*******************************************************************************
** Getter for headerName
*******************************************************************************/
public String getHeaderName()
{
return (this.headerName);
}
/*******************************************************************************
** Setter for headerName
*******************************************************************************/
public void setHeaderName(String headerName)
{
this.headerName = headerName;
}
/*******************************************************************************
** Fluent setter for headerName
*******************************************************************************/
public BulkLoadProfileField withHeaderName(String headerName)
{
this.headerName = headerName;
return (this);
}
} }

15
qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertV2Test.java → qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertV2FullProcessTest.java
View File

@ -42,6 +42,7 @@ import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.frontend.QFrontendFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.frontend.QFrontendFieldMetaData;
import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryRecordStore; import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryRecordStore;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadProfile;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadProfileField; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadProfileField;
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess; import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess;
import com.kingsrook.qqq.backend.core.utils.JsonUtils; import com.kingsrook.qqq.backend.core.utils.JsonUtils;
@ -58,7 +59,7 @@ import static org.junit.jupiter.api.Assertions.assertNull;
/******************************************************************************* /*******************************************************************************
** Unit test for full bulk insert process ** Unit test for full bulk insert process
*******************************************************************************/ *******************************************************************************/
class BulkInsertV2Test extends BaseTest class BulkInsertV2FullProcessTest extends BaseTest
{ {
/******************************************************************************* /*******************************************************************************
@ -124,7 +125,7 @@ class BulkInsertV2Test extends BaseTest
QInstance qInstance = QContext.getQInstance(); QInstance qInstance = QContext.getQInstance();
String processName = "PersonBulkInsertV2"; String processName = "PersonBulkInsertV2";
new QInstanceEnricher(qInstance).defineTableBulkInsertV2(qInstance, qInstance.getTable(TestUtils.TABLE_NAME_PERSON_MEMORY), processName); new QInstanceEnricher(qInstance).defineTableBulkInsert(qInstance, qInstance.getTable(TestUtils.TABLE_NAME_PERSON_MEMORY), processName);
///////////////////////////////////////////////////////// /////////////////////////////////////////////////////////
// start the process - expect to go to the upload step // // start the process - expect to go to the upload step //
@ -159,6 +160,16 @@ class BulkInsertV2Test extends BaseTest
runProcessOutput = new RunProcessAction().execute(runProcessInput); runProcessOutput = new RunProcessAction().execute(runProcessInput);
assertEquals(List.of("Id", "Create Date", "Modify Date", "First Name", "Last Name", "Birth Date", "Email", "Home State", "noOfShoes"), runProcessOutput.getValue("headerValues")); assertEquals(List.of("Id", "Create Date", "Modify Date", "First Name", "Last Name", "Birth Date", "Email", "Home State", "noOfShoes"), runProcessOutput.getValue("headerValues"));
assertEquals(List.of("A", "B", "C", "D", "E", "F", "G", "H", "I"), runProcessOutput.getValue("headerLetters")); assertEquals(List.of("A", "B", "C", "D", "E", "F", "G", "H", "I"), runProcessOutput.getValue("headerLetters"));
//////////////////////////////////////////////////////
// assert about the suggested mapping that was done //
//////////////////////////////////////////////////////
Serializable bulkLoadProfile = runProcessOutput.getValue("bulkLoadProfile");
assertThat(bulkLoadProfile).isInstanceOf(BulkLoadProfile.class);
assertThat(((BulkLoadProfile) bulkLoadProfile).getFieldList()).hasSizeGreaterThan(5);
assertEquals("birthDate", ((BulkLoadProfile) bulkLoadProfile).getFieldList().get(0).getFieldName());
assertEquals(5, ((BulkLoadProfile) bulkLoadProfile).getFieldList().get(0).getColumnIndex());
assertThat(runProcessOutput.getProcessState().getNextStepName()).isPresent().get().isEqualTo("fileMapping"); assertThat(runProcessOutput.getProcessState().getNextStepName()).isPresent().get().isEqualTo("fileMapping");
//////////////////////////////////////////////////////////////////////////////// ////////////////////////////////////////////////////////////////////////////////

209
qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadMappingSuggesterTest.java Normal file
View File

@ -0,0 +1,209 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2024. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping;
import java.util.List;
import com.kingsrook.qqq.backend.core.BaseTest;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadProfile;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadProfileField;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadTableStructure;
import com.kingsrook.qqq.backend.core.utils.TestUtils;
import org.junit.jupiter.api.Test;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNull;
/*******************************************************************************
** Unit test for BulkLoadMappingSuggester
*******************************************************************************/
class BulkLoadMappingSuggesterTest extends BaseTest
{
/*******************************************************************************
**
*******************************************************************************/
@Test
void testSimpleFlat()
{
BulkLoadTableStructure tableStructure = BulkLoadTableStructureBuilder.buildTableStructure(TestUtils.TABLE_NAME_PERSON_MEMORY);
List<String> headerRow = List.of("Id", "First Name", "lastname", "email", "homestate");
BulkLoadProfile bulkLoadProfile = new BulkLoadMappingSuggester().suggestBulkLoadMappingProfile(tableStructure, headerRow);
assertEquals("v1", bulkLoadProfile.getVersion());
assertEquals("FLAT", bulkLoadProfile.getLayout());
assertNull(getFieldByName(bulkLoadProfile, "id"));
assertEquals(1, getFieldByName(bulkLoadProfile, "firstName").getColumnIndex());
assertEquals(2, getFieldByName(bulkLoadProfile, "lastName").getColumnIndex());
assertEquals(3, getFieldByName(bulkLoadProfile, "email").getColumnIndex());
assertEquals(4, getFieldByName(bulkLoadProfile, "homeStateId").getColumnIndex());
}
/*******************************************************************************
**
*******************************************************************************/
@Test
void testSimpleTall()
{
BulkLoadTableStructure tableStructure = BulkLoadTableStructureBuilder.buildTableStructure(TestUtils.TABLE_NAME_ORDER);
List<String> headerRow = List.of("orderNo", "shipto name", "sku", "quantity");
BulkLoadProfile bulkLoadProfile = new BulkLoadMappingSuggester().suggestBulkLoadMappingProfile(tableStructure, headerRow);
assertEquals("v1", bulkLoadProfile.getVersion());
assertEquals("TALL", bulkLoadProfile.getLayout());
assertEquals(0, getFieldByName(bulkLoadProfile, "orderNo").getColumnIndex());
assertEquals(1, getFieldByName(bulkLoadProfile, "shipToName").getColumnIndex());
assertEquals(2, getFieldByName(bulkLoadProfile, "orderLine.sku").getColumnIndex());
assertEquals(3, getFieldByName(bulkLoadProfile, "orderLine.quantity").getColumnIndex());
}
/*******************************************************************************
**
*******************************************************************************/
@Test
void testTallWithTableNamesOnAssociations()
{
BulkLoadTableStructure tableStructure = BulkLoadTableStructureBuilder.buildTableStructure(TestUtils.TABLE_NAME_ORDER);
List<String> headerRow = List.of("Order No", "Ship To Name", "Order Line: SKU", "Order Line: Quantity");
BulkLoadProfile bulkLoadProfile = new BulkLoadMappingSuggester().suggestBulkLoadMappingProfile(tableStructure, headerRow);
assertEquals("v1", bulkLoadProfile.getVersion());
assertEquals("TALL", bulkLoadProfile.getLayout());
assertEquals(0, getFieldByName(bulkLoadProfile, "orderNo").getColumnIndex());
assertEquals(1, getFieldByName(bulkLoadProfile, "shipToName").getColumnIndex());
assertEquals(2, getFieldByName(bulkLoadProfile, "orderLine.sku").getColumnIndex());
assertEquals(3, getFieldByName(bulkLoadProfile, "orderLine.quantity").getColumnIndex());
}
/*******************************************************************************
**
*******************************************************************************/
@Test
void testChallengingAddress1And2()
{
try
{
{
QTableMetaData table = QContext.getQInstance().getTable(TestUtils.TABLE_NAME_ORDER);
table.addField(new QFieldMetaData("address1", QFieldType.STRING));
table.addField(new QFieldMetaData("address2", QFieldType.STRING));
BulkLoadTableStructure tableStructure = BulkLoadTableStructureBuilder.buildTableStructure(TestUtils.TABLE_NAME_ORDER);
List<String> headerRow = List.of("orderNo", "ship to name", "address 1", "address 2");
BulkLoadProfile bulkLoadProfile = new BulkLoadMappingSuggester().suggestBulkLoadMappingProfile(tableStructure, headerRow);
assertEquals(0, getFieldByName(bulkLoadProfile, "orderNo").getColumnIndex());
assertEquals(1, getFieldByName(bulkLoadProfile, "shipToName").getColumnIndex());
assertEquals(2, getFieldByName(bulkLoadProfile, "address1").getColumnIndex());
assertEquals(3, getFieldByName(bulkLoadProfile, "address2").getColumnIndex());
reInitInstanceInContext(TestUtils.defineInstance());
}
{
QTableMetaData table = QContext.getQInstance().getTable(TestUtils.TABLE_NAME_ORDER);
table.addField(new QFieldMetaData("address", QFieldType.STRING));
table.addField(new QFieldMetaData("address2", QFieldType.STRING));
BulkLoadTableStructure tableStructure = BulkLoadTableStructureBuilder.buildTableStructure(TestUtils.TABLE_NAME_ORDER);
List<String> headerRow = List.of("orderNo", "ship to name", "address 1", "address 2");
BulkLoadProfile bulkLoadProfile = new BulkLoadMappingSuggester().suggestBulkLoadMappingProfile(tableStructure, headerRow);
assertEquals(0, getFieldByName(bulkLoadProfile, "orderNo").getColumnIndex());
assertEquals(1, getFieldByName(bulkLoadProfile, "shipToName").getColumnIndex());
assertEquals(2, getFieldByName(bulkLoadProfile, "address").getColumnIndex());
assertEquals(3, getFieldByName(bulkLoadProfile, "address2").getColumnIndex());
reInitInstanceInContext(TestUtils.defineInstance());
}
{
QTableMetaData table = QContext.getQInstance().getTable(TestUtils.TABLE_NAME_ORDER);
table.addField(new QFieldMetaData("address1", QFieldType.STRING));
table.addField(new QFieldMetaData("address2", QFieldType.STRING));
BulkLoadTableStructure tableStructure = BulkLoadTableStructureBuilder.buildTableStructure(TestUtils.TABLE_NAME_ORDER);
List<String> headerRow = List.of("orderNo", "ship to name", "address", "address 2");
BulkLoadProfile bulkLoadProfile = new BulkLoadMappingSuggester().suggestBulkLoadMappingProfile(tableStructure, headerRow);
assertEquals(0, getFieldByName(bulkLoadProfile, "orderNo").getColumnIndex());
assertEquals(1, getFieldByName(bulkLoadProfile, "shipToName").getColumnIndex());
assertEquals(2, getFieldByName(bulkLoadProfile, "address1").getColumnIndex());
assertEquals(3, getFieldByName(bulkLoadProfile, "address2").getColumnIndex());
reInitInstanceInContext(TestUtils.defineInstance());
}
}
finally
{
reInitInstanceInContext(TestUtils.defineInstance());
}
}
/*******************************************************************************
**
*******************************************************************************/
@Test
void testSimpleWide()
{
BulkLoadTableStructure tableStructure = BulkLoadTableStructureBuilder.buildTableStructure(TestUtils.TABLE_NAME_ORDER);
List<String> headerRow = List.of("orderNo", "ship to name", "sku", "quantity1", "sku 2", "quantity 2");
BulkLoadProfile bulkLoadProfile = new BulkLoadMappingSuggester().suggestBulkLoadMappingProfile(tableStructure, headerRow);
assertEquals("v1", bulkLoadProfile.getVersion());
assertEquals("WIDE", bulkLoadProfile.getLayout());
assertEquals(0, getFieldByName(bulkLoadProfile, "orderNo").getColumnIndex());
assertEquals(1, getFieldByName(bulkLoadProfile, "shipToName").getColumnIndex());
assertEquals(2, getFieldByName(bulkLoadProfile, "orderLine.sku,0").getColumnIndex());
assertEquals(3, getFieldByName(bulkLoadProfile, "orderLine.quantity,0").getColumnIndex());
assertEquals(4, getFieldByName(bulkLoadProfile, "orderLine.sku,1").getColumnIndex());
assertEquals(5, getFieldByName(bulkLoadProfile, "orderLine.quantity,1").getColumnIndex());
/////////////////////////////////////////////////////////////////
// assert that the order of fields matches the file's ordering //
/////////////////////////////////////////////////////////////////
assertEquals(List.of("orderNo", "shipToName", "orderLine.sku,0", "orderLine.quantity,0", "orderLine.sku,1", "orderLine.quantity,1"),
bulkLoadProfile.getFieldList().stream().map(f -> f.getFieldName()).toList());
}
/***************************************************************************
**
***************************************************************************/
private BulkLoadProfileField getFieldByName(BulkLoadProfile bulkLoadProfile, String fieldName)
{
return (bulkLoadProfile.getFieldList().stream()
.filter(f -> f.getFieldName().equals(fieldName))
.findFirst().orElse(null));
}
}

1
qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/FlatRowsToRecordTest.java
View File

@ -30,6 +30,7 @@ import com.kingsrook.qqq.backend.core.BaseTest;
import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.TestFileToRows; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.TestFileToRows;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkInsertMapping;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow;
import com.kingsrook.qqq.backend.core.utils.TestUtils; import com.kingsrook.qqq.backend.core.utils.TestUtils;
import com.kingsrook.qqq.backend.core.utils.collections.ListBuilder; import com.kingsrook.qqq.backend.core.utils.collections.ListBuilder;

1
qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/TallRowsToRecordTest.java
View File

@ -29,6 +29,7 @@ import com.kingsrook.qqq.backend.core.BaseTest;
import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.CsvFileToRows; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.CsvFileToRows;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkInsertMapping;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow;
import com.kingsrook.qqq.backend.core.utils.TestUtils; import com.kingsrook.qqq.backend.core.utils.TestUtils;
import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Test;

1
qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/ValueMapperTest.java
View File

@ -29,6 +29,7 @@ import com.kingsrook.qqq.backend.core.BaseTest;
import com.kingsrook.qqq.backend.core.context.QContext; import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkInsertMapping;
import com.kingsrook.qqq.backend.core.utils.CollectionUtils; import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
import com.kingsrook.qqq.backend.core.utils.TestUtils; import com.kingsrook.qqq.backend.core.utils.TestUtils;
import org.json.JSONArray; import org.json.JSONArray;

175
qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMappingTest.java Normal file
View File

@ -0,0 +1,175 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2024. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping;
import java.io.Serializable;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import com.kingsrook.qqq.backend.core.BaseTest;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.CsvFileToRows;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkInsertMapping;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow;
import com.kingsrook.qqq.backend.core.utils.TestUtils;
import com.kingsrook.qqq.backend.core.utils.collections.MapBuilder;
import org.junit.jupiter.api.Test;
import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.jupiter.api.Assertions.assertEquals;
/*******************************************************************************
** Unit test for WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMapping
*******************************************************************************/
class WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMappingTest extends BaseTest
{
/*******************************************************************************
**
*******************************************************************************/
@Test
void testOrderAndLinesWithoutDupes() throws QException
{
String csv = """
orderNo, Ship To, lastName, SKU 1, Quantity 1, SKU 2, Quantity 2, SKU 3, Quantity 3
1, Homer, Simpson, DONUT, 12, BEER, 500, COUCH, 1
2, Ned, Flanders, BIBLE, 7, LAWNMOWER, 1
""";
CsvFileToRows fileToRows = CsvFileToRows.forString(csv);
BulkLoadFileRow header = fileToRows.next();
WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMapping rowsToRecord = new WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMapping();
BulkInsertMapping mapping = new BulkInsertMapping()
.withFieldNameToHeaderNameMap(Map.of(
"orderNo", "orderNo",
"shipToName", "Ship To",
"orderLine.sku,0", "SKU 1",
"orderLine.quantity,0", "Quantity 1",
"orderLine.sku,1", "SKU 2",
"orderLine.quantity,1", "Quantity 2",
"orderLine.sku,2", "SKU 3",
"orderLine.quantity,2", "Quantity 3"
))
.withMappedAssociations(List.of("orderLine"))
.withTableName(TestUtils.TABLE_NAME_ORDER)
.withLayout(BulkInsertMapping.Layout.WIDE)
.withHasHeaderRow(true);
List<QRecord> records = rowsToRecord.nextPage(fileToRows, header, mapping, Integer.MAX_VALUE);
assertEquals(2, records.size());
QRecord order = records.get(0);
assertEquals(1, order.getValueInteger("orderNo"));
assertEquals("Homer", order.getValueString("shipToName"));
assertEquals(List.of("DONUT", "BEER", "COUCH"), getValues(order.getAssociatedRecords().get("orderLine"), "sku"));
assertEquals(List.of(12, 500, 1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity"));
order = records.get(1);
assertEquals(2, order.getValueInteger("orderNo"));
assertEquals("Ned", order.getValueString("shipToName"));
assertEquals(List.of("BIBLE", "LAWNMOWER"), getValues(order.getAssociatedRecords().get("orderLine"), "sku"));
assertEquals(List.of(7, 1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity"));
}
/*******************************************************************************
**
*******************************************************************************/
@Test
void testOrderLinesAndOrderExtrinsicWithoutDupes() throws QException
{
String csv = """
orderNo, Ship To, lastName, SKU 1, Quantity 1, SKU 2, Quantity 2, SKU 3, Quantity 3, Extrinsic Key 1, Extrinsic Value 1, Extrinsic Key 2, Extrinsic Value 2
1, Homer, Simpson, DONUT, 12, BEER, 500, COUCH, 1, Store Name, QQQ Mart, Coupon Code, 10QOff
2, Ned, Flanders, BIBLE, 7, LAWNMOWER, 1
""";
CsvFileToRows fileToRows = CsvFileToRows.forString(csv);
BulkLoadFileRow header = fileToRows.next();
WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMapping rowsToRecord = new WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMapping();
BulkInsertMapping mapping = new BulkInsertMapping()
.withFieldNameToHeaderNameMap(MapBuilder.of(() -> new HashMap<String, String>())
.with("orderNo", "orderNo")
.with("shipToName", "Ship To")
.with("orderLine.sku,0", "SKU 1")
.with("orderLine.quantity,0", "Quantity 1")
.with("orderLine.sku,1", "SKU 2")
.with("orderLine.quantity,1", "Quantity 2")
.with("orderLine.sku,2", "SKU 3")
.with("orderLine.quantity,2", "Quantity 3")
.with("extrinsics.key,0", "Extrinsic Key 1")
.with("extrinsics.value,0", "Extrinsic Value 1")
.with("extrinsics.key,1", "Extrinsic Key 2")
.with("extrinsics.value,1", "Extrinsic Value 2")
.build()
)
.withFieldNameToDefaultValueMap(Map.of(
"orderLine.lineNumber,0", "1",
"orderLine.lineNumber,1", "2",
"orderLine.lineNumber,2", "3"
))
.withMappedAssociations(List.of("orderLine", "extrinsics"))
.withTableName(TestUtils.TABLE_NAME_ORDER)
.withLayout(BulkInsertMapping.Layout.WIDE)
.withHasHeaderRow(true);
List<QRecord> records = rowsToRecord.nextPage(fileToRows, header, mapping, Integer.MAX_VALUE);
assertEquals(2, records.size());
QRecord order = records.get(0);
assertEquals(1, order.getValueInteger("orderNo"));
assertEquals("Homer", order.getValueString("shipToName"));
assertEquals(List.of("DONUT", "BEER", "COUCH"), getValues(order.getAssociatedRecords().get("orderLine"), "sku"));
assertEquals(List.of(12, 500, 1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity"));
assertEquals(List.of("1", "2", "3"), getValues(order.getAssociatedRecords().get("orderLine"), "lineNumber"));
assertEquals(List.of("Store Name", "Coupon Code"), getValues(order.getAssociatedRecords().get("extrinsics"), "key"));
assertEquals(List.of("QQQ Mart", "10QOff"), getValues(order.getAssociatedRecords().get("extrinsics"), "value"));
order = records.get(1);
assertEquals(2, order.getValueInteger("orderNo"));
assertEquals("Ned", order.getValueString("shipToName"));
assertEquals(List.of("BIBLE", "LAWNMOWER"), getValues(order.getAssociatedRecords().get("orderLine"), "sku"));
assertEquals(List.of(7, 1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity"));
assertEquals(List.of("1", "2"), getValues(order.getAssociatedRecords().get("orderLine"), "lineNumber"));
assertThat(order.getAssociatedRecords().get("extrinsics")).isNullOrEmpty();
}
/***************************************************************************
**
***************************************************************************/
private List<Serializable> getValues(List<QRecord> records, String fieldName)
{
return (records.stream().map(r -> r.getValue(fieldName)).toList());
}
}

269
qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithExplicitMappingTest.java
View File

@ -1,269 +0,0 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2024. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping;
import java.io.Serializable;
import java.util.List;
import java.util.Map;
import com.kingsrook.qqq.backend.core.BaseTest;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.CsvFileToRows;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow;
import com.kingsrook.qqq.backend.core.utils.TestUtils;
import org.junit.jupiter.api.Test;
import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.jupiter.api.Assertions.assertEquals;
/*******************************************************************************
** Unit test for WideRowsToRecord
*******************************************************************************/
class WideRowsToRecordWithExplicitMappingTest extends BaseTest
{
/*******************************************************************************
**
*******************************************************************************/
@Test
void testOrderAndLinesWithoutDupes() throws QException
{
String csv = """
orderNo, Ship To, lastName, SKU 1, Quantity 1, SKU 2, Quantity 2, SKU 3, Quantity 3
1, Homer, Simpson, DONUT, 12, BEER, 500, COUCH, 1
2, Ned, Flanders, BIBLE, 7, LAWNMOWER, 1
""";
CsvFileToRows fileToRows = CsvFileToRows.forString(csv);
BulkLoadFileRow header = fileToRows.next();
WideRowsToRecordWithExplicitMapping rowsToRecord = new WideRowsToRecordWithExplicitMapping();
BulkInsertMapping mapping = new BulkInsertMapping()
.withFieldNameToHeaderNameMap(Map.of(
"orderNo", "orderNo",
"shipToName", "Ship To"
))
.withWideLayoutMapping(Map.of(
"orderLine", new BulkInsertWideLayoutMapping(List.of(
new BulkInsertWideLayoutMapping.ChildRecordMapping(Map.of("sku", "SKU 1", "quantity", "Quantity 1")),
new BulkInsertWideLayoutMapping.ChildRecordMapping(Map.of("sku", "SKU 2", "quantity", "Quantity 2")),
new BulkInsertWideLayoutMapping.ChildRecordMapping(Map.of("sku", "SKU 3", "quantity", "Quantity 3"))
))
))
.withMappedAssociations(List.of("orderLine"))
.withTableName(TestUtils.TABLE_NAME_ORDER)
.withLayout(BulkInsertMapping.Layout.WIDE)
.withHasHeaderRow(true);
List<QRecord> records = rowsToRecord.nextPage(fileToRows, header, mapping, Integer.MAX_VALUE);
assertEquals(2, records.size());
QRecord order = records.get(0);
assertEquals(1, order.getValueInteger("orderNo"));
assertEquals("Homer", order.getValueString("shipToName"));
assertEquals(List.of("DONUT", "BEER", "COUCH"), getValues(order.getAssociatedRecords().get("orderLine"), "sku"));
assertEquals(List.of(12, 500, 1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity"));
order = records.get(1);
assertEquals(2, order.getValueInteger("orderNo"));
assertEquals("Ned", order.getValueString("shipToName"));
assertEquals(List.of("BIBLE", "LAWNMOWER"), getValues(order.getAssociatedRecords().get("orderLine"), "sku"));
assertEquals(List.of(7, 1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity"));
}
/*******************************************************************************
**
*******************************************************************************/
@Test
void testOrderLinesAndOrderExtrinsicWithoutDupes() throws QException
{
String csv = """
orderNo, Ship To, lastName, SKU 1, Quantity 1, SKU 2, Quantity 2, SKU 3, Quantity 3, Extrinsic Key 1, Extrinsic Value 1, Extrinsic Key 2, Extrinsic Value 2
1, Homer, Simpson, DONUT, 12, BEER, 500, COUCH, 1, Store Name, QQQ Mart, Coupon Code, 10QOff
2, Ned, Flanders, BIBLE, 7, LAWNMOWER, 1
""";
CsvFileToRows fileToRows = CsvFileToRows.forString(csv);
BulkLoadFileRow header = fileToRows.next();
WideRowsToRecordWithExplicitMapping rowsToRecord = new WideRowsToRecordWithExplicitMapping();
BulkInsertMapping mapping = new BulkInsertMapping()
.withFieldNameToHeaderNameMap(Map.of(
"orderNo", "orderNo",
"shipToName", "Ship To"
))
.withWideLayoutMapping(Map.of(
"orderLine", new BulkInsertWideLayoutMapping(List.of(
new BulkInsertWideLayoutMapping.ChildRecordMapping(Map.of("sku", "SKU 1", "quantity", "Quantity 1")),
new BulkInsertWideLayoutMapping.ChildRecordMapping(Map.of("sku", "SKU 2", "quantity", "Quantity 2")),
new BulkInsertWideLayoutMapping.ChildRecordMapping(Map.of("sku", "SKU 3", "quantity", "Quantity 3"))
)),
"extrinsics", new BulkInsertWideLayoutMapping(List.of(
new BulkInsertWideLayoutMapping.ChildRecordMapping(Map.of("key", "Extrinsic Key 1", "value", "Extrinsic Value 1")),
new BulkInsertWideLayoutMapping.ChildRecordMapping(Map.of("key", "Extrinsic Key 2", "value", "Extrinsic Value 2"))
))
))
.withMappedAssociations(List.of("orderLine", "extrinsics"))
.withTableName(TestUtils.TABLE_NAME_ORDER)
.withLayout(BulkInsertMapping.Layout.WIDE)
.withHasHeaderRow(true);
List<QRecord> records = rowsToRecord.nextPage(fileToRows, header, mapping, Integer.MAX_VALUE);
assertEquals(2, records.size());
QRecord order = records.get(0);
assertEquals(1, order.getValueInteger("orderNo"));
assertEquals("Homer", order.getValueString("shipToName"));
assertEquals(List.of("DONUT", "BEER", "COUCH"), getValues(order.getAssociatedRecords().get("orderLine"), "sku"));
assertEquals(List.of(12, 500, 1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity"));
assertEquals(List.of("Store Name", "Coupon Code"), getValues(order.getAssociatedRecords().get("extrinsics"), "key"));
assertEquals(List.of("QQQ Mart", "10QOff"), getValues(order.getAssociatedRecords().get("extrinsics"), "value"));
order = records.get(1);
assertEquals(2, order.getValueInteger("orderNo"));
assertEquals("Ned", order.getValueString("shipToName"));
assertEquals(List.of("BIBLE", "LAWNMOWER"), getValues(order.getAssociatedRecords().get("orderLine"), "sku"));
assertEquals(List.of(7, 1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity"));
assertThat(order.getAssociatedRecords().get("extrinsics")).isNullOrEmpty();
}
/*******************************************************************************
**
*******************************************************************************/
@Test
void testOrderLinesWithLineExtrinsicsAndOrderExtrinsicWithoutDupes() throws QException
{
String csv = """
orderNo, Ship To, lastName, Extrinsic Key 1, Extrinsic Value 1, Extrinsic Key 2, Extrinsic Value 2, SKU 1, Quantity 1, Line Extrinsic Key 1.1, Line Extrinsic Value 1.1, Line Extrinsic Key 1.2, Line Extrinsic Value 1.2, SKU 2, Quantity 2, Line Extrinsic Key 2.1, Line Extrinsic Value 2.1, SKU 3, Quantity 3, Line Extrinsic Key 3.1, Line Extrinsic Value 3.1, Line Extrinsic Key 3.2
1, Homer, Simpson, Store Name, QQQ Mart, Coupon Code, 10QOff, DONUT, 12, Flavor, Chocolate, Size, Large, BEER, 500, Flavor, Hops, COUCH, 1, Color, Brown, foo,
2, Ned, Flanders, , , , , BIBLE, 7, Flavor, King James, Size, X-Large, LAWNMOWER, 1
""";
Integer defaultStoreId = 42;
String defaultLineNo = "47";
String defaultLineExtraValue = "bar";
CsvFileToRows fileToRows = CsvFileToRows.forString(csv);
BulkLoadFileRow header = fileToRows.next();
WideRowsToRecordWithExplicitMapping rowsToRecord = new WideRowsToRecordWithExplicitMapping();
BulkInsertMapping mapping = new BulkInsertMapping()
.withFieldNameToHeaderNameMap(Map.of(
"orderNo", "orderNo",
"shipToName", "Ship To"
))
.withMappedAssociations(List.of("orderLine", "extrinsics", "orderLine.extrinsics"))
.withWideLayoutMapping(Map.of(
"orderLine", new BulkInsertWideLayoutMapping(List.of(
new BulkInsertWideLayoutMapping.ChildRecordMapping(
Map.of("sku", "SKU 1", "quantity", "Quantity 1"),
Map.of("extrinsics", new BulkInsertWideLayoutMapping(List.of(
new BulkInsertWideLayoutMapping.ChildRecordMapping(Map.of("key", "Line Extrinsic Key 1.1", "value", "Line Extrinsic Value 1.1")),
new BulkInsertWideLayoutMapping.ChildRecordMapping(Map.of("key", "Line Extrinsic Key 1.2", "value", "Line Extrinsic Value 1.2"))
)))),
new BulkInsertWideLayoutMapping.ChildRecordMapping(
Map.of("sku", "SKU 2", "quantity", "Quantity 2"),
Map.of("extrinsics", new BulkInsertWideLayoutMapping(List.of(
new BulkInsertWideLayoutMapping.ChildRecordMapping(Map.of("key", "Line Extrinsic Key 2.1", "value", "Line Extrinsic Value 2.1")),
new BulkInsertWideLayoutMapping.ChildRecordMapping(Map.of("key", "Line Extrinsic Key 2.2", "value", "Line Extrinsic Value 2.2"))
)))),
new BulkInsertWideLayoutMapping.ChildRecordMapping(
Map.of("sku", "SKU 3", "quantity", "Quantity 3"),
Map.of("extrinsics", new BulkInsertWideLayoutMapping(List.of(
new BulkInsertWideLayoutMapping.ChildRecordMapping(Map.of("key", "Line Extrinsic Key 3.1", "value", "Line Extrinsic Value 3.1")),
new BulkInsertWideLayoutMapping.ChildRecordMapping(Map.of("key", "Line Extrinsic Key 3.2", "value", "Line Extrinsic Value 3.2"))
))))
)),
"extrinsics", new BulkInsertWideLayoutMapping(List.of(
new BulkInsertWideLayoutMapping.ChildRecordMapping(Map.of("key", "Extrinsic Key 1", "value", "Extrinsic Value 1")),
new BulkInsertWideLayoutMapping.ChildRecordMapping(Map.of("key", "Extrinsic Key 2", "value", "Extrinsic Value 2"))
))
))
.withFieldNameToValueMapping(Map.of("orderLine.extrinsics.value", Map.of("Large", "L", "X-Large", "XL")))
.withFieldNameToDefaultValueMap(Map.of(
"storeId", defaultStoreId,
"orderLine.lineNumber", defaultLineNo,
"orderLine.extrinsics.value", defaultLineExtraValue
))
.withTableName(TestUtils.TABLE_NAME_ORDER)
.withLayout(BulkInsertMapping.Layout.WIDE)
.withHasHeaderRow(true);
List<QRecord> records = rowsToRecord.nextPage(fileToRows, header, mapping, Integer.MAX_VALUE);
assertEquals(2, records.size());
QRecord order = records.get(0);
assertEquals(1, order.getValueInteger("orderNo"));
assertEquals("Homer", order.getValueString("shipToName"));
assertEquals(defaultStoreId, order.getValue("storeId"));
assertEquals(List.of("DONUT", "BEER", "COUCH"), getValues(order.getAssociatedRecords().get("orderLine"), "sku"));
assertEquals(List.of(12, 500, 1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity"));
assertEquals(List.of(defaultLineNo, defaultLineNo, defaultLineNo), getValues(order.getAssociatedRecords().get("orderLine"), "lineNumber"));
assertEquals(List.of("Store Name", "Coupon Code"), getValues(order.getAssociatedRecords().get("extrinsics"), "key"));
assertEquals(List.of("QQQ Mart", "10QOff"), getValues(order.getAssociatedRecords().get("extrinsics"), "value"));
QRecord lineItem = order.getAssociatedRecords().get("orderLine").get(0);
assertEquals(List.of("Flavor", "Size"), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "key"));
assertEquals(List.of("Chocolate", "L"), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "value"));
lineItem = order.getAssociatedRecords().get("orderLine").get(1);
assertEquals(List.of("Flavor"), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "key"));
assertEquals(List.of("Hops"), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "value"));
lineItem = order.getAssociatedRecords().get("orderLine").get(2);
assertEquals(List.of("Color", "foo"), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "key"));
assertEquals(List.of("Brown", defaultLineExtraValue), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "value"));
order = records.get(1);
assertEquals(2, order.getValueInteger("orderNo"));
assertEquals("Ned", order.getValueString("shipToName"));
assertEquals(defaultStoreId, order.getValue("storeId"));
assertEquals(List.of("BIBLE", "LAWNMOWER"), getValues(order.getAssociatedRecords().get("orderLine"), "sku"));
assertEquals(List.of(7, 1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity"));
assertEquals(List.of(defaultLineNo, defaultLineNo), getValues(order.getAssociatedRecords().get("orderLine"), "lineNumber"));
assertThat(order.getAssociatedRecords().get("extrinsics")).isNullOrEmpty();
lineItem = order.getAssociatedRecords().get("orderLine").get(0);
assertEquals(List.of("Flavor", "Size"), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "key"));
assertEquals(List.of("King James", "XL"), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "value"));
}
/***************************************************************************
**
***************************************************************************/
private List<Serializable> getValues(List<QRecord> records, String fieldName)
{
return (records.stream().map(r -> r.getValue(fieldName)).toList());
}
}

1
qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithSpreadMappingTest.java
View File

@ -29,6 +29,7 @@ import com.kingsrook.qqq.backend.core.BaseTest;
import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.CsvFileToRows; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.CsvFileToRows;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkInsertMapping;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow;
import com.kingsrook.qqq.backend.core.utils.TestUtils; import com.kingsrook.qqq.backend.core.utils.TestUtils;
import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Test;