diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java index ee307632..9a48204a 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java @@ -73,7 +73,6 @@ import com.kingsrook.qqq.backend.core.processes.implementations.bulk.delete.Bulk import com.kingsrook.qqq.backend.core.processes.implementations.bulk.delete.BulkDeleteTransformStep; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.edit.BulkEditLoadStep; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.edit.BulkEditTransformStep; -import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkInsertExtractStep; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkInsertLoadStep; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkInsertPrepareFileMappingStep; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkInsertPrepareValueMappingStep; @@ -81,7 +80,6 @@ import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.Bulk import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkInsertReceiveValueMappingStep; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkInsertTransformStep; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkInsertV2ExtractStep; -import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping.BulkInsertMapping; import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.ExtractViaQueryStep; import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess; import com.kingsrook.qqq.backend.core.scheduler.QScheduleManager; @@ -815,75 +813,11 @@ public class QInstanceEnricher /******************************************************************************* ** *******************************************************************************/ - private void defineTableBulkInsert(QInstance qInstance, QTableMetaData table, String processName) + public void defineTableBulkInsert(QInstance qInstance, QTableMetaData table, String processName) { Map values = new HashMap<>(); values.put(StreamedETLWithFrontendProcess.FIELD_DESTINATION_TABLE, table.getName()); - QProcessMetaData process = StreamedETLWithFrontendProcess.defineProcessMetaData( - BulkInsertExtractStep.class, - BulkInsertTransformStep.class, - BulkInsertLoadStep.class, - values - ) - .withName(processName) - .withLabel(table.getLabel() + " Bulk Insert") - .withTableName(table.getName()) - .withIsHidden(true) - .withPermissionRules(qInstance.getDefaultPermissionRules().clone() - .withCustomPermissionChecker(new QCodeReference(BulkTableActionProcessPermissionChecker.class))); - - List editableFields = new ArrayList<>(); - for(QFieldSection section : CollectionUtils.nonNullList(table.getSections())) - { - for(String fieldName : CollectionUtils.nonNullList(section.getFieldNames())) - { - try - { - QFieldMetaData field = table.getField(fieldName); - if(field.getIsEditable() && !field.getType().equals(QFieldType.BLOB)) - { - editableFields.add(field); - } - } - catch(Exception e) - { - // shrug? - } - } - } - - String fieldsForHelpText = editableFields.stream() - .map(QFieldMetaData::getLabel) - .collect(Collectors.joining(", ")); - - QFrontendStepMetaData uploadScreen = new QFrontendStepMetaData() - .withName("upload") - .withLabel("Upload File") - .withFormField(new QFieldMetaData("theFile", QFieldType.BLOB).withLabel(table.getLabel() + " File").withIsRequired(true)) - .withComponent(new QFrontendComponentMetaData() - .withType(QComponentType.HELP_TEXT) - .withValue("previewText", "file upload instructions") - .withValue("text", "Upload a CSV file with the following columns:\n" + fieldsForHelpText)) - .withComponent(new QFrontendComponentMetaData().withType(QComponentType.EDIT_FORM)); - - process.addStep(0, uploadScreen); - process.getFrontendStep("review").setRecordListFields(editableFields); - qInstance.addProcess(process); - } - - - - /******************************************************************************* - ** - *******************************************************************************/ - public void defineTableBulkInsertV2(QInstance qInstance, QTableMetaData table, String processName) - { - qInstance.addPossibleValueSource(QPossibleValueSource.newForEnum("bulkInsertFileLayout", BulkInsertMapping.Layout.values())); - - Map values = new HashMap<>(); - values.put(StreamedETLWithFrontendProcess.FIELD_DESTINATION_TABLE, table.getName()); - QProcessMetaData process = StreamedETLWithFrontendProcess.defineProcessMetaData( BulkInsertV2ExtractStep.class, BulkInsertTransformStep.class, diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertPrepareFileMappingStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertPrepareFileMappingStep.java index f7ee32ce..f939df3a 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertPrepareFileMappingStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertPrepareFileMappingStep.java @@ -25,27 +25,19 @@ package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert; import java.io.File; import java.io.InputStream; import java.util.ArrayList; -import java.util.Comparator; -import java.util.HashSet; -import java.util.Set; +import java.util.List; import com.kingsrook.qqq.backend.core.actions.processes.BackendStep; import com.kingsrook.qqq.backend.core.actions.tables.StorageAction; -import com.kingsrook.qqq.backend.core.context.QContext; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; import com.kingsrook.qqq.backend.core.model.actions.tables.storage.StorageInput; -import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; -import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinOn; -import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinType; -import com.kingsrook.qqq.backend.core.model.metadata.joins.QJoinMetaData; -import com.kingsrook.qqq.backend.core.model.metadata.tables.Association; -import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.FileToRowsInterface; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping.BulkLoadMappingSuggester; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping.BulkLoadTableStructureBuilder; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadProfile; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadTableStructure; -import com.kingsrook.qqq.backend.core.utils.CollectionUtils; -import com.kingsrook.qqq.backend.core.utils.StringUtils; import com.kingsrook.qqq.backend.core.utils.ValueUtils; @@ -62,7 +54,26 @@ public class BulkInsertPrepareFileMappingStep implements BackendStep public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException { buildFileDetailsForMappingStep(runBackendStepInput, runBackendStepOutput); - buildFieldsForMappingStep(runBackendStepInput, runBackendStepOutput); + + String tableName = runBackendStepInput.getValueString("tableName"); + BulkLoadTableStructure tableStructure = BulkLoadTableStructureBuilder.buildTableStructure(tableName); + runBackendStepOutput.addValue("tableStructure", tableStructure); + + @SuppressWarnings("unchecked") + List headerValues = (List) runBackendStepOutput.getValue("headerValues"); + buildSuggestedMapping(headerValues, tableStructure, runBackendStepOutput); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private void buildSuggestedMapping(List headerValues, BulkLoadTableStructure tableStructure, RunBackendStepOutput runBackendStepOutput) + { + BulkLoadMappingSuggester bulkLoadMappingSuggester = new BulkLoadMappingSuggester(); + BulkLoadProfile bulkLoadProfile = bulkLoadMappingSuggester.suggestBulkLoadMappingProfile(tableStructure, headerValues); + runBackendStepOutput.addValue("bulkLoadProfile", bulkLoadProfile); } @@ -121,6 +132,7 @@ public class BulkInsertPrepareFileMappingStep implements BackendStep } } runBackendStepOutput.addValue("bodyValuesPreview", bodyValues); + } catch(Exception e) { @@ -147,94 +159,4 @@ public class BulkInsertPrepareFileMappingStep implements BackendStep return (rs.toString()); } - - - /*************************************************************************** - ** - ***************************************************************************/ - private static void buildFieldsForMappingStep(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) - { - String tableName = runBackendStepInput.getValueString("tableName"); - BulkLoadTableStructure tableStructure = buildTableStructure(tableName, null, null); - runBackendStepOutput.addValue("tableStructure", tableStructure); - } - - - - /*************************************************************************** - ** - ***************************************************************************/ - private static BulkLoadTableStructure buildTableStructure(String tableName, Association association, String parentAssociationPath) - { - QTableMetaData table = QContext.getQInstance().getTable(tableName); - - BulkLoadTableStructure tableStructure = new BulkLoadTableStructure(); - tableStructure.setTableName(tableName); - tableStructure.setLabel(table.getLabel()); - - Set associationJoinFieldNamesToExclude = new HashSet<>(); - - if(association == null) - { - tableStructure.setIsMain(true); - tableStructure.setIsMany(false); - tableStructure.setAssociationPath(null); - } - else - { - tableStructure.setIsMain(false); - - QJoinMetaData join = QContext.getQInstance().getJoin(association.getJoinName()); - if(join.getType().equals(JoinType.ONE_TO_MANY) || join.getType().equals(JoinType.MANY_TO_ONE)) - { - tableStructure.setIsMany(true); - } - - for(JoinOn joinOn : join.getJoinOns()) - { - //////////////////////////////////////////////////////////////////////////////////////////////// - // don't allow the user to map the "join field" from a child up to its parent // - // (e.g., you can't map lineItem.orderId -- that'll happen automatically via the association) // - //////////////////////////////////////////////////////////////////////////////////////////////// - if(join.getLeftTable().equals(tableName)) - { - associationJoinFieldNamesToExclude.add(joinOn.getLeftField()); - } - else if(join.getRightTable().equals(tableName)) - { - associationJoinFieldNamesToExclude.add(joinOn.getRightField()); - } - } - - if(!StringUtils.hasContent(parentAssociationPath)) - { - tableStructure.setAssociationPath(association.getName()); - } - else - { - tableStructure.setAssociationPath(parentAssociationPath + "." + association.getName()); - } - } - - ArrayList fields = new ArrayList<>(); - tableStructure.setFields(fields); - for(QFieldMetaData field : table.getFields().values()) - { - if(field.getIsEditable() && !associationJoinFieldNamesToExclude.contains(field.getName())) - { - fields.add(field); - } - } - - fields.sort(Comparator.comparing(f -> f.getLabel())); - - for(Association childAssociation : CollectionUtils.nonNullList(table.getAssociations())) - { - BulkLoadTableStructure associatedStructure = buildTableStructure(childAssociation.getAssociatedTableName(), childAssociation, parentAssociationPath); - tableStructure.addAssociation(associatedStructure); - } - - return (tableStructure); - } - } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertPrepareValueMappingStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertPrepareValueMappingStep.java index 85397442..19e696e5 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertPrepareValueMappingStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertPrepareValueMappingStep.java @@ -51,7 +51,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleVal import com.kingsrook.qqq.backend.core.model.metadata.tables.Association; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.FileToRowsInterface; -import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping.BulkInsertMapping; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkInsertMapping; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow; import com.kingsrook.qqq.backend.core.utils.CollectionUtils; import com.kingsrook.qqq.backend.core.utils.StringUtils; diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertReceiveFileMappingStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertReceiveFileMappingStep.java index 8f72bb82..bc6ae239 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertReceiveFileMappingStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertReceiveFileMappingStep.java @@ -37,7 +37,7 @@ import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInpu import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; import com.kingsrook.qqq.backend.core.model.actions.tables.storage.StorageInput; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.FileToRowsInterface; -import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping.BulkInsertMapping; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkInsertMapping; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadProfile; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadProfileField; @@ -102,7 +102,12 @@ public class BulkInsertReceiveFileMappingStep implements BackendStep BulkLoadFileRow headerRow = fileToRowsInterface.next(); for(BulkLoadProfileField bulkLoadProfileField : bulkLoadProfile.getFieldList()) { - if(bulkLoadProfileField.getColumnIndex() != null) + if(bulkLoadProfileField.getHeaderName() != null) + { + String headerName = bulkLoadProfileField.getHeaderName(); + fieldNameToHeaderNameMap.put(bulkLoadProfileField.getFieldName(), headerName); + } + else if(bulkLoadProfileField.getColumnIndex() != null) { String headerName = ValueUtils.getValueAsString(headerRow.getValueElseNull(bulkLoadProfileField.getColumnIndex())); fieldNameToHeaderNameMap.put(bulkLoadProfileField.getFieldName(), headerName); @@ -164,7 +169,11 @@ public class BulkInsertReceiveFileMappingStep implements BackendStep { if(bulkLoadProfileField.getFieldName().contains(".")) { - associationNameSet.add(bulkLoadProfileField.getFieldName().substring(0, bulkLoadProfileField.getFieldName().lastIndexOf('.'))); + ////////////////////////////////////////////////////////////////////////////////////////////////////////// + // handle parent.child.grandchild.fieldName,index.index.index if we do sub-indexes for grandchildren... // + ////////////////////////////////////////////////////////////////////////////////////////////////////////// + String fieldNameBeforeIndex = bulkLoadProfileField.getFieldName().split(",")[0]; + associationNameSet.add(fieldNameBeforeIndex.substring(0, fieldNameBeforeIndex.lastIndexOf('.'))); } } bulkInsertMapping.setMappedAssociations(new ArrayList<>(associationNameSet)); diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertReceiveValueMappingStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertReceiveValueMappingStep.java index f9c17667..e1b5bc9a 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertReceiveValueMappingStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertReceiveValueMappingStep.java @@ -32,7 +32,7 @@ import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.logging.QLogger; import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; -import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping.BulkInsertMapping; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkInsertMapping; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadProfile; import com.kingsrook.qqq.backend.core.utils.JsonUtils; diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertStepUtils.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertStepUtils.java index 7c7f008b..b0db3ad0 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertStepUtils.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertStepUtils.java @@ -109,6 +109,7 @@ public class BulkInsertStepUtils BulkLoadProfileField bulkLoadProfileField = new BulkLoadProfileField(); fieldList.add(bulkLoadProfileField); bulkLoadProfileField.setFieldName(jsonObject.optString("fieldName")); + bulkLoadProfileField.setHeaderName(jsonObject.has("headerName") ? jsonObject.getString("headerName") : null); bulkLoadProfileField.setColumnIndex(jsonObject.has("columnIndex") ? jsonObject.getInt("columnIndex") : null); bulkLoadProfileField.setDefaultValue((Serializable) jsonObject.opt("defaultValue")); bulkLoadProfileField.setDoValueMapping(jsonObject.optBoolean("doValueMapping")); diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTransformStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTransformStep.java index 6e28c10a..09d9ca8d 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTransformStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTransformStep.java @@ -48,6 +48,7 @@ import com.kingsrook.qqq.backend.core.model.actions.processes.Status; import com.kingsrook.qqq.backend.core.model.actions.tables.QInputSource; import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.tables.Association; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.UniqueKey; import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.AbstractTransformStep; @@ -67,7 +68,8 @@ public class BulkInsertTransformStep extends AbstractTransformStep private ProcessSummaryWarningsAndErrorsRollup processSummaryWarningsAndErrorsRollup = ProcessSummaryWarningsAndErrorsRollup.build("inserted"); - private Map ukErrorSummaries = new HashMap<>(); + private Map ukErrorSummaries = new HashMap<>(); + private Map associationsToInsertSummaries = new HashMap<>(); private QTableMetaData table; @@ -259,6 +261,13 @@ public class BulkInsertTransformStep extends AbstractTransformStep { okSummary.incrementCountAndAddPrimaryKey(null); outputRecords.add(record); + + for(Map.Entry> entry : CollectionUtils.nonNullMap(record.getAssociatedRecords()).entrySet()) + { + String associationName = entry.getKey(); + ProcessSummaryLine associationToInsertLine = associationsToInsertSummaries.computeIfAbsent(associationName, x -> new ProcessSummaryLine(Status.OK)); + associationToInsertLine.incrementCount(CollectionUtils.nonNullList(entry.getValue()).size()); + } } } @@ -366,6 +375,24 @@ public class BulkInsertTransformStep extends AbstractTransformStep okSummary.pickMessage(isForResultScreen); okSummary.addSelfToListIfAnyCount(rs); + for(Map.Entry entry : associationsToInsertSummaries.entrySet()) + { + Optional association = table.getAssociations().stream().filter(a -> a.getName().equals(entry.getKey())).findFirst(); + if(association.isPresent()) + { + QTableMetaData associationTable = QContext.getQInstance().getTable(association.get().getAssociatedTableName()); + String associationLabel = associationTable.getLabel(); + + ProcessSummaryLine line = entry.getValue(); + line.setSingularFutureMessage(associationLabel + " record will be inserted."); + line.setPluralFutureMessage(associationLabel + " records will be inserted."); + line.setSingularPastMessage(associationLabel + " record was inserted."); + line.setPluralPastMessage(associationLabel + " records were inserted."); + line.pickMessage(isForResultScreen); + line.addSelfToListIfAnyCount(rs); + } + } + for(Map.Entry entry : ukErrorSummaries.entrySet()) { UniqueKey uniqueKey = entry.getKey(); diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertV2ExtractStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertV2ExtractStep.java index efacca0f..2da27e7d 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertV2ExtractStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertV2ExtractStep.java @@ -32,8 +32,8 @@ import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutp import com.kingsrook.qqq.backend.core.model.actions.tables.storage.StorageInput; import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.FileToRowsInterface; -import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping.BulkInsertMapping; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping.RowsToRecordInterface; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkInsertMapping; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow; import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.AbstractExtractStep; diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/XlsxFileToRows.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/XlsxFileToRows.java index a6e336de..f5cecd9c 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/XlsxFileToRows.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/XlsxFileToRows.java @@ -25,9 +25,14 @@ package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.fil import java.io.IOException; import java.io.InputStream; import java.io.Serializable; +import java.math.BigDecimal; +import java.math.MathContext; +import java.time.LocalDateTime; +import java.util.Optional; import java.util.stream.Stream; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow; +import org.dhatim.fastexcel.reader.Cell; import org.dhatim.fastexcel.reader.ReadableWorkbook; import org.dhatim.fastexcel.reader.Sheet; @@ -74,7 +79,41 @@ public class XlsxFileToRows extends AbstractIteratorBasedFileToRows + { + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // ... with fastexcel reader, we don't get styles... so, we just know type = number, for dates and ints & decimals... // + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + Optional dateTime = readerRow.getCellAsDate(i); + if(dateTime.isPresent() && dateTime.get().getYear() > 1915 && dateTime.get().getYear() < 2100) + { + yield dateTime.get(); + } + + Optional optionalBigDecimal = readerRow.getCellAsNumber(i); + if(optionalBigDecimal.isPresent()) + { + BigDecimal bigDecimal = optionalBigDecimal.get(); + if(bigDecimal.subtract(bigDecimal.round(new MathContext(0))).compareTo(BigDecimal.ZERO) == 0) + { + yield bigDecimal.intValue(); + } + + yield bigDecimal; + } + + yield (null); + } + case BOOLEAN -> readerRow.getCellAsBoolean(i).orElse(null); + case STRING, FORMULA -> cell.getText(); + case EMPTY, ERROR -> null; + }; + } } return new BulkLoadFileRow(values); diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkInsertWideLayoutMapping.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkInsertWideLayoutMapping.java deleted file mode 100644 index ad0cce0f..00000000 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkInsertWideLayoutMapping.java +++ /dev/null @@ -1,216 +0,0 @@ -/* - * QQQ - Low-code Application Framework for Engineers. - * Copyright (C) 2021-2024. Kingsrook, LLC - * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States - * contact@kingsrook.com - * https://github.com/Kingsrook/ - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping; - - -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow; -import com.kingsrook.qqq.backend.core.utils.ValueUtils; - - -/******************************************************************************* - ** - *******************************************************************************/ -public class BulkInsertWideLayoutMapping -{ - private List childRecordMappings; - - - - /******************************************************************************* - ** Constructor - ** - *******************************************************************************/ - public BulkInsertWideLayoutMapping(List childRecordMappings) - { - this.childRecordMappings = childRecordMappings; - } - - - - /*************************************************************************** - ** - ***************************************************************************/ - public static class ChildRecordMapping - { - Map fieldNameToHeaderNameMaps; - Map associationNameToChildRecordMappingMap; - - - - /******************************************************************************* - ** Constructor - ** - *******************************************************************************/ - public ChildRecordMapping(Map fieldNameToHeaderNameMaps) - { - this.fieldNameToHeaderNameMaps = fieldNameToHeaderNameMaps; - } - - - - /******************************************************************************* - ** Constructor - ** - *******************************************************************************/ - public ChildRecordMapping(Map fieldNameToHeaderNameMaps, Map associationNameToChildRecordMappingMap) - { - this.fieldNameToHeaderNameMaps = fieldNameToHeaderNameMaps; - this.associationNameToChildRecordMappingMap = associationNameToChildRecordMappingMap; - } - - - - /******************************************************************************* - ** Getter for fieldNameToHeaderNameMaps - *******************************************************************************/ - public Map getFieldNameToHeaderNameMaps() - { - return (this.fieldNameToHeaderNameMaps); - } - - - - /******************************************************************************* - ** Setter for fieldNameToHeaderNameMaps - *******************************************************************************/ - public void setFieldNameToHeaderNameMaps(Map fieldNameToHeaderNameMaps) - { - this.fieldNameToHeaderNameMaps = fieldNameToHeaderNameMaps; - } - - - - /******************************************************************************* - ** Fluent setter for fieldNameToHeaderNameMaps - *******************************************************************************/ - public ChildRecordMapping withFieldNameToHeaderNameMaps(Map fieldNameToHeaderNameMaps) - { - this.fieldNameToHeaderNameMaps = fieldNameToHeaderNameMaps; - return (this); - } - - - - /******************************************************************************* - ** Getter for associationNameToChildRecordMappingMap - *******************************************************************************/ - public Map getAssociationNameToChildRecordMappingMap() - { - return (this.associationNameToChildRecordMappingMap); - } - - - - /******************************************************************************* - ** Setter for associationNameToChildRecordMappingMap - *******************************************************************************/ - public void setAssociationNameToChildRecordMappingMap(Map associationNameToChildRecordMappingMap) - { - this.associationNameToChildRecordMappingMap = associationNameToChildRecordMappingMap; - } - - - - /******************************************************************************* - ** Fluent setter for associationNameToChildRecordMappingMap - *******************************************************************************/ - public ChildRecordMapping withAssociationNameToChildRecordMappingMap(Map associationNameToChildRecordMappingMap) - { - this.associationNameToChildRecordMappingMap = associationNameToChildRecordMappingMap; - return (this); - } - - - - /*************************************************************************** - ** - ***************************************************************************/ - public Map getFieldIndexes(BulkLoadFileRow headerRow) - { - // todo memoize or otherwise don't recompute - Map rs = new HashMap<>(); - - //////////////////////////////////////////////////////// - // for the current file, map header values to indexes // - //////////////////////////////////////////////////////// - Map headerToIndexMap = new HashMap<>(); - for(int i = 0; i < headerRow.size(); i++) - { - String headerValue = ValueUtils.getValueAsString(headerRow.getValue(i)); - headerToIndexMap.put(headerValue, i); - } - - ///////////////////////////////////////////////////////////////////////////////////////////////////////// - // loop over fields - finding what header name they are mapped to - then what index that header is at. // - ///////////////////////////////////////////////////////////////////////////////////////////////////////// - for(Map.Entry entry : fieldNameToHeaderNameMaps.entrySet()) - { - String headerName = entry.getValue(); - if(headerName != null) - { - Integer headerIndex = headerToIndexMap.get(headerName); - if(headerIndex != null) - { - rs.put(entry.getKey(), headerIndex); - } - } - } - - return (rs); - } - } - - - - /******************************************************************************* - ** Getter for childRecordMappings - *******************************************************************************/ - public List getChildRecordMappings() - { - return (this.childRecordMappings); - } - - - - /******************************************************************************* - ** Setter for childRecordMappings - *******************************************************************************/ - public void setChildRecordMappings(List childRecordMappings) - { - this.childRecordMappings = childRecordMappings; - } - - - - /******************************************************************************* - ** Fluent setter for childRecordMappings - *******************************************************************************/ - public BulkInsertWideLayoutMapping withChildRecordMappings(List childRecordMappings) - { - this.childRecordMappings = childRecordMappings; - return (this); - } - -} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadMappingSuggester.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadMappingSuggester.java new file mode 100644 index 00000000..71f0edcc --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadMappingSuggester.java @@ -0,0 +1,232 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping; + + +import java.util.ArrayList; +import java.util.Comparator; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadProfile; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadProfileField; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadTableStructure; +import com.kingsrook.qqq.backend.core.utils.CollectionUtils; +import com.kingsrook.qqq.backend.core.utils.StringUtils; +import com.kingsrook.qqq.backend.core.utils.collections.ListBuilder; + + +/******************************************************************************* + ** Given a bulk-upload, create a suggested mapping + *******************************************************************************/ +public class BulkLoadMappingSuggester +{ + private Map massagedHeadersWithoutNumbersToIndexMap; + private Map massagedHeadersWithNumbersToIndexMap; + + private String layout = "FLAT"; + + + + /*************************************************************************** + ** + ***************************************************************************/ + public BulkLoadProfile suggestBulkLoadMappingProfile(BulkLoadTableStructure tableStructure, List headerRow) + { + massagedHeadersWithoutNumbersToIndexMap = new LinkedHashMap<>(); + for(int i = 0; i < headerRow.size(); i++) + { + String headerValue = massageHeader(headerRow.get(i), true); + + if(!massagedHeadersWithoutNumbersToIndexMap.containsKey(headerValue)) + { + massagedHeadersWithoutNumbersToIndexMap.put(headerValue, i); + } + } + + massagedHeadersWithNumbersToIndexMap = new LinkedHashMap<>(); + for(int i = 0; i < headerRow.size(); i++) + { + String headerValue = massageHeader(headerRow.get(i), false); + + if(!massagedHeadersWithNumbersToIndexMap.containsKey(headerValue)) + { + massagedHeadersWithNumbersToIndexMap.put(headerValue, i); + } + } + + ArrayList fieldList = new ArrayList<>(); + processTable(tableStructure, fieldList, headerRow); + + ///////////////////////////////////////////////// + // sort the fields to match the column indexes // + ///////////////////////////////////////////////// + fieldList.sort(Comparator.comparing(blpf -> blpf.getColumnIndex())); + + BulkLoadProfile bulkLoadProfile = new BulkLoadProfile() + .withVersion("v1") + .withLayout(layout) + .withHasHeaderRow(true) + .withFieldList(fieldList); + + return (bulkLoadProfile); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private void processTable(BulkLoadTableStructure tableStructure, ArrayList fieldList, List headerRow) + { + Map rs = new HashMap<>(); + for(QFieldMetaData field : tableStructure.getFields()) + { + String fieldName = massageHeader(field.getName(), false); + String fieldLabel = massageHeader(field.getLabel(), false); + String tablePlusFieldLabel = massageHeader(QContext.getQInstance().getTable(tableStructure.getTableName()).getLabel() + ": " + field.getLabel(), false); + String fullFieldName = (StringUtils.hasContent(tableStructure.getAssociationPath()) ? (tableStructure.getAssociationPath() + ".") : "") + field.getName(); + + //////////////////////////////////////////////////////////////////////////////////// + // consider, if this is a many-table, if there are many matches, for wide mode... // + //////////////////////////////////////////////////////////////////////////////////// + if(tableStructure.getIsMany()) + { + List matchingIndexes = new ArrayList<>(); + + for(Map.Entry entry : massagedHeadersWithNumbersToIndexMap.entrySet()) + { + String header = entry.getKey(); + if(header.matches(fieldName + "\\d*$") || header.matches(fieldLabel + "\\d*$")) + { + matchingIndexes.add(entry.getValue()); + } + } + + if(CollectionUtils.nullSafeHasContents(matchingIndexes)) + { + /////////////////////////////////////////////////////////////////////////////////////////////////////// + // if we found more than 1 match - consider this a likely wide file, and build fields as wide-fields // + // else, if only 1, allow us to go down into the TALL block below // + // note - should we do a merger at the end, in case we found some wide, some tall? // + /////////////////////////////////////////////////////////////////////////////////////////////////////// + if(matchingIndexes.size() > 1) + { + layout = "WIDE"; + + int i = 0; + for(Integer index : matchingIndexes) + { + fieldList.add(new BulkLoadProfileField() + .withFieldName(fullFieldName + "," + i) + .withHeaderName(headerRow.get(index)) + .withColumnIndex(index) + ); + + i++; + } + + continue; + } + } + } + + //////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // else - look for matches, first w/ headers with numbers, then headers w/o numbers checking labels and names // + //////////////////////////////////////////////////////////////////////////////////////////////////////////////// + Integer index = null; + + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // for each of these potential identities of the field: // + // 1) its label, massaged // + // 2) its name, massaged // + // 3) its label, massaged, with numbers stripped away // + // 4) its name, massaged, with numbers stripped away // + // check if that identity is in the massagedHeadersWithNumbersToIndexMap, or the massagedHeadersWithoutNumbersToIndexMap. // + // this is currently successful in the both versions of the address 1 / address 2 <=> address / address 2 use-case // + // that is, BulkLoadMappingSuggesterTest.testChallengingAddress1And2 // + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + for(String fieldIdentity : ListBuilder.of(fieldLabel, fieldName, tablePlusFieldLabel, massageHeader(fieldLabel, true), massageHeader(fieldName, true))) + { + if(massagedHeadersWithNumbersToIndexMap.containsKey(fieldIdentity)) + { + index = massagedHeadersWithNumbersToIndexMap.get(fieldIdentity); + } + else if(massagedHeadersWithoutNumbersToIndexMap.containsKey(fieldIdentity)) + { + index = massagedHeadersWithoutNumbersToIndexMap.get(fieldIdentity); + } + + if(index != null) + { + break; + } + } + + if(index != null) + { + fieldList.add(new BulkLoadProfileField() + .withFieldName(fullFieldName) + .withHeaderName(headerRow.get(index)) + .withColumnIndex(index) + ); + + if(tableStructure.getIsMany() && layout.equals("FLAT")) + { + ////////////////////////////////////////////////////////////////////////////////////////// + // the first time we find an is-many child, if we were still marked as flat, go to tall // + ////////////////////////////////////////////////////////////////////////////////////////// + layout = "TALL"; + } + } + } + + //////////////////////////////////////////// + // recursively process child associations // + //////////////////////////////////////////// + for(BulkLoadTableStructure associationTableStructure : CollectionUtils.nonNullList(tableStructure.getAssociations())) + { + processTable(associationTableStructure, fieldList, headerRow); + } + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private String massageHeader(String header, boolean stripNumbers) + { + if(header == null) + { + return (null); + } + + String massagedWithNumbers = header.toLowerCase(Locale.ROOT).replaceAll("[^a-z0-9]", ""); + return stripNumbers ? massagedWithNumbers.replaceAll("[0-9]", "") : massagedWithNumbers; + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadTableStructureBuilder.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadTableStructureBuilder.java new file mode 100644 index 00000000..0ac81f00 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadTableStructureBuilder.java @@ -0,0 +1,132 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping; + + +import java.util.ArrayList; +import java.util.Comparator; +import java.util.HashSet; +import java.util.Set; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinOn; +import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinType; +import com.kingsrook.qqq.backend.core.model.metadata.joins.QJoinMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.tables.Association; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadTableStructure; +import com.kingsrook.qqq.backend.core.utils.CollectionUtils; +import com.kingsrook.qqq.backend.core.utils.ObjectUtils; +import com.kingsrook.qqq.backend.core.utils.StringUtils; + + +/******************************************************************************* + ** utility to build BulkLoadTableStructure objects for a QQQ Table. + *******************************************************************************/ +public class BulkLoadTableStructureBuilder +{ + /*************************************************************************** + ** + ***************************************************************************/ + public static BulkLoadTableStructure buildTableStructure(String tableName) + { + return (buildTableStructure(tableName, null, null)); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private static BulkLoadTableStructure buildTableStructure(String tableName, Association association, String parentAssociationPath) + { + QTableMetaData table = QContext.getQInstance().getTable(tableName); + + BulkLoadTableStructure tableStructure = new BulkLoadTableStructure(); + tableStructure.setTableName(tableName); + tableStructure.setLabel(table.getLabel()); + + Set associationJoinFieldNamesToExclude = new HashSet<>(); + + if(association == null) + { + tableStructure.setIsMain(true); + tableStructure.setIsMany(false); + tableStructure.setAssociationPath(null); + } + else + { + tableStructure.setIsMain(false); + + QJoinMetaData join = QContext.getQInstance().getJoin(association.getJoinName()); + if(join.getType().equals(JoinType.ONE_TO_MANY) || join.getType().equals(JoinType.MANY_TO_ONE)) + { + tableStructure.setIsMany(true); + } + + for(JoinOn joinOn : join.getJoinOns()) + { + //////////////////////////////////////////////////////////////////////////////////////////////// + // don't allow the user to map the "join field" from a child up to its parent // + // (e.g., you can't map lineItem.orderId -- that'll happen automatically via the association) // + //////////////////////////////////////////////////////////////////////////////////////////////// + if(join.getLeftTable().equals(tableName)) + { + associationJoinFieldNamesToExclude.add(joinOn.getLeftField()); + } + else if(join.getRightTable().equals(tableName)) + { + associationJoinFieldNamesToExclude.add(joinOn.getRightField()); + } + } + + if(!StringUtils.hasContent(parentAssociationPath)) + { + tableStructure.setAssociationPath(association.getName()); + } + else + { + tableStructure.setAssociationPath(parentAssociationPath + "." + association.getName()); + } + } + + ArrayList fields = new ArrayList<>(); + tableStructure.setFields(fields); + for(QFieldMetaData field : table.getFields().values()) + { + if(field.getIsEditable() && !associationJoinFieldNamesToExclude.contains(field.getName())) + { + fields.add(field); + } + } + + fields.sort(Comparator.comparing(f -> ObjectUtils.requireNonNullElse(f.getLabel(), f.getName(), ""))); + + for(Association childAssociation : CollectionUtils.nonNullList(table.getAssociations())) + { + BulkLoadTableStructure associatedStructure = buildTableStructure(childAssociation.getAssociatedTableName(), childAssociation, parentAssociationPath); + tableStructure.addAssociation(associatedStructure); + } + + return (tableStructure); + } +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/FlatRowsToRecord.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/FlatRowsToRecord.java index 44702312..d1a5d4c0 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/FlatRowsToRecord.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/FlatRowsToRecord.java @@ -31,6 +31,7 @@ import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.FileToRowsInterface; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkInsertMapping; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow; diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/RowsToRecordInterface.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/RowsToRecordInterface.java index 25aa6543..740781c2 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/RowsToRecordInterface.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/RowsToRecordInterface.java @@ -27,9 +27,10 @@ import java.util.List; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; -import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.FileToRowsInterface; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkInsertMapping; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow; +import com.kingsrook.qqq.backend.core.utils.CollectionUtils; import com.kingsrook.qqq.backend.core.utils.StringUtils; @@ -48,42 +49,49 @@ public interface RowsToRecordInterface /*************************************************************************** ** returns true if value from row was used, else false. ***************************************************************************/ - default boolean setValueOrDefault(QRecord record, QFieldMetaData field, String associationNameChain, BulkInsertMapping mapping, BulkLoadFileRow row, Integer index) + default boolean setValueOrDefault(QRecord record, QFieldMetaData field, String associationNameChain, BulkInsertMapping mapping, BulkLoadFileRow row, Integer columnIndex) { - String fieldName = field.getName(); - QFieldType type = field.getType(); + return setValueOrDefault(record, field, associationNameChain, mapping, row, columnIndex, null); + } - boolean valueFromRowWasUsed = false; - String fieldNameWithAssociationPrefix = StringUtils.hasContent(associationNameChain) ? associationNameChain + "." + fieldName : fieldName; + /*************************************************************************** + ** returns true if value from row was used, else false. + ***************************************************************************/ + default boolean setValueOrDefault(QRecord record, QFieldMetaData field, String associationNameChain, BulkInsertMapping mapping, BulkLoadFileRow row, Integer columnIndex, List wideAssociationIndexes) + { + boolean valueFromRowWasUsed = false; - Serializable value = null; - if(index != null && row != null) + ///////////////////////////////////////////////////////////////////////////////////////////////// + // build full field-name -- possibly associations, then field name, then possibly index-suffix // + ///////////////////////////////////////////////////////////////////////////////////////////////// + String fieldName = field.getName(); + String fieldNameWithAssociationPrefix = StringUtils.hasContent(associationNameChain) ? associationNameChain + "." + fieldName : fieldName; + + String wideAssociationSuffix = ""; + if(CollectionUtils.nullSafeHasContents(wideAssociationIndexes)) { - value = row.getValueElseNull(index); + wideAssociationSuffix = "," + StringUtils.join(".", wideAssociationIndexes); + } + + String fullFieldName = fieldNameWithAssociationPrefix + wideAssociationSuffix; + + ////////////////////////////////////////////// + // ok - look in the row - then the defaults // + ////////////////////////////////////////////// + Serializable value = null; + if(columnIndex != null && row != null) + { + value = row.getValueElseNull(columnIndex); if(value != null && !"".equals(value)) { valueFromRowWasUsed = true; } } - else if(mapping.getFieldNameToDefaultValueMap().containsKey(fieldNameWithAssociationPrefix)) + else if(mapping.getFieldNameToDefaultValueMap().containsKey(fullFieldName)) { - value = mapping.getFieldNameToDefaultValueMap().get(fieldNameWithAssociationPrefix); + value = mapping.getFieldNameToDefaultValueMap().get(fullFieldName); } - /* note - moving this to ValueMapper... - if(value != null) - { - try - { - value = ValueUtils.getValueAsFieldType(type, value); - } - catch(Exception e) - { - record.addError(new BadInputStatusMessage("Value [" + value + "] for field [" + field.getLabel() + "] could not be converted to type [" + type + "]")); - } - } - */ - if(value != null) { record.setValue(fieldName, value); diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/TallRowsToRecord.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/TallRowsToRecord.java index 690f705d..4587336c 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/TallRowsToRecord.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/TallRowsToRecord.java @@ -36,6 +36,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.Association; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.FileToRowsInterface; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkInsertMapping; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow; import com.kingsrook.qqq.backend.core.utils.CollectionUtils; import com.kingsrook.qqq.backend.core.utils.Pair; diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/ValueMapper.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/ValueMapper.java index 0f1e8cfc..fdf1b6bd 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/ValueMapper.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/ValueMapper.java @@ -35,6 +35,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; import com.kingsrook.qqq.backend.core.model.metadata.tables.Association; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.model.statusmessages.BadInputStatusMessage; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkInsertMapping; import com.kingsrook.qqq.backend.core.utils.CollectionUtils; import com.kingsrook.qqq.backend.core.utils.StringUtils; import com.kingsrook.qqq.backend.core.utils.ValueUtils; diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMapping.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMapping.java new file mode 100644 index 00000000..9c85647e --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMapping.java @@ -0,0 +1,196 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping; + + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.tables.Association; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.FileToRowsInterface; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkInsertMapping; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow; +import com.kingsrook.qqq.backend.core.utils.CollectionUtils; +import com.kingsrook.qqq.backend.core.utils.Pair; +import com.kingsrook.qqq.backend.core.utils.StringUtils; +import com.kingsrook.qqq.backend.core.utils.memoization.Memoization; + + +/******************************************************************************* + ** use a flatter mapping object, where field names look like: + ** associationChain.fieldName,index.subIndex + *******************************************************************************/ +public class WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMapping implements RowsToRecordInterface +{ + private Memoization, Boolean> shouldProcesssAssociationMemoization = new Memoization<>(); + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public List nextPage(FileToRowsInterface fileToRowsInterface, BulkLoadFileRow headerRow, BulkInsertMapping mapping, Integer limit) throws QException + { + QTableMetaData table = QContext.getQInstance().getTable(mapping.getTableName()); + if(table == null) + { + throw (new QException("Table [" + mapping.getTableName() + "] was not found in the Instance")); + } + + List rs = new ArrayList<>(); + + Map fieldIndexes = mapping.getFieldIndexes(table, null, headerRow); + + while(fileToRowsInterface.hasNext() && rs.size() < limit) + { + BulkLoadFileRow row = fileToRowsInterface.next(); + QRecord record = makeRecordFromRow(mapping, table, "", row, fieldIndexes, headerRow, new ArrayList<>()); + rs.add(record); + } + + ValueMapper.valueMapping(rs, mapping, table); + + return (rs); + } + + + + /*************************************************************************** + ** may return null, if there were no values in the row for this (sub-wide) record. + ***************************************************************************/ + private QRecord makeRecordFromRow(BulkInsertMapping mapping, QTableMetaData table, String associationNameChain, BulkLoadFileRow row, Map fieldIndexes, BulkLoadFileRow headerRow, List wideAssociationIndexes) throws QException + { + ////////////////////////////////////////////////////// + // start by building the record with its own fields // + ////////////////////////////////////////////////////// + QRecord record = new QRecord(); + boolean hadAnyValuesInRow = false; + for(QFieldMetaData field : table.getFields().values()) + { + hadAnyValuesInRow = setValueOrDefault(record, field, associationNameChain, mapping, row, fieldIndexes.get(field.getName()), wideAssociationIndexes) || hadAnyValuesInRow; + } + + if(!hadAnyValuesInRow) + { + return (null); + } + + ///////////////////////////// + // associations (children) // + ///////////////////////////// + for(String associationName : CollectionUtils.nonNullList(mapping.getMappedAssociations())) + { + boolean processAssociation = shouldProcessAssociation(associationNameChain, associationName); + + if(processAssociation) + { + String associationNameMinusChain = StringUtils.hasContent(associationNameChain) + ? associationName.substring(associationNameChain.length() + 1) + : associationName; + + Optional association = table.getAssociations().stream().filter(a -> a.getName().equals(associationNameMinusChain)).findFirst(); + if(association.isEmpty()) + { + throw (new QException("Couldn't find association: " + associationNameMinusChain + " under table: " + table.getName())); + } + + QTableMetaData associatedTable = QContext.getQInstance().getTable(association.get().getAssociatedTableName()); + + List associatedRecords = processAssociation(associationNameMinusChain, associationNameChain, associatedTable, mapping, row, headerRow); + record.withAssociatedRecords(associationNameMinusChain, associatedRecords); + } + } + + return record; + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private List processAssociation(String associationName, String associationNameChain, QTableMetaData associatedTable, BulkInsertMapping mapping, BulkLoadFileRow row, BulkLoadFileRow headerRow) throws QException + { + List rs = new ArrayList<>(); + + String associationNameChainForRecursiveCalls = "".equals(associationNameChain) ? associationName : associationNameChain + "." + associationName; + + for(int i = 0; true; i++) + { + // todo - doesn't support grand-children + List wideAssociationIndexes = List.of(i); + Map fieldIndexes = mapping.getFieldIndexes(associatedTable, associationNameChainForRecursiveCalls, headerRow, wideAssociationIndexes); + if(fieldIndexes.isEmpty()) + { + break; + } + + QRecord record = makeRecordFromRow(mapping, associatedTable, associationNameChainForRecursiveCalls, row, fieldIndexes, headerRow, wideAssociationIndexes); + if(record != null) + { + rs.add(record); + } + } + + return (rs); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + boolean shouldProcessAssociation(String associationNameChain, String associationName) + { + return shouldProcesssAssociationMemoization.getResult(Pair.of(associationNameChain, associationName), p -> + { + List chainParts = new ArrayList<>(); + List nameParts = new ArrayList<>(); + + if(StringUtils.hasContent(associationNameChain)) + { + chainParts.addAll(Arrays.asList(associationNameChain.split("\\."))); + } + + if(StringUtils.hasContent(associationName)) + { + nameParts.addAll(Arrays.asList(associationName.split("\\."))); + } + + if(!nameParts.isEmpty()) + { + nameParts.remove(nameParts.size() - 1); + } + + return (chainParts.equals(nameParts)); + }).orElse(false); + } + +} \ No newline at end of file diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithExplicitMapping.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithExplicitMapping.java deleted file mode 100644 index adc67ec7..00000000 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithExplicitMapping.java +++ /dev/null @@ -1,260 +0,0 @@ -/* - * QQQ - Low-code Application Framework for Engineers. - * Copyright (C) 2021-2024. Kingsrook, LLC - * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States - * contact@kingsrook.com - * https://github.com/Kingsrook/ - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping; - - -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.Set; -import com.kingsrook.qqq.backend.core.context.QContext; -import com.kingsrook.qqq.backend.core.exceptions.QException; -import com.kingsrook.qqq.backend.core.model.data.QRecord; -import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; -import com.kingsrook.qqq.backend.core.model.metadata.tables.Association; -import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; -import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.FileToRowsInterface; -import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow; -import com.kingsrook.qqq.backend.core.utils.CollectionUtils; -import com.kingsrook.qqq.backend.core.utils.Pair; -import com.kingsrook.qqq.backend.core.utils.StringUtils; -import com.kingsrook.qqq.backend.core.utils.memoization.Memoization; - - -/******************************************************************************* - ** - *******************************************************************************/ -public class WideRowsToRecordWithExplicitMapping implements RowsToRecordInterface -{ - private Memoization, Boolean> shouldProcesssAssociationMemoization = new Memoization<>(); - - - - /*************************************************************************** - ** - ***************************************************************************/ - @Override - public List nextPage(FileToRowsInterface fileToRowsInterface, BulkLoadFileRow headerRow, BulkInsertMapping mapping, Integer limit) throws QException - { - QTableMetaData table = QContext.getQInstance().getTable(mapping.getTableName()); - if(table == null) - { - throw (new QException("Table [" + mapping.getTableName() + "] was not found in the Instance")); - } - - List rs = new ArrayList<>(); - - Map fieldIndexes = mapping.getFieldIndexes(table, null, headerRow); - - while(fileToRowsInterface.hasNext() && rs.size() < limit) - { - BulkLoadFileRow row = fileToRowsInterface.next(); - QRecord record = new QRecord(); - - for(QFieldMetaData field : table.getFields().values()) - { - setValueOrDefault(record, field, null, mapping, row, fieldIndexes.get(field.getName())); - } - - processAssociations(mapping.getWideLayoutMapping(), "", headerRow, mapping, table, row, record); - - rs.add(record); - } - - ValueMapper.valueMapping(rs, mapping, table); - - return (rs); - } - - - - /*************************************************************************** - ** - ***************************************************************************/ - private void processAssociations(Map mappingMap, String associationNameChain, BulkLoadFileRow headerRow, BulkInsertMapping mapping, QTableMetaData table, BulkLoadFileRow row, QRecord record) throws QException - { - for(Map.Entry entry : CollectionUtils.nonNullMap(mappingMap).entrySet()) - { - String associationName = entry.getKey(); - BulkInsertWideLayoutMapping bulkInsertWideLayoutMapping = entry.getValue(); - - Optional association = table.getAssociations().stream().filter(a -> a.getName().equals(associationName)).findFirst(); - if(association.isEmpty()) - { - throw (new QException("Couldn't find association: " + associationName + " under table: " + table.getName())); - } - - QTableMetaData associatedTable = QContext.getQInstance().getTable(association.get().getAssociatedTableName()); - - String subChain = StringUtils.hasContent(associationNameChain) ? associationNameChain + "." + associationName: associationName; - - for(BulkInsertWideLayoutMapping.ChildRecordMapping childRecordMapping : bulkInsertWideLayoutMapping.getChildRecordMappings()) - { - QRecord associatedRecord = processAssociation(associatedTable, subChain, childRecordMapping, mapping, row, headerRow); - if(associatedRecord != null) - { - record.withAssociatedRecord(associationName, associatedRecord); - } - } - } - } - - - - /*************************************************************************** - ** - ***************************************************************************/ - private QRecord processAssociation(QTableMetaData table, String associationNameChain, BulkInsertWideLayoutMapping.ChildRecordMapping childRecordMapping, BulkInsertMapping mapping, BulkLoadFileRow row, BulkLoadFileRow headerRow) throws QException - { - Map fieldIndexes = childRecordMapping.getFieldIndexes(headerRow); - - QRecord associatedRecord = new QRecord(); - boolean usedAnyValuesFromRow = false; - - for(QFieldMetaData field : table.getFields().values()) - { - boolean valueFromRowWasUsed = setValueOrDefault(associatedRecord, field, associationNameChain, mapping, row, fieldIndexes.get(field.getName())); - usedAnyValuesFromRow |= valueFromRowWasUsed; - } - - if(usedAnyValuesFromRow) - { - processAssociations(childRecordMapping.getAssociationNameToChildRecordMappingMap(), associationNameChain, headerRow, mapping, table, row, associatedRecord); - return (associatedRecord); - } - else - { - return (null); - } - } - - // /*************************************************************************** - // ** - // ***************************************************************************/ - // private List processAssociationV2(String associationName, String associationNameChain, QTableMetaData table, BulkInsertMapping mapping, BulkLoadFileRow row, BulkLoadFileRow headerRow, QRecord record, int startIndex, int endIndex) throws QException - // { - // List rs = new ArrayList<>(); - - // Map fieldNameToHeaderNameMapForThisAssociation = new HashMap<>(); - // for(Map.Entry entry : mapping.getFieldNameToHeaderNameMap().entrySet()) - // { - // if(entry.getKey().startsWith(associationName + ".")) - // { - // String fieldName = entry.getKey().substring(associationName.length() + 1); - - // ////////////////////////////////////////////////////////////////////////// - // // make sure the name here is for this table - not a sub-table under it // - // ////////////////////////////////////////////////////////////////////////// - // if(!fieldName.contains(".")) - // { - // fieldNameToHeaderNameMapForThisAssociation.put(fieldName, entry.getValue()); - // } - // } - // } - - // ///////////////////////////////////////////////////////////////////// - // // loop over the length of the record, building associated records // - // ///////////////////////////////////////////////////////////////////// - // QRecord associatedRecord = new QRecord(); - // Set processedFieldNames = new HashSet<>(); - // boolean gotAnyValues = false; - // int subStartIndex = -1; - - // for(int i = startIndex; i < endIndex; i++) - // { - // String headerValue = ValueUtils.getValueAsString(headerRow.getValue(i)); - - // for(Map.Entry entry : fieldNameToHeaderNameMapForThisAssociation.entrySet()) - // { - // if(headerValue.equals(entry.getValue()) || headerValue.matches(entry.getValue() + " ?\\d+")) - // { - // /////////////////////////////////////////////// - // // ok - this is a value for this association // - // /////////////////////////////////////////////// - // if(subStartIndex == -1) - // { - // subStartIndex = i; - // } - - // String fieldName = entry.getKey(); - // if(processedFieldNames.contains(fieldName)) - // { - // ///////////////////////////////////////////////// - // // this means we're starting a new sub-record! // - // ///////////////////////////////////////////////// - // if(gotAnyValues) - // { - // addDefaultValuesToAssociatedRecord(processedFieldNames, table, associatedRecord, mapping, associationName); - // processAssociations(associationName, headerRow, mapping, table, row, associatedRecord, subStartIndex, i); - // rs.add(associatedRecord); - // } - - // associatedRecord = new QRecord(); - // processedFieldNames = new HashSet<>(); - // gotAnyValues = false; - // subStartIndex = i + 1; - // } - - // processedFieldNames.add(fieldName); - - // Serializable value = row.getValueElseNull(i); - // if(value != null && !"".equals(value)) - // { - // gotAnyValues = true; - // } - - // setValueOrDefault(associatedRecord, fieldName, associationName, mapping, row, i); - // } - // } - // } - - // //////////////////////// - // // handle final value // - // //////////////////////// - // if(gotAnyValues) - // { - // addDefaultValuesToAssociatedRecord(processedFieldNames, table, associatedRecord, mapping, associationName); - // processAssociations(associationName, headerRow, mapping, table, row, associatedRecord, subStartIndex, endIndex); - // rs.add(associatedRecord); - // } - - // return (rs); - // } - - - - /*************************************************************************** - ** - ***************************************************************************/ - private void addDefaultValuesToAssociatedRecord(Set processedFieldNames, QTableMetaData table, QRecord associatedRecord, BulkInsertMapping mapping, String associationNameChain) - { - for(QFieldMetaData field : table.getFields().values()) - { - if(!processedFieldNames.contains(field.getName())) - { - setValueOrDefault(associatedRecord, field, associationNameChain, mapping, null, null); - } - } - } - -} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithSpreadMapping.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithSpreadMapping.java index f87fa17b..7b215d2e 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithSpreadMapping.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithSpreadMapping.java @@ -38,6 +38,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.Association; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.FileToRowsInterface; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkInsertMapping; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow; import com.kingsrook.qqq.backend.core.utils.Pair; import com.kingsrook.qqq.backend.core.utils.StringUtils; diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkInsertMapping.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/model/BulkInsertMapping.java similarity index 93% rename from qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkInsertMapping.java rename to qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/model/BulkInsertMapping.java index fef870b1..86c19775 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkInsertMapping.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/model/BulkInsertMapping.java @@ -19,7 +19,7 @@ * along with this program. If not, see . */ -package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping; +package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model; import java.io.Serializable; @@ -34,7 +34,10 @@ import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.PossibleValueEnum; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; -import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping.FlatRowsToRecord; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping.RowsToRecordInterface; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping.TallRowsToRecord; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping.WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMapping; import com.kingsrook.qqq.backend.core.utils.CollectionUtils; import com.kingsrook.qqq.backend.core.utils.Pair; import com.kingsrook.qqq.backend.core.utils.StringUtils; @@ -63,8 +66,7 @@ public class BulkInsertMapping implements Serializable private Map fieldNameToDefaultValueMap = new HashMap<>(); private Map> fieldNameToValueMapping = new HashMap<>(); - private Map> tallLayoutGroupByIndexMap = new HashMap<>(); - private Map wideLayoutMapping = new HashMap<>(); + private Map> tallLayoutGroupByIndexMap = new HashMap<>(); private List mappedAssociations = new ArrayList<>(); @@ -79,7 +81,7 @@ public class BulkInsertMapping implements Serializable { FLAT(FlatRowsToRecord::new), TALL(TallRowsToRecord::new), - WIDE(WideRowsToRecordWithExplicitMapping::new); + WIDE(WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMapping::new); /*************************************************************************** @@ -137,10 +139,21 @@ public class BulkInsertMapping implements Serializable ***************************************************************************/ @JsonIgnore public Map getFieldIndexes(QTableMetaData table, String associationNameChain, BulkLoadFileRow headerRow) throws QException + { + return getFieldIndexes(table, associationNameChain, headerRow, null); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @JsonIgnore + public Map getFieldIndexes(QTableMetaData table, String associationNameChain, BulkLoadFileRow headerRow, List wideAssociationIndexes) throws QException { if(hasHeaderRow && fieldNameToHeaderNameMap != null) { - return (getFieldIndexesForHeaderMappedUseCase(table, associationNameChain, headerRow)); + return (getFieldIndexesForHeaderMappedUseCase(table, associationNameChain, headerRow, wideAssociationIndexes)); } else if(fieldNameToIndexMap != null) { @@ -208,7 +221,7 @@ public class BulkInsertMapping implements Serializable /*************************************************************************** ** ***************************************************************************/ - private Map getFieldIndexesForHeaderMappedUseCase(QTableMetaData table, String associationNameChain, BulkLoadFileRow headerRow) + private Map getFieldIndexesForHeaderMappedUseCase(QTableMetaData table, String associationNameChain, BulkLoadFileRow headerRow, List wideAssociationIndexes) { Map rs = new HashMap<>(); @@ -222,13 +235,19 @@ public class BulkInsertMapping implements Serializable headerToIndexMap.put(headerValue, i); } + String wideAssociationSuffix = ""; + if(CollectionUtils.nullSafeHasContents(wideAssociationIndexes)) + { + wideAssociationSuffix = "," + StringUtils.join(".", wideAssociationIndexes); + } + ///////////////////////////////////////////////////////////////////////////////////////////////////////// // loop over fields - finding what header name they are mapped to - then what index that header is at. // ///////////////////////////////////////////////////////////////////////////////////////////////////////// String fieldNamePrefix = !StringUtils.hasContent(associationNameChain) ? "" : associationNameChain + "."; for(QFieldMetaData field : table.getFields().values()) { - String headerName = fieldNameToHeaderNameMap.get(fieldNamePrefix + field.getName()); + String headerName = fieldNameToHeaderNameMap.get(fieldNamePrefix + field.getName() + wideAssociationSuffix); if(headerName != null) { Integer headerIndex = headerToIndexMap.get(headerName); @@ -527,34 +546,4 @@ public class BulkInsertMapping implements Serializable } - - /******************************************************************************* - ** Getter for wideLayoutMapping - *******************************************************************************/ - public Map getWideLayoutMapping() - { - return (this.wideLayoutMapping); - } - - - - /******************************************************************************* - ** Setter for wideLayoutMapping - *******************************************************************************/ - public void setWideLayoutMapping(Map wideLayoutMapping) - { - this.wideLayoutMapping = wideLayoutMapping; - } - - - - /******************************************************************************* - ** Fluent setter for wideLayoutMapping - *******************************************************************************/ - public BulkInsertMapping withWideLayoutMapping(Map wideLayoutMapping) - { - this.wideLayoutMapping = wideLayoutMapping; - return (this); - } - } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/model/BulkLoadProfileField.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/model/BulkLoadProfileField.java index 6bedb18b..f7ce0f6c 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/model/BulkLoadProfileField.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/model/BulkLoadProfileField.java @@ -33,6 +33,7 @@ public class BulkLoadProfileField { private String fieldName; private Integer columnIndex; + private String headerName; private Serializable defaultValue; private Boolean doValueMapping; private Map valueMappings; @@ -192,4 +193,35 @@ public class BulkLoadProfileField return (this); } + + /******************************************************************************* + ** Getter for headerName + *******************************************************************************/ + public String getHeaderName() + { + return (this.headerName); + } + + + + /******************************************************************************* + ** Setter for headerName + *******************************************************************************/ + public void setHeaderName(String headerName) + { + this.headerName = headerName; + } + + + + /******************************************************************************* + ** Fluent setter for headerName + *******************************************************************************/ + public BulkLoadProfileField withHeaderName(String headerName) + { + this.headerName = headerName; + return (this); + } + + } diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertV2Test.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertV2FullProcessTest.java similarity index 92% rename from qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertV2Test.java rename to qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertV2FullProcessTest.java index 8beaa891..7edbaafa 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertV2Test.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertV2FullProcessTest.java @@ -42,6 +42,7 @@ import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.frontend.QFrontendFieldMetaData; import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryRecordStore; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadProfile; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadProfileField; import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess; import com.kingsrook.qqq.backend.core.utils.JsonUtils; @@ -58,7 +59,7 @@ import static org.junit.jupiter.api.Assertions.assertNull; /******************************************************************************* ** Unit test for full bulk insert process *******************************************************************************/ -class BulkInsertV2Test extends BaseTest +class BulkInsertV2FullProcessTest extends BaseTest { /******************************************************************************* @@ -124,7 +125,7 @@ class BulkInsertV2Test extends BaseTest QInstance qInstance = QContext.getQInstance(); String processName = "PersonBulkInsertV2"; - new QInstanceEnricher(qInstance).defineTableBulkInsertV2(qInstance, qInstance.getTable(TestUtils.TABLE_NAME_PERSON_MEMORY), processName); + new QInstanceEnricher(qInstance).defineTableBulkInsert(qInstance, qInstance.getTable(TestUtils.TABLE_NAME_PERSON_MEMORY), processName); ///////////////////////////////////////////////////////// // start the process - expect to go to the upload step // @@ -159,6 +160,16 @@ class BulkInsertV2Test extends BaseTest runProcessOutput = new RunProcessAction().execute(runProcessInput); assertEquals(List.of("Id", "Create Date", "Modify Date", "First Name", "Last Name", "Birth Date", "Email", "Home State", "noOfShoes"), runProcessOutput.getValue("headerValues")); assertEquals(List.of("A", "B", "C", "D", "E", "F", "G", "H", "I"), runProcessOutput.getValue("headerLetters")); + + ////////////////////////////////////////////////////// + // assert about the suggested mapping that was done // + ////////////////////////////////////////////////////// + Serializable bulkLoadProfile = runProcessOutput.getValue("bulkLoadProfile"); + assertThat(bulkLoadProfile).isInstanceOf(BulkLoadProfile.class); + assertThat(((BulkLoadProfile) bulkLoadProfile).getFieldList()).hasSizeGreaterThan(5); + assertEquals("birthDate", ((BulkLoadProfile) bulkLoadProfile).getFieldList().get(0).getFieldName()); + assertEquals(5, ((BulkLoadProfile) bulkLoadProfile).getFieldList().get(0).getColumnIndex()); + assertThat(runProcessOutput.getProcessState().getNextStepName()).isPresent().get().isEqualTo("fileMapping"); //////////////////////////////////////////////////////////////////////////////// diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadMappingSuggesterTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadMappingSuggesterTest.java new file mode 100644 index 00000000..a54e08ef --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadMappingSuggesterTest.java @@ -0,0 +1,209 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping; + + +import java.util.List; +import com.kingsrook.qqq.backend.core.BaseTest; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadProfile; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadProfileField; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadTableStructure; +import com.kingsrook.qqq.backend.core.utils.TestUtils; +import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; + + +/******************************************************************************* + ** Unit test for BulkLoadMappingSuggester + *******************************************************************************/ +class BulkLoadMappingSuggesterTest extends BaseTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testSimpleFlat() + { + BulkLoadTableStructure tableStructure = BulkLoadTableStructureBuilder.buildTableStructure(TestUtils.TABLE_NAME_PERSON_MEMORY); + List headerRow = List.of("Id", "First Name", "lastname", "email", "homestate"); + + BulkLoadProfile bulkLoadProfile = new BulkLoadMappingSuggester().suggestBulkLoadMappingProfile(tableStructure, headerRow); + assertEquals("v1", bulkLoadProfile.getVersion()); + assertEquals("FLAT", bulkLoadProfile.getLayout()); + assertNull(getFieldByName(bulkLoadProfile, "id")); + assertEquals(1, getFieldByName(bulkLoadProfile, "firstName").getColumnIndex()); + assertEquals(2, getFieldByName(bulkLoadProfile, "lastName").getColumnIndex()); + assertEquals(3, getFieldByName(bulkLoadProfile, "email").getColumnIndex()); + assertEquals(4, getFieldByName(bulkLoadProfile, "homeStateId").getColumnIndex()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testSimpleTall() + { + BulkLoadTableStructure tableStructure = BulkLoadTableStructureBuilder.buildTableStructure(TestUtils.TABLE_NAME_ORDER); + List headerRow = List.of("orderNo", "shipto name", "sku", "quantity"); + + BulkLoadProfile bulkLoadProfile = new BulkLoadMappingSuggester().suggestBulkLoadMappingProfile(tableStructure, headerRow); + assertEquals("v1", bulkLoadProfile.getVersion()); + assertEquals("TALL", bulkLoadProfile.getLayout()); + assertEquals(0, getFieldByName(bulkLoadProfile, "orderNo").getColumnIndex()); + assertEquals(1, getFieldByName(bulkLoadProfile, "shipToName").getColumnIndex()); + assertEquals(2, getFieldByName(bulkLoadProfile, "orderLine.sku").getColumnIndex()); + assertEquals(3, getFieldByName(bulkLoadProfile, "orderLine.quantity").getColumnIndex()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testTallWithTableNamesOnAssociations() + { + BulkLoadTableStructure tableStructure = BulkLoadTableStructureBuilder.buildTableStructure(TestUtils.TABLE_NAME_ORDER); + List headerRow = List.of("Order No", "Ship To Name", "Order Line: SKU", "Order Line: Quantity"); + + BulkLoadProfile bulkLoadProfile = new BulkLoadMappingSuggester().suggestBulkLoadMappingProfile(tableStructure, headerRow); + assertEquals("v1", bulkLoadProfile.getVersion()); + assertEquals("TALL", bulkLoadProfile.getLayout()); + assertEquals(0, getFieldByName(bulkLoadProfile, "orderNo").getColumnIndex()); + assertEquals(1, getFieldByName(bulkLoadProfile, "shipToName").getColumnIndex()); + assertEquals(2, getFieldByName(bulkLoadProfile, "orderLine.sku").getColumnIndex()); + assertEquals(3, getFieldByName(bulkLoadProfile, "orderLine.quantity").getColumnIndex()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testChallengingAddress1And2() + { + try + { + { + QTableMetaData table = QContext.getQInstance().getTable(TestUtils.TABLE_NAME_ORDER); + table.addField(new QFieldMetaData("address1", QFieldType.STRING)); + table.addField(new QFieldMetaData("address2", QFieldType.STRING)); + + BulkLoadTableStructure tableStructure = BulkLoadTableStructureBuilder.buildTableStructure(TestUtils.TABLE_NAME_ORDER); + List headerRow = List.of("orderNo", "ship to name", "address 1", "address 2"); + + BulkLoadProfile bulkLoadProfile = new BulkLoadMappingSuggester().suggestBulkLoadMappingProfile(tableStructure, headerRow); + assertEquals(0, getFieldByName(bulkLoadProfile, "orderNo").getColumnIndex()); + assertEquals(1, getFieldByName(bulkLoadProfile, "shipToName").getColumnIndex()); + assertEquals(2, getFieldByName(bulkLoadProfile, "address1").getColumnIndex()); + assertEquals(3, getFieldByName(bulkLoadProfile, "address2").getColumnIndex()); + reInitInstanceInContext(TestUtils.defineInstance()); + } + + { + QTableMetaData table = QContext.getQInstance().getTable(TestUtils.TABLE_NAME_ORDER); + table.addField(new QFieldMetaData("address", QFieldType.STRING)); + table.addField(new QFieldMetaData("address2", QFieldType.STRING)); + + BulkLoadTableStructure tableStructure = BulkLoadTableStructureBuilder.buildTableStructure(TestUtils.TABLE_NAME_ORDER); + List headerRow = List.of("orderNo", "ship to name", "address 1", "address 2"); + + BulkLoadProfile bulkLoadProfile = new BulkLoadMappingSuggester().suggestBulkLoadMappingProfile(tableStructure, headerRow); + assertEquals(0, getFieldByName(bulkLoadProfile, "orderNo").getColumnIndex()); + assertEquals(1, getFieldByName(bulkLoadProfile, "shipToName").getColumnIndex()); + assertEquals(2, getFieldByName(bulkLoadProfile, "address").getColumnIndex()); + assertEquals(3, getFieldByName(bulkLoadProfile, "address2").getColumnIndex()); + reInitInstanceInContext(TestUtils.defineInstance()); + } + + { + QTableMetaData table = QContext.getQInstance().getTable(TestUtils.TABLE_NAME_ORDER); + table.addField(new QFieldMetaData("address1", QFieldType.STRING)); + table.addField(new QFieldMetaData("address2", QFieldType.STRING)); + + BulkLoadTableStructure tableStructure = BulkLoadTableStructureBuilder.buildTableStructure(TestUtils.TABLE_NAME_ORDER); + List headerRow = List.of("orderNo", "ship to name", "address", "address 2"); + + BulkLoadProfile bulkLoadProfile = new BulkLoadMappingSuggester().suggestBulkLoadMappingProfile(tableStructure, headerRow); + assertEquals(0, getFieldByName(bulkLoadProfile, "orderNo").getColumnIndex()); + assertEquals(1, getFieldByName(bulkLoadProfile, "shipToName").getColumnIndex()); + assertEquals(2, getFieldByName(bulkLoadProfile, "address1").getColumnIndex()); + assertEquals(3, getFieldByName(bulkLoadProfile, "address2").getColumnIndex()); + reInitInstanceInContext(TestUtils.defineInstance()); + } + } + finally + { + reInitInstanceInContext(TestUtils.defineInstance()); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testSimpleWide() + { + BulkLoadTableStructure tableStructure = BulkLoadTableStructureBuilder.buildTableStructure(TestUtils.TABLE_NAME_ORDER); + List headerRow = List.of("orderNo", "ship to name", "sku", "quantity1", "sku 2", "quantity 2"); + + BulkLoadProfile bulkLoadProfile = new BulkLoadMappingSuggester().suggestBulkLoadMappingProfile(tableStructure, headerRow); + assertEquals("v1", bulkLoadProfile.getVersion()); + assertEquals("WIDE", bulkLoadProfile.getLayout()); + assertEquals(0, getFieldByName(bulkLoadProfile, "orderNo").getColumnIndex()); + assertEquals(1, getFieldByName(bulkLoadProfile, "shipToName").getColumnIndex()); + assertEquals(2, getFieldByName(bulkLoadProfile, "orderLine.sku,0").getColumnIndex()); + assertEquals(3, getFieldByName(bulkLoadProfile, "orderLine.quantity,0").getColumnIndex()); + assertEquals(4, getFieldByName(bulkLoadProfile, "orderLine.sku,1").getColumnIndex()); + assertEquals(5, getFieldByName(bulkLoadProfile, "orderLine.quantity,1").getColumnIndex()); + + ///////////////////////////////////////////////////////////////// + // assert that the order of fields matches the file's ordering // + ///////////////////////////////////////////////////////////////// + assertEquals(List.of("orderNo", "shipToName", "orderLine.sku,0", "orderLine.quantity,0", "orderLine.sku,1", "orderLine.quantity,1"), + bulkLoadProfile.getFieldList().stream().map(f -> f.getFieldName()).toList()); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private BulkLoadProfileField getFieldByName(BulkLoadProfile bulkLoadProfile, String fieldName) + { + return (bulkLoadProfile.getFieldList().stream() + .filter(f -> f.getFieldName().equals(fieldName)) + .findFirst().orElse(null)); + } + +} \ No newline at end of file diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/FlatRowsToRecordTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/FlatRowsToRecordTest.java index 435dcf92..7124e946 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/FlatRowsToRecordTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/FlatRowsToRecordTest.java @@ -30,6 +30,7 @@ import com.kingsrook.qqq.backend.core.BaseTest; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.TestFileToRows; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkInsertMapping; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow; import com.kingsrook.qqq.backend.core.utils.TestUtils; import com.kingsrook.qqq.backend.core.utils.collections.ListBuilder; diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/TallRowsToRecordTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/TallRowsToRecordTest.java index 5c942fa2..1a0ef2ab 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/TallRowsToRecordTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/TallRowsToRecordTest.java @@ -29,6 +29,7 @@ import com.kingsrook.qqq.backend.core.BaseTest; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.CsvFileToRows; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkInsertMapping; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow; import com.kingsrook.qqq.backend.core.utils.TestUtils; import org.junit.jupiter.api.Test; diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/ValueMapperTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/ValueMapperTest.java index 7317d963..4348cb2b 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/ValueMapperTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/ValueMapperTest.java @@ -29,6 +29,7 @@ import com.kingsrook.qqq.backend.core.BaseTest; import com.kingsrook.qqq.backend.core.context.QContext; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkInsertMapping; import com.kingsrook.qqq.backend.core.utils.CollectionUtils; import com.kingsrook.qqq.backend.core.utils.TestUtils; import org.json.JSONArray; diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMappingTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMappingTest.java new file mode 100644 index 00000000..6e59c527 --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMappingTest.java @@ -0,0 +1,175 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping; + + +import java.io.Serializable; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import com.kingsrook.qqq.backend.core.BaseTest; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.CsvFileToRows; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkInsertMapping; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow; +import com.kingsrook.qqq.backend.core.utils.TestUtils; +import com.kingsrook.qqq.backend.core.utils.collections.MapBuilder; +import org.junit.jupiter.api.Test; +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertEquals; + + +/******************************************************************************* + ** Unit test for WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMapping + *******************************************************************************/ +class WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMappingTest extends BaseTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testOrderAndLinesWithoutDupes() throws QException + { + String csv = """ + orderNo, Ship To, lastName, SKU 1, Quantity 1, SKU 2, Quantity 2, SKU 3, Quantity 3 + 1, Homer, Simpson, DONUT, 12, BEER, 500, COUCH, 1 + 2, Ned, Flanders, BIBLE, 7, LAWNMOWER, 1 + """; + + CsvFileToRows fileToRows = CsvFileToRows.forString(csv); + BulkLoadFileRow header = fileToRows.next(); + + WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMapping rowsToRecord = new WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMapping(); + + BulkInsertMapping mapping = new BulkInsertMapping() + .withFieldNameToHeaderNameMap(Map.of( + "orderNo", "orderNo", + "shipToName", "Ship To", + "orderLine.sku,0", "SKU 1", + "orderLine.quantity,0", "Quantity 1", + "orderLine.sku,1", "SKU 2", + "orderLine.quantity,1", "Quantity 2", + "orderLine.sku,2", "SKU 3", + "orderLine.quantity,2", "Quantity 3" + )) + .withMappedAssociations(List.of("orderLine")) + .withTableName(TestUtils.TABLE_NAME_ORDER) + .withLayout(BulkInsertMapping.Layout.WIDE) + .withHasHeaderRow(true); + + List records = rowsToRecord.nextPage(fileToRows, header, mapping, Integer.MAX_VALUE); + assertEquals(2, records.size()); + + QRecord order = records.get(0); + assertEquals(1, order.getValueInteger("orderNo")); + assertEquals("Homer", order.getValueString("shipToName")); + assertEquals(List.of("DONUT", "BEER", "COUCH"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); + assertEquals(List.of(12, 500, 1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); + + order = records.get(1); + assertEquals(2, order.getValueInteger("orderNo")); + assertEquals("Ned", order.getValueString("shipToName")); + assertEquals(List.of("BIBLE", "LAWNMOWER"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); + assertEquals(List.of(7, 1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testOrderLinesAndOrderExtrinsicWithoutDupes() throws QException + { + String csv = """ + orderNo, Ship To, lastName, SKU 1, Quantity 1, SKU 2, Quantity 2, SKU 3, Quantity 3, Extrinsic Key 1, Extrinsic Value 1, Extrinsic Key 2, Extrinsic Value 2 + 1, Homer, Simpson, DONUT, 12, BEER, 500, COUCH, 1, Store Name, QQQ Mart, Coupon Code, 10QOff + 2, Ned, Flanders, BIBLE, 7, LAWNMOWER, 1 + """; + + CsvFileToRows fileToRows = CsvFileToRows.forString(csv); + BulkLoadFileRow header = fileToRows.next(); + + WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMapping rowsToRecord = new WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMapping(); + + BulkInsertMapping mapping = new BulkInsertMapping() + .withFieldNameToHeaderNameMap(MapBuilder.of(() -> new HashMap()) + .with("orderNo", "orderNo") + .with("shipToName", "Ship To") + + .with("orderLine.sku,0", "SKU 1") + .with("orderLine.quantity,0", "Quantity 1") + .with("orderLine.sku,1", "SKU 2") + .with("orderLine.quantity,1", "Quantity 2") + .with("orderLine.sku,2", "SKU 3") + .with("orderLine.quantity,2", "Quantity 3") + + .with("extrinsics.key,0", "Extrinsic Key 1") + .with("extrinsics.value,0", "Extrinsic Value 1") + .with("extrinsics.key,1", "Extrinsic Key 2") + .with("extrinsics.value,1", "Extrinsic Value 2") + .build() + ) + .withFieldNameToDefaultValueMap(Map.of( + "orderLine.lineNumber,0", "1", + "orderLine.lineNumber,1", "2", + "orderLine.lineNumber,2", "3" + )) + .withMappedAssociations(List.of("orderLine", "extrinsics")) + .withTableName(TestUtils.TABLE_NAME_ORDER) + .withLayout(BulkInsertMapping.Layout.WIDE) + .withHasHeaderRow(true); + + List records = rowsToRecord.nextPage(fileToRows, header, mapping, Integer.MAX_VALUE); + assertEquals(2, records.size()); + + QRecord order = records.get(0); + assertEquals(1, order.getValueInteger("orderNo")); + assertEquals("Homer", order.getValueString("shipToName")); + assertEquals(List.of("DONUT", "BEER", "COUCH"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); + assertEquals(List.of(12, 500, 1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); + assertEquals(List.of("1", "2", "3"), getValues(order.getAssociatedRecords().get("orderLine"), "lineNumber")); + assertEquals(List.of("Store Name", "Coupon Code"), getValues(order.getAssociatedRecords().get("extrinsics"), "key")); + assertEquals(List.of("QQQ Mart", "10QOff"), getValues(order.getAssociatedRecords().get("extrinsics"), "value")); + + order = records.get(1); + assertEquals(2, order.getValueInteger("orderNo")); + assertEquals("Ned", order.getValueString("shipToName")); + assertEquals(List.of("BIBLE", "LAWNMOWER"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); + assertEquals(List.of(7, 1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); + assertEquals(List.of("1", "2"), getValues(order.getAssociatedRecords().get("orderLine"), "lineNumber")); + assertThat(order.getAssociatedRecords().get("extrinsics")).isNullOrEmpty(); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private List getValues(List records, String fieldName) + { + return (records.stream().map(r -> r.getValue(fieldName)).toList()); + } + +} \ No newline at end of file diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithExplicitMappingTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithExplicitMappingTest.java deleted file mode 100644 index 4f8bd7a1..00000000 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithExplicitMappingTest.java +++ /dev/null @@ -1,269 +0,0 @@ -/* - * QQQ - Low-code Application Framework for Engineers. - * Copyright (C) 2021-2024. Kingsrook, LLC - * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States - * contact@kingsrook.com - * https://github.com/Kingsrook/ - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping; - - -import java.io.Serializable; -import java.util.List; -import java.util.Map; -import com.kingsrook.qqq.backend.core.BaseTest; -import com.kingsrook.qqq.backend.core.exceptions.QException; -import com.kingsrook.qqq.backend.core.model.data.QRecord; -import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.CsvFileToRows; -import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow; -import com.kingsrook.qqq.backend.core.utils.TestUtils; -import org.junit.jupiter.api.Test; -import static org.assertj.core.api.Assertions.assertThat; -import static org.junit.jupiter.api.Assertions.assertEquals; - - -/******************************************************************************* - ** Unit test for WideRowsToRecord - *******************************************************************************/ -class WideRowsToRecordWithExplicitMappingTest extends BaseTest -{ - - /******************************************************************************* - ** - *******************************************************************************/ - @Test - void testOrderAndLinesWithoutDupes() throws QException - { - String csv = """ - orderNo, Ship To, lastName, SKU 1, Quantity 1, SKU 2, Quantity 2, SKU 3, Quantity 3 - 1, Homer, Simpson, DONUT, 12, BEER, 500, COUCH, 1 - 2, Ned, Flanders, BIBLE, 7, LAWNMOWER, 1 - """; - - CsvFileToRows fileToRows = CsvFileToRows.forString(csv); - BulkLoadFileRow header = fileToRows.next(); - - WideRowsToRecordWithExplicitMapping rowsToRecord = new WideRowsToRecordWithExplicitMapping(); - - BulkInsertMapping mapping = new BulkInsertMapping() - .withFieldNameToHeaderNameMap(Map.of( - "orderNo", "orderNo", - "shipToName", "Ship To" - )) - .withWideLayoutMapping(Map.of( - "orderLine", new BulkInsertWideLayoutMapping(List.of( - new BulkInsertWideLayoutMapping.ChildRecordMapping(Map.of("sku", "SKU 1", "quantity", "Quantity 1")), - new BulkInsertWideLayoutMapping.ChildRecordMapping(Map.of("sku", "SKU 2", "quantity", "Quantity 2")), - new BulkInsertWideLayoutMapping.ChildRecordMapping(Map.of("sku", "SKU 3", "quantity", "Quantity 3")) - )) - )) - .withMappedAssociations(List.of("orderLine")) - .withTableName(TestUtils.TABLE_NAME_ORDER) - .withLayout(BulkInsertMapping.Layout.WIDE) - .withHasHeaderRow(true); - - List records = rowsToRecord.nextPage(fileToRows, header, mapping, Integer.MAX_VALUE); - assertEquals(2, records.size()); - - QRecord order = records.get(0); - assertEquals(1, order.getValueInteger("orderNo")); - assertEquals("Homer", order.getValueString("shipToName")); - assertEquals(List.of("DONUT", "BEER", "COUCH"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); - assertEquals(List.of(12, 500, 1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); - - order = records.get(1); - assertEquals(2, order.getValueInteger("orderNo")); - assertEquals("Ned", order.getValueString("shipToName")); - assertEquals(List.of("BIBLE", "LAWNMOWER"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); - assertEquals(List.of(7, 1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); - } - - - - /******************************************************************************* - ** - *******************************************************************************/ - @Test - void testOrderLinesAndOrderExtrinsicWithoutDupes() throws QException - { - String csv = """ - orderNo, Ship To, lastName, SKU 1, Quantity 1, SKU 2, Quantity 2, SKU 3, Quantity 3, Extrinsic Key 1, Extrinsic Value 1, Extrinsic Key 2, Extrinsic Value 2 - 1, Homer, Simpson, DONUT, 12, BEER, 500, COUCH, 1, Store Name, QQQ Mart, Coupon Code, 10QOff - 2, Ned, Flanders, BIBLE, 7, LAWNMOWER, 1 - """; - - CsvFileToRows fileToRows = CsvFileToRows.forString(csv); - BulkLoadFileRow header = fileToRows.next(); - - WideRowsToRecordWithExplicitMapping rowsToRecord = new WideRowsToRecordWithExplicitMapping(); - - BulkInsertMapping mapping = new BulkInsertMapping() - .withFieldNameToHeaderNameMap(Map.of( - "orderNo", "orderNo", - "shipToName", "Ship To" - )) - .withWideLayoutMapping(Map.of( - "orderLine", new BulkInsertWideLayoutMapping(List.of( - new BulkInsertWideLayoutMapping.ChildRecordMapping(Map.of("sku", "SKU 1", "quantity", "Quantity 1")), - new BulkInsertWideLayoutMapping.ChildRecordMapping(Map.of("sku", "SKU 2", "quantity", "Quantity 2")), - new BulkInsertWideLayoutMapping.ChildRecordMapping(Map.of("sku", "SKU 3", "quantity", "Quantity 3")) - )), - "extrinsics", new BulkInsertWideLayoutMapping(List.of( - new BulkInsertWideLayoutMapping.ChildRecordMapping(Map.of("key", "Extrinsic Key 1", "value", "Extrinsic Value 1")), - new BulkInsertWideLayoutMapping.ChildRecordMapping(Map.of("key", "Extrinsic Key 2", "value", "Extrinsic Value 2")) - )) - )) - .withMappedAssociations(List.of("orderLine", "extrinsics")) - .withTableName(TestUtils.TABLE_NAME_ORDER) - .withLayout(BulkInsertMapping.Layout.WIDE) - .withHasHeaderRow(true); - - List records = rowsToRecord.nextPage(fileToRows, header, mapping, Integer.MAX_VALUE); - assertEquals(2, records.size()); - - QRecord order = records.get(0); - assertEquals(1, order.getValueInteger("orderNo")); - assertEquals("Homer", order.getValueString("shipToName")); - assertEquals(List.of("DONUT", "BEER", "COUCH"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); - assertEquals(List.of(12, 500, 1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); - assertEquals(List.of("Store Name", "Coupon Code"), getValues(order.getAssociatedRecords().get("extrinsics"), "key")); - assertEquals(List.of("QQQ Mart", "10QOff"), getValues(order.getAssociatedRecords().get("extrinsics"), "value")); - - order = records.get(1); - assertEquals(2, order.getValueInteger("orderNo")); - assertEquals("Ned", order.getValueString("shipToName")); - assertEquals(List.of("BIBLE", "LAWNMOWER"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); - assertEquals(List.of(7, 1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); - assertThat(order.getAssociatedRecords().get("extrinsics")).isNullOrEmpty(); - } - - - - /******************************************************************************* - ** - *******************************************************************************/ - @Test - void testOrderLinesWithLineExtrinsicsAndOrderExtrinsicWithoutDupes() throws QException - { - String csv = """ - orderNo, Ship To, lastName, Extrinsic Key 1, Extrinsic Value 1, Extrinsic Key 2, Extrinsic Value 2, SKU 1, Quantity 1, Line Extrinsic Key 1.1, Line Extrinsic Value 1.1, Line Extrinsic Key 1.2, Line Extrinsic Value 1.2, SKU 2, Quantity 2, Line Extrinsic Key 2.1, Line Extrinsic Value 2.1, SKU 3, Quantity 3, Line Extrinsic Key 3.1, Line Extrinsic Value 3.1, Line Extrinsic Key 3.2 - 1, Homer, Simpson, Store Name, QQQ Mart, Coupon Code, 10QOff, DONUT, 12, Flavor, Chocolate, Size, Large, BEER, 500, Flavor, Hops, COUCH, 1, Color, Brown, foo, - 2, Ned, Flanders, , , , , BIBLE, 7, Flavor, King James, Size, X-Large, LAWNMOWER, 1 - """; - - Integer defaultStoreId = 42; - String defaultLineNo = "47"; - String defaultLineExtraValue = "bar"; - - CsvFileToRows fileToRows = CsvFileToRows.forString(csv); - BulkLoadFileRow header = fileToRows.next(); - - WideRowsToRecordWithExplicitMapping rowsToRecord = new WideRowsToRecordWithExplicitMapping(); - - BulkInsertMapping mapping = new BulkInsertMapping() - .withFieldNameToHeaderNameMap(Map.of( - "orderNo", "orderNo", - "shipToName", "Ship To" - )) - .withMappedAssociations(List.of("orderLine", "extrinsics", "orderLine.extrinsics")) - .withWideLayoutMapping(Map.of( - "orderLine", new BulkInsertWideLayoutMapping(List.of( - new BulkInsertWideLayoutMapping.ChildRecordMapping( - Map.of("sku", "SKU 1", "quantity", "Quantity 1"), - Map.of("extrinsics", new BulkInsertWideLayoutMapping(List.of( - new BulkInsertWideLayoutMapping.ChildRecordMapping(Map.of("key", "Line Extrinsic Key 1.1", "value", "Line Extrinsic Value 1.1")), - new BulkInsertWideLayoutMapping.ChildRecordMapping(Map.of("key", "Line Extrinsic Key 1.2", "value", "Line Extrinsic Value 1.2")) - )))), - new BulkInsertWideLayoutMapping.ChildRecordMapping( - Map.of("sku", "SKU 2", "quantity", "Quantity 2"), - Map.of("extrinsics", new BulkInsertWideLayoutMapping(List.of( - new BulkInsertWideLayoutMapping.ChildRecordMapping(Map.of("key", "Line Extrinsic Key 2.1", "value", "Line Extrinsic Value 2.1")), - new BulkInsertWideLayoutMapping.ChildRecordMapping(Map.of("key", "Line Extrinsic Key 2.2", "value", "Line Extrinsic Value 2.2")) - )))), - new BulkInsertWideLayoutMapping.ChildRecordMapping( - Map.of("sku", "SKU 3", "quantity", "Quantity 3"), - Map.of("extrinsics", new BulkInsertWideLayoutMapping(List.of( - new BulkInsertWideLayoutMapping.ChildRecordMapping(Map.of("key", "Line Extrinsic Key 3.1", "value", "Line Extrinsic Value 3.1")), - new BulkInsertWideLayoutMapping.ChildRecordMapping(Map.of("key", "Line Extrinsic Key 3.2", "value", "Line Extrinsic Value 3.2")) - )))) - )), - "extrinsics", new BulkInsertWideLayoutMapping(List.of( - new BulkInsertWideLayoutMapping.ChildRecordMapping(Map.of("key", "Extrinsic Key 1", "value", "Extrinsic Value 1")), - new BulkInsertWideLayoutMapping.ChildRecordMapping(Map.of("key", "Extrinsic Key 2", "value", "Extrinsic Value 2")) - )) - )) - - .withFieldNameToValueMapping(Map.of("orderLine.extrinsics.value", Map.of("Large", "L", "X-Large", "XL"))) - .withFieldNameToDefaultValueMap(Map.of( - "storeId", defaultStoreId, - "orderLine.lineNumber", defaultLineNo, - "orderLine.extrinsics.value", defaultLineExtraValue - )) - .withTableName(TestUtils.TABLE_NAME_ORDER) - .withLayout(BulkInsertMapping.Layout.WIDE) - .withHasHeaderRow(true); - - List records = rowsToRecord.nextPage(fileToRows, header, mapping, Integer.MAX_VALUE); - assertEquals(2, records.size()); - - QRecord order = records.get(0); - assertEquals(1, order.getValueInteger("orderNo")); - assertEquals("Homer", order.getValueString("shipToName")); - assertEquals(defaultStoreId, order.getValue("storeId")); - assertEquals(List.of("DONUT", "BEER", "COUCH"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); - assertEquals(List.of(12, 500, 1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); - assertEquals(List.of(defaultLineNo, defaultLineNo, defaultLineNo), getValues(order.getAssociatedRecords().get("orderLine"), "lineNumber")); - assertEquals(List.of("Store Name", "Coupon Code"), getValues(order.getAssociatedRecords().get("extrinsics"), "key")); - assertEquals(List.of("QQQ Mart", "10QOff"), getValues(order.getAssociatedRecords().get("extrinsics"), "value")); - - QRecord lineItem = order.getAssociatedRecords().get("orderLine").get(0); - assertEquals(List.of("Flavor", "Size"), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "key")); - assertEquals(List.of("Chocolate", "L"), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "value")); - - lineItem = order.getAssociatedRecords().get("orderLine").get(1); - assertEquals(List.of("Flavor"), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "key")); - assertEquals(List.of("Hops"), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "value")); - - lineItem = order.getAssociatedRecords().get("orderLine").get(2); - assertEquals(List.of("Color", "foo"), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "key")); - assertEquals(List.of("Brown", defaultLineExtraValue), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "value")); - - order = records.get(1); - assertEquals(2, order.getValueInteger("orderNo")); - assertEquals("Ned", order.getValueString("shipToName")); - assertEquals(defaultStoreId, order.getValue("storeId")); - assertEquals(List.of("BIBLE", "LAWNMOWER"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); - assertEquals(List.of(7, 1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); - assertEquals(List.of(defaultLineNo, defaultLineNo), getValues(order.getAssociatedRecords().get("orderLine"), "lineNumber")); - assertThat(order.getAssociatedRecords().get("extrinsics")).isNullOrEmpty(); - - lineItem = order.getAssociatedRecords().get("orderLine").get(0); - assertEquals(List.of("Flavor", "Size"), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "key")); - assertEquals(List.of("King James", "XL"), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "value")); - } - - - - /*************************************************************************** - ** - ***************************************************************************/ - private List getValues(List records, String fieldName) - { - return (records.stream().map(r -> r.getValue(fieldName)).toList()); - } - -} \ No newline at end of file diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithSpreadMappingTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithSpreadMappingTest.java index 61fc6d35..8bd49beb 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithSpreadMappingTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithSpreadMappingTest.java @@ -29,6 +29,7 @@ import com.kingsrook.qqq.backend.core.BaseTest; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.CsvFileToRows; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkInsertMapping; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow; import com.kingsrook.qqq.backend.core.utils.TestUtils; import org.junit.jupiter.api.Test;