diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertPrepareFileMappingStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertPrepareFileMappingStep.java new file mode 100644 index 00000000..f7ee32ce --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertPrepareFileMappingStep.java @@ -0,0 +1,240 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert; + + +import java.io.File; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.HashSet; +import java.util.Set; +import com.kingsrook.qqq.backend.core.actions.processes.BackendStep; +import com.kingsrook.qqq.backend.core.actions.tables.StorageAction; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; +import com.kingsrook.qqq.backend.core.model.actions.tables.storage.StorageInput; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinOn; +import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinType; +import com.kingsrook.qqq.backend.core.model.metadata.joins.QJoinMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.tables.Association; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.FileToRowsInterface; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadTableStructure; +import com.kingsrook.qqq.backend.core.utils.CollectionUtils; +import com.kingsrook.qqq.backend.core.utils.StringUtils; +import com.kingsrook.qqq.backend.core.utils.ValueUtils; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class BulkInsertPrepareFileMappingStep implements BackendStep +{ + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException + { + buildFileDetailsForMappingStep(runBackendStepInput, runBackendStepOutput); + buildFieldsForMappingStep(runBackendStepInput, runBackendStepOutput); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private static void buildFileDetailsForMappingStep(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException + { + StorageInput storageInput = BulkInsertStepUtils.getStorageInputForTheFile(runBackendStepInput); + File file = new File(storageInput.getReference()); + runBackendStepOutput.addValue("fileBaseName", file.getName()); + + try + ( + /////////////////////////////////////////////////////////////////////////////////////////////////////////// + // open a stream to read from our file, and a FileToRows object, that knows how to read from that stream // + /////////////////////////////////////////////////////////////////////////////////////////////////////////// + InputStream inputStream = new StorageAction().getInputStream(storageInput); + FileToRowsInterface fileToRowsInterface = FileToRowsInterface.forFile(storageInput.getReference(), inputStream); + ) + { + ///////////////////////////////////////////////// + // read the 1st row, and assume it is a header // + ///////////////////////////////////////////////// + BulkLoadFileRow headerRow = fileToRowsInterface.next(); + ArrayList headerValues = new ArrayList<>(); + ArrayList headerLetters = new ArrayList<>(); + for(int i = 0; i < headerRow.size(); i++) + { + headerValues.add(ValueUtils.getValueAsString(headerRow.getValue(i))); + headerLetters.add(toHeaderLetter(i)); + } + runBackendStepOutput.addValue("headerValues", headerValues); + runBackendStepOutput.addValue("headerLetters", headerLetters); + + /////////////////////////////////////////////////////////////////////////////////////////// + // while there are more rows in the file - and we're under preview-rows limit, read rows // + /////////////////////////////////////////////////////////////////////////////////////////// + int previewRows = 0; + int previewRowsLimit = 5; + ArrayList> bodyValues = new ArrayList<>(); + for(int i = 0; i < headerRow.size(); i++) + { + bodyValues.add(new ArrayList<>()); + } + + while(fileToRowsInterface.hasNext() && previewRows < previewRowsLimit) + { + BulkLoadFileRow bodyRow = fileToRowsInterface.next(); + previewRows++; + + for(int i = 0; i < headerRow.size(); i++) + { + bodyValues.get(i).add(ValueUtils.getValueAsString(bodyRow.getValueElseNull(i))); + } + } + runBackendStepOutput.addValue("bodyValuesPreview", bodyValues); + } + catch(Exception e) + { + throw (new QException("Error reading bulk load file", e)); + } + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + static String toHeaderLetter(int i) + { + StringBuilder rs = new StringBuilder(); + + do + { + rs.insert(0, (char) ('A' + (i % 26))); + i = (i / 26) - 1; + } + while(i >= 0); + + return (rs.toString()); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private static void buildFieldsForMappingStep(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) + { + String tableName = runBackendStepInput.getValueString("tableName"); + BulkLoadTableStructure tableStructure = buildTableStructure(tableName, null, null); + runBackendStepOutput.addValue("tableStructure", tableStructure); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private static BulkLoadTableStructure buildTableStructure(String tableName, Association association, String parentAssociationPath) + { + QTableMetaData table = QContext.getQInstance().getTable(tableName); + + BulkLoadTableStructure tableStructure = new BulkLoadTableStructure(); + tableStructure.setTableName(tableName); + tableStructure.setLabel(table.getLabel()); + + Set associationJoinFieldNamesToExclude = new HashSet<>(); + + if(association == null) + { + tableStructure.setIsMain(true); + tableStructure.setIsMany(false); + tableStructure.setAssociationPath(null); + } + else + { + tableStructure.setIsMain(false); + + QJoinMetaData join = QContext.getQInstance().getJoin(association.getJoinName()); + if(join.getType().equals(JoinType.ONE_TO_MANY) || join.getType().equals(JoinType.MANY_TO_ONE)) + { + tableStructure.setIsMany(true); + } + + for(JoinOn joinOn : join.getJoinOns()) + { + //////////////////////////////////////////////////////////////////////////////////////////////// + // don't allow the user to map the "join field" from a child up to its parent // + // (e.g., you can't map lineItem.orderId -- that'll happen automatically via the association) // + //////////////////////////////////////////////////////////////////////////////////////////////// + if(join.getLeftTable().equals(tableName)) + { + associationJoinFieldNamesToExclude.add(joinOn.getLeftField()); + } + else if(join.getRightTable().equals(tableName)) + { + associationJoinFieldNamesToExclude.add(joinOn.getRightField()); + } + } + + if(!StringUtils.hasContent(parentAssociationPath)) + { + tableStructure.setAssociationPath(association.getName()); + } + else + { + tableStructure.setAssociationPath(parentAssociationPath + "." + association.getName()); + } + } + + ArrayList fields = new ArrayList<>(); + tableStructure.setFields(fields); + for(QFieldMetaData field : table.getFields().values()) + { + if(field.getIsEditable() && !associationJoinFieldNamesToExclude.contains(field.getName())) + { + fields.add(field); + } + } + + fields.sort(Comparator.comparing(f -> f.getLabel())); + + for(Association childAssociation : CollectionUtils.nonNullList(table.getAssociations())) + { + BulkLoadTableStructure associatedStructure = buildTableStructure(childAssociation.getAssociatedTableName(), childAssociation, parentAssociationPath); + tableStructure.addAssociation(associatedStructure); + } + + return (tableStructure); + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertPrepareValueMappingStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertPrepareValueMappingStep.java new file mode 100644 index 00000000..85397442 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertPrepareValueMappingStep.java @@ -0,0 +1,250 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert; + + +import java.io.InputStream; +import java.io.Serializable; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import com.google.gson.reflect.TypeToken; +import com.kingsrook.qqq.backend.core.actions.processes.BackendStep; +import com.kingsrook.qqq.backend.core.actions.tables.StorageAction; +import com.kingsrook.qqq.backend.core.actions.values.SearchPossibleValueSourceAction; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.exceptions.QUserFacingException; +import com.kingsrook.qqq.backend.core.logging.QLogger; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; +import com.kingsrook.qqq.backend.core.model.actions.tables.storage.StorageInput; +import com.kingsrook.qqq.backend.core.model.actions.values.SearchPossibleValueSourceInput; +import com.kingsrook.qqq.backend.core.model.actions.values.SearchPossibleValueSourceOutput; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.frontend.QFrontendFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValue; +import com.kingsrook.qqq.backend.core.model.metadata.tables.Association; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.FileToRowsInterface; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping.BulkInsertMapping; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow; +import com.kingsrook.qqq.backend.core.utils.CollectionUtils; +import com.kingsrook.qqq.backend.core.utils.StringUtils; +import com.kingsrook.qqq.backend.core.utils.ValueUtils; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class BulkInsertPrepareValueMappingStep implements BackendStep +{ + private static final QLogger LOG = QLogger.getLogger(BulkInsertPrepareValueMappingStep.class); + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException + { + try + { + ///////////////////////////////////////////////////////////// + // prep the frontend for what field we're going to map now // + ///////////////////////////////////////////////////////////// + List fieldNamesToDoValueMapping = (List) runBackendStepInput.getValue("fieldNamesToDoValueMapping"); + Integer valueMappingFieldIndex = runBackendStepInput.getValueInteger("valueMappingFieldIndex"); + if(valueMappingFieldIndex == null) + { + valueMappingFieldIndex = 0; + } + else + { + valueMappingFieldIndex++; + } + + //////////////////////////////////////////////////////////////////////////////////////////////////// + // if there are no more fields (values) to map, then proceed to the standard streamed-ETL preview // + //////////////////////////////////////////////////////////////////////////////////////////////////// + if(valueMappingFieldIndex >= fieldNamesToDoValueMapping.size()) + { + BulkInsertStepUtils.setNextStepStreamedETLPreview(runBackendStepOutput); + return; + } + + runBackendStepInput.addValue("valueMappingFieldIndex", valueMappingFieldIndex); + + String fullFieldName = fieldNamesToDoValueMapping.get(valueMappingFieldIndex); + TableAndField tableAndField = getTableAndField(runBackendStepInput.getValueString("tableName"), fullFieldName); + + runBackendStepInput.addValue("valueMappingField", new QFrontendFieldMetaData(tableAndField.field())); + runBackendStepInput.addValue("valueMappingFullFieldName", fullFieldName); + runBackendStepInput.addValue("valueMappingFieldTableName", tableAndField.table().getName()); + + //////////////////////////////////////////////////// + // get all the values from the file in this field // + // todo - should do all mapping fields at once? // + //////////////////////////////////////////////////// + ArrayList fileValues = getValuesForField(tableAndField.table(), tableAndField.field(), fullFieldName, runBackendStepInput); + runBackendStepOutput.addValue("fileValues", fileValues); + + /////////////////////////////////////////////// + // clear these in case not getting set below // + /////////////////////////////////////////////// + runBackendStepOutput.addValue("valueMapping", new HashMap<>()); + runBackendStepOutput.addValue("mappedValueLabels", new HashMap<>()); + + BulkInsertMapping bulkInsertMapping = (BulkInsertMapping) runBackendStepInput.getValue("bulkInsertMapping"); + HashMap valueMapping = null; + if(bulkInsertMapping.getFieldNameToValueMapping() != null && bulkInsertMapping.getFieldNameToValueMapping().containsKey(fullFieldName)) + { + valueMapping = CollectionUtils.useOrWrap(bulkInsertMapping.getFieldNameToValueMapping().get(fullFieldName), new TypeToken<>() {}); + runBackendStepOutput.addValue("valueMapping", valueMapping); + + if(StringUtils.hasContent(tableAndField.field().getPossibleValueSourceName())) + { + HashMap possibleValueLabels = loadPossibleValues(tableAndField.field(), valueMapping); + runBackendStepOutput.addValue("mappedValueLabels", possibleValueLabels); + } + } + } + catch(Exception e) + { + LOG.warn("Error in bulk insert prepare value mapping", e); + throw new QException("Unhandled error in bulk insert prepare value mapping step", e); + } + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + public static TableAndField getTableAndField(String tableName, String fullFieldName) throws QException + { + List parts = new ArrayList<>(List.of(fullFieldName.split("\\."))); + String fieldBaseName = parts.remove(parts.size() - 1); + + QTableMetaData table = QContext.getQInstance().getTable(tableName); + for(String associationName : parts) + { + Optional association = table.getAssociations().stream().filter(a -> a.getName().equals(associationName)).findFirst(); + if(association.isPresent()) + { + table = QContext.getQInstance().getTable(association.get().getAssociatedTableName()); + } + else + { + throw new QException("Missing association [" + associationName + "] on table [" + table.getName() + "]"); + } + } + + TableAndField result = new TableAndField(table, table.getField(fieldBaseName)); + return result; + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + public record TableAndField(QTableMetaData table, QFieldMetaData field) {} + + + + /*************************************************************************** + ** + ***************************************************************************/ + private HashMap loadPossibleValues(QFieldMetaData field, Map valueMapping) throws QException + { + SearchPossibleValueSourceInput input = new SearchPossibleValueSourceInput(); + input.setPossibleValueSourceName(field.getPossibleValueSourceName()); + input.setIdList(new ArrayList<>(new HashSet<>(valueMapping.values()))); // go through a set to strip dupes + SearchPossibleValueSourceOutput output = new SearchPossibleValueSourceAction().execute(input); + + HashMap rs = new HashMap<>(); + for(QPossibleValue result : output.getResults()) + { + Serializable id = (Serializable) result.getId(); + rs.put(id, result.getLabel()); + } + return rs; + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private ArrayList getValuesForField(QTableMetaData table, QFieldMetaData field, String fullFieldName, RunBackendStepInput runBackendStepInput) throws QException + { + StorageInput storageInput = BulkInsertStepUtils.getStorageInputForTheFile(runBackendStepInput); + BulkInsertMapping bulkInsertMapping = (BulkInsertMapping) runBackendStepInput.getValue("bulkInsertMapping"); + + String associationNameChain = null; + if(fullFieldName.contains(".")) + { + associationNameChain = fullFieldName.substring(0, fullFieldName.lastIndexOf('.')); + } + + try + ( + InputStream inputStream = new StorageAction().getInputStream(storageInput); + FileToRowsInterface fileToRowsInterface = FileToRowsInterface.forFile(storageInput.getReference(), inputStream); + ) + { + Set values = new LinkedHashSet<>(); + BulkLoadFileRow headerRow = bulkInsertMapping.getHasHeaderRow() ? fileToRowsInterface.next() : null; + Map fieldIndexes = bulkInsertMapping.getFieldIndexes(table, associationNameChain, headerRow); + int index = fieldIndexes.get(field.getName()); + + while(fileToRowsInterface.hasNext()) + { + BulkLoadFileRow row = fileToRowsInterface.next(); + Serializable value = row.getValueElseNull(index); + if(value != null) + { + values.add(ValueUtils.getValueAsString(value)); + } + + if(values.size() > 100) + { + throw (new QUserFacingException("Too many unique values were found for mapping for field: " + field.getName())); + } + } + + return (new ArrayList<>(values)); + } + catch(Exception e) + { + throw (new QException("Error getting values from file", e)); + } + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertReceiveFileMappingStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertReceiveFileMappingStep.java new file mode 100644 index 00000000..8f72bb82 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertReceiveFileMappingStep.java @@ -0,0 +1,200 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert; + + +import java.io.InputStream; +import java.io.Serializable; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; +import com.kingsrook.qqq.backend.core.actions.processes.BackendStep; +import com.kingsrook.qqq.backend.core.actions.tables.StorageAction; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.logging.QLogger; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; +import com.kingsrook.qqq.backend.core.model.actions.tables.storage.StorageInput; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.FileToRowsInterface; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping.BulkInsertMapping; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadProfile; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadProfileField; +import com.kingsrook.qqq.backend.core.utils.CollectionUtils; +import com.kingsrook.qqq.backend.core.utils.ValueUtils; +import org.apache.commons.lang3.BooleanUtils; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class BulkInsertReceiveFileMappingStep implements BackendStep +{ + private static final QLogger LOG = QLogger.getLogger(BulkInsertReceiveFileMappingStep.class); + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException + { + try + { + BulkInsertStepUtils.handleSavedBulkLoadProfileIdValue(runBackendStepInput, runBackendStepOutput); + + /////////////////////////////////////////////////////////////////// + // read process values - construct a bulkLoadProfile out of them // + /////////////////////////////////////////////////////////////////// + BulkLoadProfile bulkLoadProfile = BulkInsertStepUtils.getBulkLoadProfile(runBackendStepInput); + + ///////////////////////////////////////////////////////////////////////// + // put the list of bulk load profile into the process state - it's the // + // thing that the frontend will be looking at as the saved profile // + ///////////////////////////////////////////////////////////////////////// + runBackendStepOutput.addValue("bulkLoadProfile", bulkLoadProfile); + + ////////////////////////////////////////////////////////////////////////////////////////////////////////// + // now build the mapping object that the backend wants - based on the bulkLoadProfile from the frontend // + ////////////////////////////////////////////////////////////////////////////////////////////////////////// + BulkInsertMapping bulkInsertMapping = new BulkInsertMapping(); + bulkInsertMapping.setTableName(runBackendStepInput.getTableName()); + bulkInsertMapping.setHasHeaderRow(bulkLoadProfile.getHasHeaderRow()); + bulkInsertMapping.setLayout(BulkInsertMapping.Layout.valueOf(bulkLoadProfile.getLayout())); + + ////////////////////////////////////////////////////////////////////////////////////////////// + // handle field to name or index mappings (depending on if there's a header row being used) // + ////////////////////////////////////////////////////////////////////////////////////////////// + if(BooleanUtils.isTrue(bulkInsertMapping.getHasHeaderRow())) + { + StorageInput storageInput = BulkInsertStepUtils.getStorageInputForTheFile(runBackendStepInput); + try + ( + InputStream inputStream = new StorageAction().getInputStream(storageInput); + FileToRowsInterface fileToRowsInterface = FileToRowsInterface.forFile(storageInput.getReference(), inputStream); + ) + { + Map fieldNameToHeaderNameMap = new HashMap<>(); + bulkInsertMapping.setFieldNameToHeaderNameMap(fieldNameToHeaderNameMap); + + BulkLoadFileRow headerRow = fileToRowsInterface.next(); + for(BulkLoadProfileField bulkLoadProfileField : bulkLoadProfile.getFieldList()) + { + if(bulkLoadProfileField.getColumnIndex() != null) + { + String headerName = ValueUtils.getValueAsString(headerRow.getValueElseNull(bulkLoadProfileField.getColumnIndex())); + fieldNameToHeaderNameMap.put(bulkLoadProfileField.getFieldName(), headerName); + } + } + } + } + else + { + Map fieldNameToIndexMap = new HashMap<>(); + bulkInsertMapping.setFieldNameToIndexMap(fieldNameToIndexMap); + for(BulkLoadProfileField bulkLoadProfileField : bulkLoadProfile.getFieldList()) + { + if(bulkLoadProfileField.getColumnIndex() != null) + { + fieldNameToIndexMap.put(bulkLoadProfileField.getFieldName(), bulkLoadProfileField.getColumnIndex()); + } + } + } + + ///////////////////////////////////// + // do fields w/ default values now // + ///////////////////////////////////// + HashMap fieldNameToDefaultValueMap = new HashMap<>(); + bulkInsertMapping.setFieldNameToDefaultValueMap(fieldNameToDefaultValueMap); + for(BulkLoadProfileField bulkLoadProfileField : bulkLoadProfile.getFieldList()) + { + if(bulkLoadProfileField.getDefaultValue() != null) + { + fieldNameToDefaultValueMap.put(bulkLoadProfileField.getFieldName(), bulkLoadProfileField.getDefaultValue()); + } + } + + ///////////////////////////////////////////////////////////////////////////////////////////// + // frontend at this point will have sent just told us which field names need value mapping // + // store those - and let them drive the value-mapping screens that we'll go through next // + // todo - uh, what if those come from profile, dummy!? + ///////////////////////////////////////////////////////////////////////////////////////////// + ArrayList fieldNamesToDoValueMapping = new ArrayList<>(); + for(BulkLoadProfileField bulkLoadProfileField : bulkLoadProfile.getFieldList()) + { + if(BooleanUtils.isTrue(bulkLoadProfileField.getDoValueMapping())) + { + fieldNamesToDoValueMapping.add(bulkLoadProfileField.getFieldName()); + + if(CollectionUtils.nullSafeHasContents(bulkLoadProfileField.getValueMappings())) + { + bulkInsertMapping.getFieldNameToValueMapping().put(bulkLoadProfileField.getFieldName(), bulkLoadProfileField.getValueMappings()); + } + } + } + runBackendStepOutput.addValue("fieldNamesToDoValueMapping", new ArrayList<>(fieldNamesToDoValueMapping)); + + /////////////////////////////////////////////////////////////////////////////////////// + // figure out what associations are being mapped, by looking at the full field names // + /////////////////////////////////////////////////////////////////////////////////////// + Set associationNameSet = new HashSet<>(); + for(BulkLoadProfileField bulkLoadProfileField : bulkLoadProfile.getFieldList()) + { + if(bulkLoadProfileField.getFieldName().contains(".")) + { + associationNameSet.add(bulkLoadProfileField.getFieldName().substring(0, bulkLoadProfileField.getFieldName().lastIndexOf('.'))); + } + } + bulkInsertMapping.setMappedAssociations(new ArrayList<>(associationNameSet)); + + ///////////////////////////////////////////////////////////////////////////////////////////////////// + // at this point we're done populating the bulkInsertMapping object. put it in the process state. // + ///////////////////////////////////////////////////////////////////////////////////////////////////// + runBackendStepOutput.addValue("bulkInsertMapping", bulkInsertMapping); + + if(CollectionUtils.nullSafeHasContents(fieldNamesToDoValueMapping)) + { + ////////////////////////////////////////////////////////////////////////////////// + // just go to the prepareValueMapping backend step - it'll figure out the rest. // + // it's also where the value-mapping loop of steps points. // + // and, this will actually be the default (e.g., the step after this one). // + ////////////////////////////////////////////////////////////////////////////////// + } + else + { + ////////////////////////////////////////////////////////////////////////////////// + // else - if no values to map - continue with the standard streamed-ETL preview // + ////////////////////////////////////////////////////////////////////////////////// + BulkInsertStepUtils.setNextStepStreamedETLPreview(runBackendStepOutput); + } + } + catch(Exception e) + { + LOG.warn("Error in bulk insert receive mapping", e); + throw new QException("Unhandled error in bulk insert receive mapping step", e); + } + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertReceiveFileStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertReceiveFileStep.java deleted file mode 100644 index 6ebcf3a8..00000000 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertReceiveFileStep.java +++ /dev/null @@ -1,81 +0,0 @@ -/* - * QQQ - Low-code Application Framework for Engineers. - * Copyright (C) 2021-2024. Kingsrook, LLC - * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States - * contact@kingsrook.com - * https://github.com/Kingsrook/ - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert; - - -import java.io.InputStream; -import java.util.ArrayList; -import java.util.List; -import com.kingsrook.qqq.backend.core.actions.processes.BackendStep; -import com.kingsrook.qqq.backend.core.actions.tables.StorageAction; -import com.kingsrook.qqq.backend.core.exceptions.QException; -import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; -import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; -import com.kingsrook.qqq.backend.core.model.actions.tables.storage.StorageInput; -import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.FileToRowsInterface; -import com.kingsrook.qqq.backend.core.utils.JsonUtils; - - -/******************************************************************************* - ** - *******************************************************************************/ -public class BulkInsertReceiveFileStep implements BackendStep -{ - - /*************************************************************************** - ** - ***************************************************************************/ - @Override - public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException - { - StorageInput storageInput = BulkInsertStepUtils.getStorageInputForTheFile(runBackendStepInput); - - try - ( - InputStream inputStream = new StorageAction().getInputStream(storageInput); - FileToRowsInterface fileToRowsInterface = FileToRowsInterface.forFile(storageInput.getReference(), inputStream); - ) - { - BulkLoadFileRow headerRow = fileToRowsInterface.next(); - - List bodyRows = new ArrayList<>(); - while(fileToRowsInterface.hasNext() && bodyRows.size() < 20) - { - bodyRows.add(fileToRowsInterface.next().toString()); - } - - runBackendStepOutput.addValue("header", headerRow.toString()); - runBackendStepOutput.addValue("body", JsonUtils.toPrettyJson(bodyRows)); - System.out.println("Done receiving file"); - } - catch(QException qe) - { - throw qe; - } - catch(Exception e) - { - throw new QException("Unhandled error in bulk insert extract step", e); - } - - } - -} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertReceiveValueMappingStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertReceiveValueMappingStep.java new file mode 100644 index 00000000..f9c17667 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertReceiveValueMappingStep.java @@ -0,0 +1,105 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert; + + +import java.io.Serializable; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import com.fasterxml.jackson.core.type.TypeReference; +import com.kingsrook.qqq.backend.core.actions.processes.BackendStep; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.logging.QLogger; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping.BulkInsertMapping; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadProfile; +import com.kingsrook.qqq.backend.core.utils.JsonUtils; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class BulkInsertReceiveValueMappingStep implements BackendStep +{ + private static final QLogger LOG = QLogger.getLogger(BulkInsertReceiveValueMappingStep.class); + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException + { + try + { + BulkInsertStepUtils.handleSavedBulkLoadProfileIdValue(runBackendStepInput, runBackendStepOutput); + + List fieldNamesToDoValueMapping = (List) runBackendStepInput.getValue("fieldNamesToDoValueMapping"); + Integer valueMappingFieldIndex = runBackendStepInput.getValueInteger("valueMappingFieldIndex"); + + String fieldName = fieldNamesToDoValueMapping.get(valueMappingFieldIndex); + + /////////////////////////////////////////////////////////////////// + // read process values - construct a bulkLoadProfile out of them // + /////////////////////////////////////////////////////////////////// + BulkLoadProfile bulkLoadProfile = BulkInsertStepUtils.getBulkLoadProfile(runBackendStepInput); + + ///////////////////////////////////////////////////////////////////////// + // put the list of bulk load profile into the process state - it's the // + // thing that the frontend will be looking at as the saved profile // + ///////////////////////////////////////////////////////////////////////// + runBackendStepOutput.addValue("bulkLoadProfile", bulkLoadProfile); + + /////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // get the bulkInsertMapping object from the process, creating a fieldNameToValueMapping map within it if needed // + /////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + BulkInsertMapping bulkInsertMapping = (BulkInsertMapping) runBackendStepOutput.getValue("bulkInsertMapping"); + Map> fieldNameToValueMapping = bulkInsertMapping.getFieldNameToValueMapping(); + if(fieldNameToValueMapping == null) + { + fieldNameToValueMapping = new HashMap<>(); + bulkInsertMapping.setFieldNameToValueMapping(fieldNameToValueMapping); + } + runBackendStepOutput.addValue("bulkInsertMapping", bulkInsertMapping); + + //////////////////////////////////////////////// + // put the mapped values into the mapping map // + //////////////////////////////////////////////// + Map mappedValues = JsonUtils.toObject(runBackendStepInput.getValueString("mappedValuesJSON"), new TypeReference<>() {}); + fieldNameToValueMapping.put(fieldName, mappedValues); + + ///////////////////////////////////////////////////////////////////////////////////////////////////////////// + // always return to the prepare-mapping step - as it will determine if it's time to break the loop or not. // + ///////////////////////////////////////////////////////////////////////////////////////////////////////////// + BulkInsertStepUtils.setNextStepPrepareValueMapping(runBackendStepOutput); + } + catch(Exception e) + { + LOG.warn("Error in bulk insert receive mapping", e); + throw new QException("Unhandled error in bulk insert receive mapping step", e); + } + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertStepUtils.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertStepUtils.java index e3dac892..3e4ffa40 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertStepUtils.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertStepUtils.java @@ -22,10 +22,22 @@ package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert; +import java.io.Serializable; import java.util.ArrayList; +import java.util.HashMap; +import com.kingsrook.qqq.backend.core.actions.tables.GetAction; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; import com.kingsrook.qqq.backend.core.model.actions.tables.storage.StorageInput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.savedbulkloadprofiles.SavedBulkLoadProfile; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadProfile; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadProfileField; +import com.kingsrook.qqq.backend.core.utils.ValueUtils; +import org.apache.commons.lang3.BooleanUtils; +import org.json.JSONArray; +import org.json.JSONObject; /******************************************************************************* @@ -55,4 +67,88 @@ public class BulkInsertStepUtils return (storageInput); } + + + /*************************************************************************** + ** + ***************************************************************************/ + public static void setNextStepStreamedETLPreview(RunBackendStepOutput runBackendStepOutput) + { + runBackendStepOutput.setOverrideLastStepName("receiveValueMapping"); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + public static void setNextStepPrepareValueMapping(RunBackendStepOutput runBackendStepOutput) + { + runBackendStepOutput.setOverrideLastStepName("receiveFileMapping"); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + public static BulkLoadProfile getBulkLoadProfile(RunBackendStepInput runBackendStepInput) + { + String version = runBackendStepInput.getValueString("version"); + if("v1".equals(version)) + { + String layout = runBackendStepInput.getValueString("layout"); + Boolean hasHeaderRow = runBackendStepInput.getValueBoolean("hasHeaderRow"); + + ArrayList fieldList = new ArrayList<>(); + + JSONArray array = new JSONArray(runBackendStepInput.getValueString("fieldListJSON")); + for(int i = 0; i < array.length(); i++) + { + JSONObject jsonObject = array.getJSONObject(i); + BulkLoadProfileField bulkLoadProfileField = new BulkLoadProfileField(); + fieldList.add(bulkLoadProfileField); + bulkLoadProfileField.setFieldName(jsonObject.optString("fieldName")); + bulkLoadProfileField.setColumnIndex(jsonObject.has("columnIndex") ? jsonObject.getInt("columnIndex") : null); + bulkLoadProfileField.setDefaultValue((Serializable) jsonObject.opt("defaultValue")); + bulkLoadProfileField.setDoValueMapping(jsonObject.optBoolean("doValueMapping")); + + if(BooleanUtils.isTrue(bulkLoadProfileField.getDoValueMapping()) && jsonObject.has("valueMappings")) + { + bulkLoadProfileField.setValueMappings(new HashMap<>()); + JSONObject valueMappingsJsonObject = jsonObject.getJSONObject("valueMappings"); + for(String fileValue : valueMappingsJsonObject.keySet()) + { + bulkLoadProfileField.getValueMappings().put(fileValue, ValueUtils.getValueAsString(valueMappingsJsonObject.get(fileValue))); + } + } + } + + BulkLoadProfile bulkLoadProfile = new BulkLoadProfile() + .withFieldList(fieldList) + .withHasHeaderRow(hasHeaderRow) + .withLayout(layout); + + return (bulkLoadProfile); + } + else + { + throw (new IllegalArgumentException("Unexpected version for bulk load profile: " + version)); + } + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + public static void handleSavedBulkLoadProfileIdValue(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException + { + Integer savedBulkLoadProfileId = runBackendStepInput.getValueInteger("savedBulkLoadProfileId"); + if(savedBulkLoadProfileId != null) + { + QRecord savedBulkLoadProfileRecord = GetAction.execute(SavedBulkLoadProfile.TABLE_NAME, savedBulkLoadProfileId); + runBackendStepOutput.addValue("savedBulkLoadProfileRecord", savedBulkLoadProfileRecord); + } + } } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTransformStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTransformStep.java index f4eff9a3..18c03671 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTransformStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTransformStep.java @@ -111,6 +111,11 @@ public class BulkInsertTransformStep extends AbstractTransformStep // since we're doing a unique key check in this class, we can tell the loadViaInsert step that it (rather, the InsertAction) doesn't need to re-do one. // ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// runBackendStepOutput.addValue(LoadViaInsertStep.FIELD_SKIP_UNIQUE_KEY_CHECK, true); + + /////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // make sure that if a saved profile was selected on a review screen, that the result screen knows about it. // + /////////////////////////////////////////////////////////////////////////////////////////////////////////////// + BulkInsertStepUtils.handleSavedBulkLoadProfileIdValue(runBackendStepInput, runBackendStepOutput); } @@ -121,8 +126,43 @@ public class BulkInsertTransformStep extends AbstractTransformStep @Override public void runOnePage(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException { - int rowsInThisPage = runBackendStepInput.getRecords().size(); - QTableMetaData table = QContext.getQInstance().getTable(runBackendStepInput.getTableName()); + int recordsInThisPage = runBackendStepInput.getRecords().size(); + QTableMetaData table = QContext.getQInstance().getTable(runBackendStepInput.getTableName()); + + // split the records w/o UK errors into those w/ e + List recordsWithoutAnyErrors = new ArrayList<>(); + List recordsWithSomeErrors = new ArrayList<>(); + for(QRecord record : runBackendStepInput.getRecords()) + { + if(CollectionUtils.nullSafeHasContents(record.getErrors())) + { + recordsWithSomeErrors.add(record); + } + else + { + recordsWithoutAnyErrors.add(record); + } + } + + ////////////////////////////////////////////////////////////////// + // propagate errors that came into this step out to the summary // + ////////////////////////////////////////////////////////////////// + if(!recordsWithSomeErrors.isEmpty()) + { + for(QRecord record : recordsWithSomeErrors) + { + String message = record.getErrors().get(0).getMessage(); + processSummaryWarningsAndErrorsRollup.addError(message, null); + } + } + + if(recordsWithoutAnyErrors.isEmpty()) + { + //////////////////////////////////////////////////////////////////////////////// + // skip th rest of this method if there aren't any records w/o errors in them // + //////////////////////////////////////////////////////////////////////////////// + this.rowsProcessed += recordsInThisPage; + } ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////// // set up an insert-input, which will be used as input to the pre-customizer as well as for additional validations // @@ -130,7 +170,7 @@ public class BulkInsertTransformStep extends AbstractTransformStep InsertInput insertInput = new InsertInput(); insertInput.setInputSource(QInputSource.USER); insertInput.setTableName(runBackendStepInput.getTableName()); - insertInput.setRecords(runBackendStepInput.getRecords()); + insertInput.setRecords(recordsWithoutAnyErrors); insertInput.setSkipUniqueKeyCheck(true); ////////////////////////////////////////////////////////////////////// @@ -145,7 +185,7 @@ public class BulkInsertTransformStep extends AbstractTransformStep AbstractPreInsertCustomizer.WhenToRun whenToRun = preInsertCustomizer.get().whenToRunPreInsert(insertInput, true); if(WhenToRun.BEFORE_ALL_VALIDATIONS.equals(whenToRun) || WhenToRun.BEFORE_UNIQUE_KEY_CHECKS.equals(whenToRun)) { - List recordsAfterCustomizer = preInsertCustomizer.get().preInsert(insertInput, runBackendStepInput.getRecords(), true); + List recordsAfterCustomizer = preInsertCustomizer.get().preInsert(insertInput, recordsWithoutAnyErrors, true); runBackendStepInput.setRecords(recordsAfterCustomizer); /////////////////////////////////////////////////////////////////////////////////////// @@ -159,13 +199,14 @@ public class BulkInsertTransformStep extends AbstractTransformStep List uniqueKeys = CollectionUtils.nonNullList(table.getUniqueKeys()); for(UniqueKey uniqueKey : uniqueKeys) { - existingKeys.put(uniqueKey, UniqueKeyHelper.getExistingKeys(null, table, runBackendStepInput.getRecords(), uniqueKey).keySet()); + existingKeys.put(uniqueKey, UniqueKeyHelper.getExistingKeys(null, table, recordsWithoutAnyErrors, uniqueKey).keySet()); ukErrorSummaries.computeIfAbsent(uniqueKey, x -> new ProcessSummaryLineWithUKSampleValues(Status.ERROR)); } ///////////////////////////////////////////////////////////////////////////////////////////////////////////////// // on the validate step, we haven't read the full file, so we don't know how many rows there are - thus // // record count is null, and the ValidateStep won't be setting status counters - so - do it here in that case. // + // todo - move this up (before the early return?) // ///////////////////////////////////////////////////////////////////////////////////////////////////////////////// if(runBackendStepInput.getStepName().equals(StreamedETLWithFrontendProcess.STEP_NAME_VALIDATE)) { @@ -187,12 +228,12 @@ public class BulkInsertTransformStep extends AbstractTransformStep // Note, we want to do our own UK checking here, even though InsertAction also tries to do it, because InsertAction // // will only be getting the records in pages, but in here, we'll track UK's across pages!! // ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - List recordsWithoutUkErrors = getRecordsWithoutUniqueKeyErrors(runBackendStepInput, existingKeys, uniqueKeys, table); + List recordsWithoutUkErrors = getRecordsWithoutUniqueKeyErrors(recordsWithoutAnyErrors, existingKeys, uniqueKeys, table); ///////////////////////////////////////////////////////////////////////////////// // run all validation from the insert action - in Preview mode (boolean param) // ///////////////////////////////////////////////////////////////////////////////// - insertInput.setRecords(recordsWithoutUkErrors); + insertInput.setRecords(recordsWithoutAnyErrors); InsertAction insertAction = new InsertAction(); insertAction.performValidations(insertInput, true); List validationResultRecords = insertInput.getRecords(); @@ -222,8 +263,7 @@ public class BulkInsertTransformStep extends AbstractTransformStep } runBackendStepOutput.setRecords(outputRecords); - - this.rowsProcessed += rowsInThisPage; + this.rowsProcessed += recordsInThisPage; } @@ -231,7 +271,7 @@ public class BulkInsertTransformStep extends AbstractTransformStep /******************************************************************************* ** *******************************************************************************/ - private List getRecordsWithoutUniqueKeyErrors(RunBackendStepInput runBackendStepInput, Map>> existingKeys, List uniqueKeys, QTableMetaData table) + private List getRecordsWithoutUniqueKeyErrors(List records, Map>> existingKeys, List uniqueKeys, QTableMetaData table) { //////////////////////////////////////////////////// // if there are no UK's, proceed with all records // @@ -239,7 +279,7 @@ public class BulkInsertTransformStep extends AbstractTransformStep List recordsWithoutUkErrors = new ArrayList<>(); if(existingKeys.isEmpty()) { - recordsWithoutUkErrors.addAll(runBackendStepInput.getRecords()); + recordsWithoutUkErrors.addAll(records); } else { @@ -255,7 +295,7 @@ public class BulkInsertTransformStep extends AbstractTransformStep // else, get each records keys and see if it already exists or not // // also, build a set of keys we've seen (within this page (or overall?)) // /////////////////////////////////////////////////////////////////////////// - for(QRecord record : runBackendStepInput.getRecords()) + for(QRecord record : records) { if(CollectionUtils.nullSafeHasContents(record.getErrors())) { @@ -333,8 +373,8 @@ public class BulkInsertTransformStep extends AbstractTransformStep ukErrorSummary .withMessageSuffix(" inserted, because of duplicate values in a unique key on the fields (" + uniqueKey.getDescription(table) + "), with values" - + (ukErrorSummary.areThereMoreSampleValues ? " such as: " : ": ") - + StringUtils.joinWithCommasAndAnd(new ArrayList<>(ukErrorSummary.sampleValues))) + + (ukErrorSummary.areThereMoreSampleValues ? " such as: " : ": ") + + StringUtils.joinWithCommasAndAnd(new ArrayList<>(ukErrorSummary.sampleValues))) .withSingularFutureMessage(" record will not be") .withPluralFutureMessage(" records will not be") diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertV2ExtractStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertV2ExtractStep.java index 2e49e48a..efacca0f 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertV2ExtractStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertV2ExtractStep.java @@ -34,6 +34,7 @@ import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.FileToRowsInterface; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping.BulkInsertMapping; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping.RowsToRecordInterface; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow; import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.AbstractExtractStep; diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/AbstractIteratorBasedFileToRows.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/AbstractIteratorBasedFileToRows.java index 383dfa21..be80c82a 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/AbstractIteratorBasedFileToRows.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/AbstractIteratorBasedFileToRows.java @@ -23,7 +23,7 @@ package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.fil import java.util.Iterator; -import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkLoadFileRow; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow; /******************************************************************************* diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/CsvFileToRows.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/CsvFileToRows.java index f39f4d45..7a58ddef 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/CsvFileToRows.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/CsvFileToRows.java @@ -28,7 +28,7 @@ import java.io.InputStream; import java.io.InputStreamReader; import java.io.Serializable; import com.kingsrook.qqq.backend.core.exceptions.QException; -import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkLoadFileRow; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow; import org.apache.commons.csv.CSVFormat; import org.apache.commons.csv.CSVParser; import org.apache.commons.csv.CSVRecord; diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/FileToRowsInterface.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/FileToRowsInterface.java index d2d6c78a..9d02ee0f 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/FileToRowsInterface.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/FileToRowsInterface.java @@ -27,7 +27,7 @@ import java.util.Iterator; import java.util.Locale; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.exceptions.QUserFacingException; -import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkLoadFileRow; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow; /******************************************************************************* diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/XlsxFileToRows.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/XlsxFileToRows.java index 21c9d928..a6e336de 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/XlsxFileToRows.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/XlsxFileToRows.java @@ -27,7 +27,7 @@ import java.io.InputStream; import java.io.Serializable; import java.util.stream.Stream; import com.kingsrook.qqq.backend.core.exceptions.QException; -import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkLoadFileRow; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow; import org.dhatim.fastexcel.reader.ReadableWorkbook; import org.dhatim.fastexcel.reader.Sheet; diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkInsertMapping.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkInsertMapping.java index d95d48e4..fef870b1 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkInsertMapping.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkInsertMapping.java @@ -32,8 +32,9 @@ import java.util.function.Supplier; import com.fasterxml.jackson.annotation.JsonIgnore; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.PossibleValueEnum; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; -import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkLoadFileRow; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow; import com.kingsrook.qqq.backend.core.utils.CollectionUtils; import com.kingsrook.qqq.backend.core.utils.Pair; import com.kingsrook.qqq.backend.core.utils.StringUtils; @@ -62,8 +63,10 @@ public class BulkInsertMapping implements Serializable private Map fieldNameToDefaultValueMap = new HashMap<>(); private Map> fieldNameToValueMapping = new HashMap<>(); - private Map> tallLayoutGroupByIndexMap = new HashMap<>(); - private List mappedAssociations = new ArrayList<>(); + private Map> tallLayoutGroupByIndexMap = new HashMap<>(); + private Map wideLayoutMapping = new HashMap<>(); + + private List mappedAssociations = new ArrayList<>(); private Memoization, Boolean> shouldProcessFieldForTable = new Memoization<>(); @@ -72,11 +75,11 @@ public class BulkInsertMapping implements Serializable /*************************************************************************** ** ***************************************************************************/ - public enum Layout + public enum Layout implements PossibleValueEnum { FLAT(FlatRowsToRecord::new), TALL(TallRowsToRecord::new), - WIDE(WideRowsToRecord::new); + WIDE(WideRowsToRecordWithExplicitMapping::new); /*************************************************************************** @@ -95,6 +98,7 @@ public class BulkInsertMapping implements Serializable } + /*************************************************************************** ** ***************************************************************************/ @@ -102,6 +106,28 @@ public class BulkInsertMapping implements Serializable { return (supplier.get()); } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public String getPossibleValueId() + { + return name(); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public String getPossibleValueLabel() + { + return StringUtils.ucFirst(name().toLowerCase()); + } } @@ -500,4 +526,35 @@ public class BulkInsertMapping implements Serializable return (this); } + + + /******************************************************************************* + ** Getter for wideLayoutMapping + *******************************************************************************/ + public Map getWideLayoutMapping() + { + return (this.wideLayoutMapping); + } + + + + /******************************************************************************* + ** Setter for wideLayoutMapping + *******************************************************************************/ + public void setWideLayoutMapping(Map wideLayoutMapping) + { + this.wideLayoutMapping = wideLayoutMapping; + } + + + + /******************************************************************************* + ** Fluent setter for wideLayoutMapping + *******************************************************************************/ + public BulkInsertMapping withWideLayoutMapping(Map wideLayoutMapping) + { + this.wideLayoutMapping = wideLayoutMapping; + return (this); + } + } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkInsertWideLayoutMapping.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkInsertWideLayoutMapping.java new file mode 100644 index 00000000..ad0cce0f --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkInsertWideLayoutMapping.java @@ -0,0 +1,216 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping; + + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow; +import com.kingsrook.qqq.backend.core.utils.ValueUtils; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class BulkInsertWideLayoutMapping +{ + private List childRecordMappings; + + + + /******************************************************************************* + ** Constructor + ** + *******************************************************************************/ + public BulkInsertWideLayoutMapping(List childRecordMappings) + { + this.childRecordMappings = childRecordMappings; + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + public static class ChildRecordMapping + { + Map fieldNameToHeaderNameMaps; + Map associationNameToChildRecordMappingMap; + + + + /******************************************************************************* + ** Constructor + ** + *******************************************************************************/ + public ChildRecordMapping(Map fieldNameToHeaderNameMaps) + { + this.fieldNameToHeaderNameMaps = fieldNameToHeaderNameMaps; + } + + + + /******************************************************************************* + ** Constructor + ** + *******************************************************************************/ + public ChildRecordMapping(Map fieldNameToHeaderNameMaps, Map associationNameToChildRecordMappingMap) + { + this.fieldNameToHeaderNameMaps = fieldNameToHeaderNameMaps; + this.associationNameToChildRecordMappingMap = associationNameToChildRecordMappingMap; + } + + + + /******************************************************************************* + ** Getter for fieldNameToHeaderNameMaps + *******************************************************************************/ + public Map getFieldNameToHeaderNameMaps() + { + return (this.fieldNameToHeaderNameMaps); + } + + + + /******************************************************************************* + ** Setter for fieldNameToHeaderNameMaps + *******************************************************************************/ + public void setFieldNameToHeaderNameMaps(Map fieldNameToHeaderNameMaps) + { + this.fieldNameToHeaderNameMaps = fieldNameToHeaderNameMaps; + } + + + + /******************************************************************************* + ** Fluent setter for fieldNameToHeaderNameMaps + *******************************************************************************/ + public ChildRecordMapping withFieldNameToHeaderNameMaps(Map fieldNameToHeaderNameMaps) + { + this.fieldNameToHeaderNameMaps = fieldNameToHeaderNameMaps; + return (this); + } + + + + /******************************************************************************* + ** Getter for associationNameToChildRecordMappingMap + *******************************************************************************/ + public Map getAssociationNameToChildRecordMappingMap() + { + return (this.associationNameToChildRecordMappingMap); + } + + + + /******************************************************************************* + ** Setter for associationNameToChildRecordMappingMap + *******************************************************************************/ + public void setAssociationNameToChildRecordMappingMap(Map associationNameToChildRecordMappingMap) + { + this.associationNameToChildRecordMappingMap = associationNameToChildRecordMappingMap; + } + + + + /******************************************************************************* + ** Fluent setter for associationNameToChildRecordMappingMap + *******************************************************************************/ + public ChildRecordMapping withAssociationNameToChildRecordMappingMap(Map associationNameToChildRecordMappingMap) + { + this.associationNameToChildRecordMappingMap = associationNameToChildRecordMappingMap; + return (this); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + public Map getFieldIndexes(BulkLoadFileRow headerRow) + { + // todo memoize or otherwise don't recompute + Map rs = new HashMap<>(); + + //////////////////////////////////////////////////////// + // for the current file, map header values to indexes // + //////////////////////////////////////////////////////// + Map headerToIndexMap = new HashMap<>(); + for(int i = 0; i < headerRow.size(); i++) + { + String headerValue = ValueUtils.getValueAsString(headerRow.getValue(i)); + headerToIndexMap.put(headerValue, i); + } + + ///////////////////////////////////////////////////////////////////////////////////////////////////////// + // loop over fields - finding what header name they are mapped to - then what index that header is at. // + ///////////////////////////////////////////////////////////////////////////////////////////////////////// + for(Map.Entry entry : fieldNameToHeaderNameMaps.entrySet()) + { + String headerName = entry.getValue(); + if(headerName != null) + { + Integer headerIndex = headerToIndexMap.get(headerName); + if(headerIndex != null) + { + rs.put(entry.getKey(), headerIndex); + } + } + } + + return (rs); + } + } + + + + /******************************************************************************* + ** Getter for childRecordMappings + *******************************************************************************/ + public List getChildRecordMappings() + { + return (this.childRecordMappings); + } + + + + /******************************************************************************* + ** Setter for childRecordMappings + *******************************************************************************/ + public void setChildRecordMappings(List childRecordMappings) + { + this.childRecordMappings = childRecordMappings; + } + + + + /******************************************************************************* + ** Fluent setter for childRecordMappings + *******************************************************************************/ + public BulkInsertWideLayoutMapping withChildRecordMappings(List childRecordMappings) + { + this.childRecordMappings = childRecordMappings; + return (this); + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/FlatRowsToRecord.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/FlatRowsToRecord.java index ac98adf4..44702312 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/FlatRowsToRecord.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/FlatRowsToRecord.java @@ -30,8 +30,8 @@ import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; -import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkLoadFileRow; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.FileToRowsInterface; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow; /******************************************************************************* @@ -62,13 +62,13 @@ public class FlatRowsToRecord implements RowsToRecordInterface for(QFieldMetaData field : table.getFields().values()) { - setValueOrDefault(record, field.getName(), null, mapping, row, fieldIndexes.get(field.getName())); + setValueOrDefault(record, field, null, mapping, row, fieldIndexes.get(field.getName())); } rs.add(record); } - ValueMapper.valueMapping(rs, mapping); + ValueMapper.valueMapping(rs, mapping, table); return (rs); } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/RowsToRecordInterface.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/RowsToRecordInterface.java index 91481980..25aa6543 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/RowsToRecordInterface.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/RowsToRecordInterface.java @@ -26,8 +26,10 @@ import java.io.Serializable; import java.util.List; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.data.QRecord; -import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkLoadFileRow; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.FileToRowsInterface; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow; import com.kingsrook.qqq.backend.core.utils.StringUtils; @@ -44,26 +46,50 @@ public interface RowsToRecordInterface /*************************************************************************** - ** + ** returns true if value from row was used, else false. ***************************************************************************/ - default void setValueOrDefault(QRecord record, String fieldName, String associationNameChain, BulkInsertMapping mapping, BulkLoadFileRow row, Integer index) + default boolean setValueOrDefault(QRecord record, QFieldMetaData field, String associationNameChain, BulkInsertMapping mapping, BulkLoadFileRow row, Integer index) { - String fieldNameWithAssociationPrefix = StringUtils.hasContent(associationNameChain) ? associationNameChain + "." + fieldName : fieldName; + String fieldName = field.getName(); + QFieldType type = field.getType(); + + boolean valueFromRowWasUsed = false; + String fieldNameWithAssociationPrefix = StringUtils.hasContent(associationNameChain) ? associationNameChain + "." + fieldName : fieldName; Serializable value = null; if(index != null && row != null) { value = row.getValueElseNull(index); + if(value != null && !"".equals(value)) + { + valueFromRowWasUsed = true; + } } else if(mapping.getFieldNameToDefaultValueMap().containsKey(fieldNameWithAssociationPrefix)) { value = mapping.getFieldNameToDefaultValueMap().get(fieldNameWithAssociationPrefix); } + /* note - moving this to ValueMapper... + if(value != null) + { + try + { + value = ValueUtils.getValueAsFieldType(type, value); + } + catch(Exception e) + { + record.addError(new BadInputStatusMessage("Value [" + value + "] for field [" + field.getLabel() + "] could not be converted to type [" + type + "]")); + } + } + */ + if(value != null) { record.setValue(fieldName, value); } + + return (valueFromRowWasUsed); } } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/TallRowsToRecord.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/TallRowsToRecord.java index 1dbccd92..690f705d 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/TallRowsToRecord.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/TallRowsToRecord.java @@ -35,8 +35,8 @@ import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.Association; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; -import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkLoadFileRow; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.FileToRowsInterface; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow; import com.kingsrook.qqq.backend.core.utils.CollectionUtils; import com.kingsrook.qqq.backend.core.utils.Pair; import com.kingsrook.qqq.backend.core.utils.StringUtils; @@ -75,7 +75,12 @@ public class TallRowsToRecord implements RowsToRecordInterface { BulkLoadFileRow row = fileToRowsInterface.next(); - List groupByIndexes = mapping.getTallLayoutGroupByIndexMap().get(table.getName()); + List groupByIndexes = mapping.getTallLayoutGroupByIndexMap().get(table.getName()); + if(CollectionUtils.nullSafeIsEmpty(groupByIndexes)) + { + groupByIndexes = groupByAllIndexesFromTable(mapping, table, headerRow, null); + } + List rowGroupByValues = getGroupByValues(row, groupByIndexes); if(rowGroupByValues == null) { @@ -126,13 +131,24 @@ public class TallRowsToRecord implements RowsToRecordInterface rs.add(makeRecordFromRows(table, associationNameChain, mapping, headerRow, rowsForCurrentRecord)); } - ValueMapper.valueMapping(rs, mapping); + ValueMapper.valueMapping(rs, mapping, table); return (rs); } + /*************************************************************************** + ** + ***************************************************************************/ + private List groupByAllIndexesFromTable(BulkInsertMapping mapping, QTableMetaData table, BulkLoadFileRow headerRow, String name) throws QException + { + Map fieldIndexes = mapping.getFieldIndexes(table, name, headerRow); + return new ArrayList<>(fieldIndexes.values()); + } + + + /*************************************************************************** ** ***************************************************************************/ @@ -148,7 +164,7 @@ public class TallRowsToRecord implements RowsToRecordInterface BulkLoadFileRow row = rows.get(0); for(QFieldMetaData field : table.getFields().values()) { - setValueOrDefault(record, field.getName(), associationNameChain, mapping, row, fieldIndexes.get(field.getName())); + setValueOrDefault(record, field, associationNameChain, mapping, row, fieldIndexes.get(field.getName())); } ///////////////////////////// @@ -230,7 +246,8 @@ public class TallRowsToRecord implements RowsToRecordInterface List groupByIndexes = mapping.getTallLayoutGroupByIndexMap().get(associationNameChainForRecursiveCalls); if(CollectionUtils.nullSafeIsEmpty(groupByIndexes)) { - throw (new QException("Missing group-by-index(es) for association: " + associationNameChainForRecursiveCalls)); + groupByIndexes = groupByAllIndexesFromTable(mapping, table, headerRow, associationNameChainForRecursiveCalls); + // throw (new QException("Missing group-by-index(es) for association: " + associationNameChainForRecursiveCalls)); } List rowGroupByValues = getGroupByValues(row, groupByIndexes); diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/ValueMapper.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/ValueMapper.java index 09e074d3..0f1e8cfc 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/ValueMapper.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/ValueMapper.java @@ -25,9 +25,19 @@ package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.map import java.io.Serializable; import java.util.List; import java.util.Map; +import java.util.Optional; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.logging.QLogger; import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; +import com.kingsrook.qqq.backend.core.model.metadata.tables.Association; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.model.statusmessages.BadInputStatusMessage; import com.kingsrook.qqq.backend.core.utils.CollectionUtils; import com.kingsrook.qqq.backend.core.utils.StringUtils; +import com.kingsrook.qqq.backend.core.utils.ValueUtils; /******************************************************************************* @@ -35,12 +45,16 @@ import com.kingsrook.qqq.backend.core.utils.StringUtils; *******************************************************************************/ public class ValueMapper { + private static final QLogger LOG = QLogger.getLogger(ValueMapper.class); + + + /*************************************************************************** ** ***************************************************************************/ - public static void valueMapping(List records, BulkInsertMapping mapping) + public static void valueMapping(List records, BulkInsertMapping mapping, QTableMetaData table) throws QException { - valueMapping(records, mapping, null); + valueMapping(records, mapping, table, null); } @@ -48,7 +62,7 @@ public class ValueMapper /*************************************************************************** ** ***************************************************************************/ - public static void valueMapping(List records, BulkInsertMapping mapping, String associationNameChain) + private static void valueMapping(List records, BulkInsertMapping mapping, QTableMetaData table, String associationNameChain) throws QException { if(CollectionUtils.nullSafeIsEmpty(records)) { @@ -58,20 +72,58 @@ public class ValueMapper Map> mappingForTable = mapping.getFieldNameToValueMappingForTable(associationNameChain); for(QRecord record : records) { - for(Map.Entry> entry : mappingForTable.entrySet()) + for(Map.Entry valueEntry : record.getValues().entrySet()) { - String fieldName = entry.getKey(); - Map map = entry.getValue(); - String value = record.getValueString(fieldName); - if(value != null && map.containsKey(value)) + QFieldMetaData field = table.getField(valueEntry.getKey()); + Serializable value = valueEntry.getValue(); + + /////////////////// + // value mappin' // + /////////////////// + if(mappingForTable.containsKey(field.getName()) && value != null) { - record.setValue(fieldName, map.get(value)); + Serializable mappedValue = mappingForTable.get(field.getName()).get(ValueUtils.getValueAsString(value)); + if(mappedValue != null) + { + value = mappedValue; + } } + + ///////////////////// + // type convertin' // + ///////////////////// + if(value != null) + { + QFieldType type = field.getType(); + try + { + value = ValueUtils.getValueAsFieldType(type, value); + } + catch(Exception e) + { + record.addError(new BadInputStatusMessage("Value [" + value + "] for field [" + field.getLabel() + "] could not be converted to type [" + type + "]")); + } + } + + record.setValue(field.getName(), value); } + ////////////////////////////////////// + // recursively process associations // + ////////////////////////////////////// for(Map.Entry> entry : record.getAssociatedRecords().entrySet()) { - valueMapping(entry.getValue(), mapping, StringUtils.hasContent(associationNameChain) ? associationNameChain + "." + entry.getKey() : entry.getKey()); + String associationName = entry.getKey(); + Optional association = table.getAssociations().stream().filter(a -> a.getName().equals(associationName)).findFirst(); + if(association.isPresent()) + { + QTableMetaData associatedTable = QContext.getQInstance().getTable(association.get().getAssociatedTableName()); + valueMapping(entry.getValue(), mapping, associatedTable, StringUtils.hasContent(associationNameChain) ? associationNameChain + "." + associationName : associationName); + } + else + { + throw new QException("Missing association [" + associationName + "] on table [" + table.getName() + "]"); + } } } } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithExplicitMapping.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithExplicitMapping.java new file mode 100644 index 00000000..adc67ec7 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithExplicitMapping.java @@ -0,0 +1,260 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping; + + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.tables.Association; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.FileToRowsInterface; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow; +import com.kingsrook.qqq.backend.core.utils.CollectionUtils; +import com.kingsrook.qqq.backend.core.utils.Pair; +import com.kingsrook.qqq.backend.core.utils.StringUtils; +import com.kingsrook.qqq.backend.core.utils.memoization.Memoization; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class WideRowsToRecordWithExplicitMapping implements RowsToRecordInterface +{ + private Memoization, Boolean> shouldProcesssAssociationMemoization = new Memoization<>(); + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public List nextPage(FileToRowsInterface fileToRowsInterface, BulkLoadFileRow headerRow, BulkInsertMapping mapping, Integer limit) throws QException + { + QTableMetaData table = QContext.getQInstance().getTable(mapping.getTableName()); + if(table == null) + { + throw (new QException("Table [" + mapping.getTableName() + "] was not found in the Instance")); + } + + List rs = new ArrayList<>(); + + Map fieldIndexes = mapping.getFieldIndexes(table, null, headerRow); + + while(fileToRowsInterface.hasNext() && rs.size() < limit) + { + BulkLoadFileRow row = fileToRowsInterface.next(); + QRecord record = new QRecord(); + + for(QFieldMetaData field : table.getFields().values()) + { + setValueOrDefault(record, field, null, mapping, row, fieldIndexes.get(field.getName())); + } + + processAssociations(mapping.getWideLayoutMapping(), "", headerRow, mapping, table, row, record); + + rs.add(record); + } + + ValueMapper.valueMapping(rs, mapping, table); + + return (rs); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private void processAssociations(Map mappingMap, String associationNameChain, BulkLoadFileRow headerRow, BulkInsertMapping mapping, QTableMetaData table, BulkLoadFileRow row, QRecord record) throws QException + { + for(Map.Entry entry : CollectionUtils.nonNullMap(mappingMap).entrySet()) + { + String associationName = entry.getKey(); + BulkInsertWideLayoutMapping bulkInsertWideLayoutMapping = entry.getValue(); + + Optional association = table.getAssociations().stream().filter(a -> a.getName().equals(associationName)).findFirst(); + if(association.isEmpty()) + { + throw (new QException("Couldn't find association: " + associationName + " under table: " + table.getName())); + } + + QTableMetaData associatedTable = QContext.getQInstance().getTable(association.get().getAssociatedTableName()); + + String subChain = StringUtils.hasContent(associationNameChain) ? associationNameChain + "." + associationName: associationName; + + for(BulkInsertWideLayoutMapping.ChildRecordMapping childRecordMapping : bulkInsertWideLayoutMapping.getChildRecordMappings()) + { + QRecord associatedRecord = processAssociation(associatedTable, subChain, childRecordMapping, mapping, row, headerRow); + if(associatedRecord != null) + { + record.withAssociatedRecord(associationName, associatedRecord); + } + } + } + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private QRecord processAssociation(QTableMetaData table, String associationNameChain, BulkInsertWideLayoutMapping.ChildRecordMapping childRecordMapping, BulkInsertMapping mapping, BulkLoadFileRow row, BulkLoadFileRow headerRow) throws QException + { + Map fieldIndexes = childRecordMapping.getFieldIndexes(headerRow); + + QRecord associatedRecord = new QRecord(); + boolean usedAnyValuesFromRow = false; + + for(QFieldMetaData field : table.getFields().values()) + { + boolean valueFromRowWasUsed = setValueOrDefault(associatedRecord, field, associationNameChain, mapping, row, fieldIndexes.get(field.getName())); + usedAnyValuesFromRow |= valueFromRowWasUsed; + } + + if(usedAnyValuesFromRow) + { + processAssociations(childRecordMapping.getAssociationNameToChildRecordMappingMap(), associationNameChain, headerRow, mapping, table, row, associatedRecord); + return (associatedRecord); + } + else + { + return (null); + } + } + + // /*************************************************************************** + // ** + // ***************************************************************************/ + // private List processAssociationV2(String associationName, String associationNameChain, QTableMetaData table, BulkInsertMapping mapping, BulkLoadFileRow row, BulkLoadFileRow headerRow, QRecord record, int startIndex, int endIndex) throws QException + // { + // List rs = new ArrayList<>(); + + // Map fieldNameToHeaderNameMapForThisAssociation = new HashMap<>(); + // for(Map.Entry entry : mapping.getFieldNameToHeaderNameMap().entrySet()) + // { + // if(entry.getKey().startsWith(associationName + ".")) + // { + // String fieldName = entry.getKey().substring(associationName.length() + 1); + + // ////////////////////////////////////////////////////////////////////////// + // // make sure the name here is for this table - not a sub-table under it // + // ////////////////////////////////////////////////////////////////////////// + // if(!fieldName.contains(".")) + // { + // fieldNameToHeaderNameMapForThisAssociation.put(fieldName, entry.getValue()); + // } + // } + // } + + // ///////////////////////////////////////////////////////////////////// + // // loop over the length of the record, building associated records // + // ///////////////////////////////////////////////////////////////////// + // QRecord associatedRecord = new QRecord(); + // Set processedFieldNames = new HashSet<>(); + // boolean gotAnyValues = false; + // int subStartIndex = -1; + + // for(int i = startIndex; i < endIndex; i++) + // { + // String headerValue = ValueUtils.getValueAsString(headerRow.getValue(i)); + + // for(Map.Entry entry : fieldNameToHeaderNameMapForThisAssociation.entrySet()) + // { + // if(headerValue.equals(entry.getValue()) || headerValue.matches(entry.getValue() + " ?\\d+")) + // { + // /////////////////////////////////////////////// + // // ok - this is a value for this association // + // /////////////////////////////////////////////// + // if(subStartIndex == -1) + // { + // subStartIndex = i; + // } + + // String fieldName = entry.getKey(); + // if(processedFieldNames.contains(fieldName)) + // { + // ///////////////////////////////////////////////// + // // this means we're starting a new sub-record! // + // ///////////////////////////////////////////////// + // if(gotAnyValues) + // { + // addDefaultValuesToAssociatedRecord(processedFieldNames, table, associatedRecord, mapping, associationName); + // processAssociations(associationName, headerRow, mapping, table, row, associatedRecord, subStartIndex, i); + // rs.add(associatedRecord); + // } + + // associatedRecord = new QRecord(); + // processedFieldNames = new HashSet<>(); + // gotAnyValues = false; + // subStartIndex = i + 1; + // } + + // processedFieldNames.add(fieldName); + + // Serializable value = row.getValueElseNull(i); + // if(value != null && !"".equals(value)) + // { + // gotAnyValues = true; + // } + + // setValueOrDefault(associatedRecord, fieldName, associationName, mapping, row, i); + // } + // } + // } + + // //////////////////////// + // // handle final value // + // //////////////////////// + // if(gotAnyValues) + // { + // addDefaultValuesToAssociatedRecord(processedFieldNames, table, associatedRecord, mapping, associationName); + // processAssociations(associationName, headerRow, mapping, table, row, associatedRecord, subStartIndex, endIndex); + // rs.add(associatedRecord); + // } + + // return (rs); + // } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private void addDefaultValuesToAssociatedRecord(Set processedFieldNames, QTableMetaData table, QRecord associatedRecord, BulkInsertMapping mapping, String associationNameChain) + { + for(QFieldMetaData field : table.getFields().values()) + { + if(!processedFieldNames.contains(field.getName())) + { + setValueOrDefault(associatedRecord, field, associationNameChain, mapping, null, null); + } + } + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecord.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithSpreadMapping.java similarity index 75% rename from qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecord.java rename to qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithSpreadMapping.java index 40e37071..f87fa17b 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecord.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithSpreadMapping.java @@ -37,8 +37,8 @@ import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.Association; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; -import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkLoadFileRow; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.FileToRowsInterface; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow; import com.kingsrook.qqq.backend.core.utils.Pair; import com.kingsrook.qqq.backend.core.utils.StringUtils; import com.kingsrook.qqq.backend.core.utils.ValueUtils; @@ -48,7 +48,7 @@ import com.kingsrook.qqq.backend.core.utils.memoization.Memoization; /******************************************************************************* ** *******************************************************************************/ -public class WideRowsToRecord implements RowsToRecordInterface +public class WideRowsToRecordWithSpreadMapping implements RowsToRecordInterface { private Memoization, Boolean> shouldProcesssAssociationMemoization = new Memoization<>(); @@ -77,7 +77,7 @@ public class WideRowsToRecord implements RowsToRecordInterface for(QFieldMetaData field : table.getFields().values()) { - setValueOrDefault(record, field.getName(), null, mapping, row, fieldIndexes.get(field.getName())); + setValueOrDefault(record, field, null, mapping, row, fieldIndexes.get(field.getName())); } processAssociations("", headerRow, mapping, table, row, record, 0, headerRow.size()); @@ -85,7 +85,7 @@ public class WideRowsToRecord implements RowsToRecordInterface rs.add(record); } - ValueMapper.valueMapping(rs, mapping); + ValueMapper.valueMapping(rs, mapping, table); return (rs); } @@ -199,7 +199,7 @@ public class WideRowsToRecord implements RowsToRecordInterface gotAnyValues = true; } - setValueOrDefault(associatedRecord, fieldName, associationName, mapping, row, i); + setValueOrDefault(associatedRecord, table.getField(fieldName), associationName, mapping, row, i); } } } @@ -228,77 +228,11 @@ public class WideRowsToRecord implements RowsToRecordInterface { if(!processedFieldNames.contains(field.getName())) { - setValueOrDefault(associatedRecord, field.getName(), associationNameChain, mapping, null, null); + setValueOrDefault(associatedRecord, field, associationNameChain, mapping, null, null); } } } - /*************************************************************************** - ** - ***************************************************************************/ - // private List processAssociation(String associationName, String associationNameChain, QTableMetaData table, BulkInsertMapping mapping, Row row, Row headerRow, QRecord record) throws QException - // { - // List rs = new ArrayList<>(); - // String associationNameChainForRecursiveCalls = associationName; - - // Map fieldNameToHeaderNameMapForThisAssociation = new HashMap<>(); - // for(Map.Entry entry : mapping.getFieldNameToHeaderNameMap().entrySet()) - // { - // if(entry.getKey().startsWith(associationNameChainForRecursiveCalls + ".")) - // { - // fieldNameToHeaderNameMapForThisAssociation.put(entry.getKey().substring(associationNameChainForRecursiveCalls.length() + 1), entry.getValue()); - // } - // } - - // Map> indexes = new HashMap<>(); - // for(int i = 0; i < headerRow.size(); i++) - // { - // String headerValue = ValueUtils.getValueAsString(headerRow.getValue(i)); - // for(Map.Entry entry : fieldNameToHeaderNameMapForThisAssociation.entrySet()) - // { - // if(headerValue.equals(entry.getValue()) || headerValue.matches(entry.getValue() + " ?\\d+")) - // { - // indexes.computeIfAbsent(entry.getKey(), k -> new ArrayList<>()).add(i); - // } - // } - // } - - // int maxIndex = indexes.values().stream().map(l -> l.size()).max(Integer::compareTo).orElse(0); - - // ////////////////////////////////////////////////////// - // // figure out how many sub-rows we'll be processing // - // ////////////////////////////////////////////////////// - // for(int i = 0; i < maxIndex; i++) - // { - // QRecord associatedRecord = new QRecord(); - // boolean gotAnyValues = false; - - // for(Map.Entry entry : fieldNameToHeaderNameMapForThisAssociation.entrySet()) - // { - // String fieldName = entry.getKey(); - // if(indexes.containsKey(fieldName) && indexes.get(fieldName).size() > i) - // { - // Integer index = indexes.get(fieldName).get(i); - // Serializable value = row.getValueElseNull(index); - // if(value != null && !"".equals(value)) - // { - // gotAnyValues = true; - // } - - // setValueOrDefault(associatedRecord, fieldName, mapping, row, index); - // } - // } - - // if(gotAnyValues) - // { - // processAssociations(associationNameChainForRecursiveCalls, headerRow, mapping, table, row, associatedRecord, 0, headerRow.size()); - // rs.add(associatedRecord); - // } - // } - - // return (rs); - // } - /*************************************************************************** diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkLoadFileRow.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/model/BulkLoadFileRow.java similarity index 99% rename from qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkLoadFileRow.java rename to qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/model/BulkLoadFileRow.java index 487f606a..23b80a56 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkLoadFileRow.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/model/BulkLoadFileRow.java @@ -19,7 +19,7 @@ * along with this program. If not, see . */ -package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert; +package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model; import java.io.Serializable; diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/model/BulkLoadProfile.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/model/BulkLoadProfile.java new file mode 100644 index 00000000..8bcfb97c --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/model/BulkLoadProfile.java @@ -0,0 +1,133 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model; + + +import java.io.Serializable; +import java.util.ArrayList; + + +/*************************************************************************** + * this is the model of a saved bulk load profile - which is what passes back + * and forth with the frontend. + ****************************************************************************/ +public class BulkLoadProfile implements Serializable +{ + private ArrayList fieldList; + private Boolean hasHeaderRow; + private String layout; + + + + /******************************************************************************* + ** Getter for fieldList + *******************************************************************************/ + public ArrayList getFieldList() + { + return (this.fieldList); + } + + + + + /******************************************************************************* + ** Getter for hasHeaderRow + *******************************************************************************/ + public Boolean getHasHeaderRow() + { + return (this.hasHeaderRow); + } + + + + /******************************************************************************* + ** Setter for hasHeaderRow + *******************************************************************************/ + public void setHasHeaderRow(Boolean hasHeaderRow) + { + this.hasHeaderRow = hasHeaderRow; + } + + + + /******************************************************************************* + ** Fluent setter for hasHeaderRow + *******************************************************************************/ + public BulkLoadProfile withHasHeaderRow(Boolean hasHeaderRow) + { + this.hasHeaderRow = hasHeaderRow; + return (this); + } + + + + /******************************************************************************* + ** Getter for layout + *******************************************************************************/ + public String getLayout() + { + return (this.layout); + } + + + + /******************************************************************************* + ** Setter for layout + *******************************************************************************/ + public void setLayout(String layout) + { + this.layout = layout; + } + + + + /******************************************************************************* + ** Fluent setter for layout + *******************************************************************************/ + public BulkLoadProfile withLayout(String layout) + { + this.layout = layout; + return (this); + } + + + /******************************************************************************* + ** Setter for fieldList + *******************************************************************************/ + public void setFieldList(ArrayList fieldList) + { + this.fieldList = fieldList; + } + + + + /******************************************************************************* + ** Fluent setter for fieldList + *******************************************************************************/ + public BulkLoadProfile withFieldList(ArrayList fieldList) + { + this.fieldList = fieldList; + return (this); + } + + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/model/BulkLoadProfileField.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/model/BulkLoadProfileField.java new file mode 100644 index 00000000..6bedb18b --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/model/BulkLoadProfileField.java @@ -0,0 +1,195 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model; + + +import java.io.Serializable; +import java.util.Map; + + +/*************************************************************************** + ** + ***************************************************************************/ +public class BulkLoadProfileField +{ + private String fieldName; + private Integer columnIndex; + private Serializable defaultValue; + private Boolean doValueMapping; + private Map valueMappings; + + + + /******************************************************************************* + ** Getter for fieldName + *******************************************************************************/ + public String getFieldName() + { + return (this.fieldName); + } + + + + /******************************************************************************* + ** Setter for fieldName + *******************************************************************************/ + public void setFieldName(String fieldName) + { + this.fieldName = fieldName; + } + + + + /******************************************************************************* + ** Fluent setter for fieldName + *******************************************************************************/ + public BulkLoadProfileField withFieldName(String fieldName) + { + this.fieldName = fieldName; + return (this); + } + + + + /******************************************************************************* + ** Getter for columnIndex + *******************************************************************************/ + public Integer getColumnIndex() + { + return (this.columnIndex); + } + + + + /******************************************************************************* + ** Setter for columnIndex + *******************************************************************************/ + public void setColumnIndex(Integer columnIndex) + { + this.columnIndex = columnIndex; + } + + + + /******************************************************************************* + ** Fluent setter for columnIndex + *******************************************************************************/ + public BulkLoadProfileField withColumnIndex(Integer columnIndex) + { + this.columnIndex = columnIndex; + return (this); + } + + + + /******************************************************************************* + ** Getter for defaultValue + *******************************************************************************/ + public Serializable getDefaultValue() + { + return (this.defaultValue); + } + + + + /******************************************************************************* + ** Setter for defaultValue + *******************************************************************************/ + public void setDefaultValue(Serializable defaultValue) + { + this.defaultValue = defaultValue; + } + + + + /******************************************************************************* + ** Fluent setter for defaultValue + *******************************************************************************/ + public BulkLoadProfileField withDefaultValue(Serializable defaultValue) + { + this.defaultValue = defaultValue; + return (this); + } + + + + /******************************************************************************* + ** Getter for doValueMapping + *******************************************************************************/ + public Boolean getDoValueMapping() + { + return (this.doValueMapping); + } + + + + /******************************************************************************* + ** Setter for doValueMapping + *******************************************************************************/ + public void setDoValueMapping(Boolean doValueMapping) + { + this.doValueMapping = doValueMapping; + } + + + + /******************************************************************************* + ** Fluent setter for doValueMapping + *******************************************************************************/ + public BulkLoadProfileField withDoValueMapping(Boolean doValueMapping) + { + this.doValueMapping = doValueMapping; + return (this); + } + + + + /******************************************************************************* + ** Getter for valueMappings + *******************************************************************************/ + public Map getValueMappings() + { + return (this.valueMappings); + } + + + + /******************************************************************************* + ** Setter for valueMappings + *******************************************************************************/ + public void setValueMappings(Map valueMappings) + { + this.valueMappings = valueMappings; + } + + + + /******************************************************************************* + ** Fluent setter for valueMappings + *******************************************************************************/ + public BulkLoadProfileField withValueMappings(Map valueMappings) + { + this.valueMappings = valueMappings; + return (this); + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/model/BulkLoadTableStructure.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/model/BulkLoadTableStructure.java new file mode 100644 index 00000000..db55198f --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/model/BulkLoadTableStructure.java @@ -0,0 +1,275 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model; + + +import java.io.Serializable; +import java.util.ArrayList; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class BulkLoadTableStructure implements Serializable +{ + private boolean isMain; + private boolean isMany; + + private String tableName; + private String label; + private String associationPath; // null/empty for main table, then associationName for a child, associationName.associationName for a grandchild + + private ArrayList fields; // mmm, not marked as serializable (at this time) - is okay? + private ArrayList associations; + + + + /******************************************************************************* + ** Getter for isMain + *******************************************************************************/ + public boolean getIsMain() + { + return (this.isMain); + } + + + + /******************************************************************************* + ** Setter for isMain + *******************************************************************************/ + public void setIsMain(boolean isMain) + { + this.isMain = isMain; + } + + + + /******************************************************************************* + ** Fluent setter for isMain + *******************************************************************************/ + public BulkLoadTableStructure withIsMain(boolean isMain) + { + this.isMain = isMain; + return (this); + } + + + + /******************************************************************************* + ** Getter for isMany + *******************************************************************************/ + public boolean getIsMany() + { + return (this.isMany); + } + + + + /******************************************************************************* + ** Setter for isMany + *******************************************************************************/ + public void setIsMany(boolean isMany) + { + this.isMany = isMany; + } + + + + /******************************************************************************* + ** Fluent setter for isMany + *******************************************************************************/ + public BulkLoadTableStructure withIsMany(boolean isMany) + { + this.isMany = isMany; + return (this); + } + + + + /******************************************************************************* + ** Getter for tableName + *******************************************************************************/ + public String getTableName() + { + return (this.tableName); + } + + + + /******************************************************************************* + ** Setter for tableName + *******************************************************************************/ + public void setTableName(String tableName) + { + this.tableName = tableName; + } + + + + /******************************************************************************* + ** Fluent setter for tableName + *******************************************************************************/ + public BulkLoadTableStructure withTableName(String tableName) + { + this.tableName = tableName; + return (this); + } + + + + /******************************************************************************* + ** Getter for label + *******************************************************************************/ + public String getLabel() + { + return (this.label); + } + + + + /******************************************************************************* + ** Setter for label + *******************************************************************************/ + public void setLabel(String label) + { + this.label = label; + } + + + + /******************************************************************************* + ** Fluent setter for label + *******************************************************************************/ + public BulkLoadTableStructure withLabel(String label) + { + this.label = label; + return (this); + } + + + + /******************************************************************************* + ** Getter for fields + *******************************************************************************/ + public ArrayList getFields() + { + return (this.fields); + } + + + + /******************************************************************************* + ** Setter for fields + *******************************************************************************/ + public void setFields(ArrayList fields) + { + this.fields = fields; + } + + + + /******************************************************************************* + ** Fluent setter for fields + *******************************************************************************/ + public BulkLoadTableStructure withFields(ArrayList fields) + { + this.fields = fields; + return (this); + } + + + + /******************************************************************************* + ** Getter for associationPath + *******************************************************************************/ + public String getAssociationPath() + { + return (this.associationPath); + } + + + + /******************************************************************************* + ** Setter for associationPath + *******************************************************************************/ + public void setAssociationPath(String associationPath) + { + this.associationPath = associationPath; + } + + + + /******************************************************************************* + ** Fluent setter for associationPath + *******************************************************************************/ + public BulkLoadTableStructure withAssociationPath(String associationPath) + { + this.associationPath = associationPath; + return (this); + } + + + + /******************************************************************************* + ** Getter for associations + *******************************************************************************/ + public ArrayList getAssociations() + { + return (this.associations); + } + + + + /******************************************************************************* + ** Setter for associations + *******************************************************************************/ + public void setAssociations(ArrayList associations) + { + this.associations = associations; + } + + + + /******************************************************************************* + ** Fluent setter for associations + *******************************************************************************/ + public BulkLoadTableStructure withAssociations(ArrayList associations) + { + this.associations = associations; + return (this); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + public void addAssociation(BulkLoadTableStructure association) + { + if(this.associations == null) + { + this.associations = new ArrayList<>(); + } + this.associations.add(association); + } +} diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertPrepareFileMappingStepTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertPrepareFileMappingStepTest.java new file mode 100644 index 00000000..ea6e0e8f --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertPrepareFileMappingStepTest.java @@ -0,0 +1,69 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert; + + +import com.kingsrook.qqq.backend.core.BaseTest; +import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.assertEquals; + + +/******************************************************************************* + ** Unit test for BulkInsertPrepareMappingStep + *******************************************************************************/ +class BulkInsertPrepareFileMappingStepTest extends BaseTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @SuppressWarnings("PointlessArithmeticExpression") + @Test + void testToHeaderLetter() + { + assertEquals("A", BulkInsertPrepareFileMappingStep.toHeaderLetter(0)); + assertEquals("B", BulkInsertPrepareFileMappingStep.toHeaderLetter(1)); + assertEquals("Z", BulkInsertPrepareFileMappingStep.toHeaderLetter(25)); + + assertEquals("AA", BulkInsertPrepareFileMappingStep.toHeaderLetter(26 + 0)); + assertEquals("AB", BulkInsertPrepareFileMappingStep.toHeaderLetter(26 + 1)); + assertEquals("AZ", BulkInsertPrepareFileMappingStep.toHeaderLetter(26 + 25)); + + assertEquals("BA", BulkInsertPrepareFileMappingStep.toHeaderLetter(2 * 26 + 0)); + assertEquals("BB", BulkInsertPrepareFileMappingStep.toHeaderLetter(2 * 26 + 1)); + assertEquals("BZ", BulkInsertPrepareFileMappingStep.toHeaderLetter(2 * 26 + 25)); + + assertEquals("ZA", BulkInsertPrepareFileMappingStep.toHeaderLetter(26 * 26 + 0)); + assertEquals("ZB", BulkInsertPrepareFileMappingStep.toHeaderLetter(26 * 26 + 1)); + assertEquals("ZZ", BulkInsertPrepareFileMappingStep.toHeaderLetter(26 * 26 + 25)); + + assertEquals("AAA", BulkInsertPrepareFileMappingStep.toHeaderLetter(27 * 26 + 0)); + assertEquals("AAB", BulkInsertPrepareFileMappingStep.toHeaderLetter(27 * 26 + 1)); + assertEquals("AAC", BulkInsertPrepareFileMappingStep.toHeaderLetter(27 * 26 + 2)); + + assertEquals("ABA", BulkInsertPrepareFileMappingStep.toHeaderLetter(28 * 26 + 0)); + assertEquals("ABB", BulkInsertPrepareFileMappingStep.toHeaderLetter(28 * 26 + 1)); + + assertEquals("BAA", BulkInsertPrepareFileMappingStep.toHeaderLetter(2 * 26 * 26 + 26 + 0)); + } + +} \ No newline at end of file diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertReceiveMappingStep.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertPrepareValueMappingStepTest.java similarity index 50% rename from qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertReceiveMappingStep.java rename to qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertPrepareValueMappingStepTest.java index 0071aa49..6953c5bc 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertReceiveMappingStep.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertPrepareValueMappingStepTest.java @@ -22,38 +22,34 @@ package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert; -import java.util.Map; -import com.kingsrook.qqq.backend.core.actions.processes.BackendStep; +import com.kingsrook.qqq.backend.core.BaseTest; import com.kingsrook.qqq.backend.core.exceptions.QException; -import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; -import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; -import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping.BulkInsertMapping; +import com.kingsrook.qqq.backend.core.utils.TestUtils; +import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.assertEquals; /******************************************************************************* - ** + ** Unit test for BulkInsertPrepareValueMappingStep *******************************************************************************/ -public class BulkInsertReceiveMappingStep implements BackendStep +class BulkInsertPrepareValueMappingStepTest extends BaseTest { - /*************************************************************************** + /******************************************************************************* ** - ***************************************************************************/ - @Override - public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException + *******************************************************************************/ + @Test + void test() throws QException { - BulkInsertMapping bulkInsertMapping = new BulkInsertMapping(); - bulkInsertMapping.setTableName(runBackendStepInput.getTableName()); - bulkInsertMapping.setHasHeaderRow(true); - bulkInsertMapping.setFieldNameToHeaderNameMap(Map.of( - "firstName", "firstName", - "lastName", "Last Name" - )); - runBackendStepOutput.addValue("bulkInsertMapping", bulkInsertMapping); + assertEquals(TestUtils.TABLE_NAME_ORDER, BulkInsertPrepareValueMappingStep.getTableAndField(TestUtils.TABLE_NAME_ORDER, "orderNo").table().getName()); + assertEquals("orderNo", BulkInsertPrepareValueMappingStep.getTableAndField(TestUtils.TABLE_NAME_ORDER, "orderNo").field().getName()); + + assertEquals(TestUtils.TABLE_NAME_LINE_ITEM, BulkInsertPrepareValueMappingStep.getTableAndField(TestUtils.TABLE_NAME_ORDER, "orderLine.sku").table().getName()); + assertEquals("sku", BulkInsertPrepareValueMappingStep.getTableAndField(TestUtils.TABLE_NAME_ORDER, "orderLine.sku").field().getName()); + + assertEquals(TestUtils.TABLE_NAME_LINE_ITEM_EXTRINSIC, BulkInsertPrepareValueMappingStep.getTableAndField(TestUtils.TABLE_NAME_ORDER, "orderLine.extrinsics.key").table().getName()); + assertEquals("key", BulkInsertPrepareValueMappingStep.getTableAndField(TestUtils.TABLE_NAME_ORDER, "orderLine.extrinsics.key").field().getName()); - // probably need to what, receive the mapping object, store it into state - // what, do we maybe return to a different sub-mapping screen (e.g., values) - // then at some point - cool - proceed to ETL's steps } -} +} \ No newline at end of file diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/CsvFileToRowsTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/CsvFileToRowsTest.java index 84bdce2f..77b60bd8 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/CsvFileToRowsTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/CsvFileToRowsTest.java @@ -25,7 +25,7 @@ package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.fil import java.io.ByteArrayInputStream; import com.kingsrook.qqq.backend.core.BaseTest; import com.kingsrook.qqq.backend.core.exceptions.QException; -import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkLoadFileRow; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow; import org.junit.jupiter.api.Test; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/TestFileToRows.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/TestFileToRows.java index e1142e89..a9d07b63 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/TestFileToRows.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/TestFileToRows.java @@ -26,7 +26,7 @@ import java.io.InputStream; import java.io.Serializable; import java.util.List; import com.kingsrook.qqq.backend.core.exceptions.QException; -import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkLoadFileRow; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow; /*************************************************************************** diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/XlsxFileToRowsTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/XlsxFileToRowsTest.java index 8a381ad1..c681f685 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/XlsxFileToRowsTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/XlsxFileToRowsTest.java @@ -37,7 +37,7 @@ import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportDestination; import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportFormat; import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportInput; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; -import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkLoadFileRow; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow; import org.junit.jupiter.api.Test; import static com.kingsrook.qqq.backend.core.actions.reporting.GenerateReportActionTest.REPORT_NAME; import static org.junit.jupiter.api.Assertions.assertEquals; diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/FlatRowsToRecordTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/FlatRowsToRecordTest.java index e5474032..e8c561e5 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/FlatRowsToRecordTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/FlatRowsToRecordTest.java @@ -29,8 +29,8 @@ import java.util.Map; import com.kingsrook.qqq.backend.core.BaseTest; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.data.QRecord; -import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkLoadFileRow; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.TestFileToRows; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow; import com.kingsrook.qqq.backend.core.utils.TestUtils; import com.kingsrook.qqq.backend.core.utils.collections.ListBuilder; import org.junit.jupiter.api.Test; diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/TallRowsToRecordTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/TallRowsToRecordTest.java index c623ddda..5c942fa2 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/TallRowsToRecordTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/TallRowsToRecordTest.java @@ -28,8 +28,8 @@ import java.util.Map; import com.kingsrook.qqq.backend.core.BaseTest; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.data.QRecord; -import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkLoadFileRow; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.CsvFileToRows; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow; import com.kingsrook.qqq.backend.core.utils.TestUtils; import org.junit.jupiter.api.Test; import static org.assertj.core.api.Assertions.assertThat; @@ -87,14 +87,14 @@ class TallRowsToRecordTest extends BaseTest assertEquals("Homer", order.getValueString("shipToName")); assertEquals(3, order.getAssociatedRecords().get("orderLine").size()); assertEquals(List.of("DONUT", "BEER", "COUCH"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); - assertEquals(List.of("12", "500", "1"), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); + assertEquals(List.of(12, 500, 1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); order = records.get(1); assertEquals(2, order.getValueInteger("orderNo")); assertEquals("Ned", order.getValueString("shipToName")); assertEquals(2, order.getAssociatedRecords().get("orderLine").size()); assertEquals(List.of("BIBLE", "LAWNMOWER"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); - assertEquals(List.of("7", "1"), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); + assertEquals(List.of(7, 1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); } @@ -145,7 +145,7 @@ class TallRowsToRecordTest extends BaseTest assertEquals("Homer", order.getValueString("shipToName")); assertEquals(3, order.getAssociatedRecords().get("orderLine").size()); assertEquals(List.of("DONUT", "BEER", "COUCH"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); - assertEquals(List.of("12", "500", "1"), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); + assertEquals(List.of(12, 500, 1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); assertEquals(2, order.getAssociatedRecords().get("extrinsics").size()); assertEquals(List.of("Store Name", "Coupon Code"), getValues(order.getAssociatedRecords().get("extrinsics"), "key")); assertEquals(List.of("QQQ Mart", "10QOff"), getValues(order.getAssociatedRecords().get("extrinsics"), "value")); @@ -155,7 +155,7 @@ class TallRowsToRecordTest extends BaseTest assertEquals("Ned", order.getValueString("shipToName")); assertEquals(2, order.getAssociatedRecords().get("orderLine").size()); assertEquals(List.of("BIBLE", "LAWNMOWER"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); - assertEquals(List.of("7", "1"), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); + assertEquals(List.of(7, 1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); assertThat(order.getAssociatedRecords().get("extrinsics")).isNullOrEmpty(); } @@ -168,7 +168,7 @@ class TallRowsToRecordTest extends BaseTest void testOrderLinesWithLineExtrinsicsAndOrderExtrinsic() throws QException { Integer defaultStoreId = 101; - Integer defaultLineNo = 102; + String defaultLineNo = "102"; String defaultOrderLineExtraSource = "file"; CsvFileToRows fileToRows = CsvFileToRows.forString(""" @@ -221,7 +221,7 @@ class TallRowsToRecordTest extends BaseTest assertEquals("Homer", order.getValueString("shipToName")); assertEquals(defaultStoreId, order.getValue("storeId")); assertEquals(List.of("D'OH-NUT", "BEER", "COUCH"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); - assertEquals(List.of("12", "500", "1"), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); + assertEquals(List.of(12, 500, 1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); assertEquals(List.of(defaultLineNo, defaultLineNo, defaultLineNo), getValues(order.getAssociatedRecords().get("orderLine"), "lineNumber")); assertEquals(List.of("Store Name", "Coupon Code"), getValues(order.getAssociatedRecords().get("extrinsics"), "key")); assertEquals(List.of("QQQ Mart", "10QOff"), getValues(order.getAssociatedRecords().get("extrinsics"), "value")); @@ -240,7 +240,92 @@ class TallRowsToRecordTest extends BaseTest assertEquals("Ned", order.getValueString("shipToName")); assertEquals(defaultStoreId, order.getValue("storeId")); assertEquals(List.of("BIBLE", "LAWNMOWER"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); - assertEquals(List.of("7", "1"), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); + assertEquals(List.of(7, 1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); + assertEquals(List.of(defaultLineNo, defaultLineNo), getValues(order.getAssociatedRecords().get("orderLine"), "lineNumber")); + assertThat(order.getAssociatedRecords().get("extrinsics")).isNullOrEmpty(); + + lineItem = order.getAssociatedRecords().get("orderLine").get(0); + assertEquals(List.of("Flavor"), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "key")); + assertEquals(List.of("King James"), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "value")); + assertEquals(List.of(defaultOrderLineExtraSource), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "source")); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testAutomaticGroupByAllIndexes() throws QException + { + Integer defaultStoreId = 101; + String defaultLineNo = "102"; + String defaultOrderLineExtraSource = "file"; + + CsvFileToRows fileToRows = CsvFileToRows.forString(""" + orderNo, Ship To, lastName, SKU, Quantity, Extrinsic Key, Extrinsic Value, Line Extrinsic Key, Line Extrinsic Value + 1, Homer, Simpson, DONUT, 12, Store Name, QQQ Mart, Flavor, Chocolate + 1, Homer, Simpson, DONUT, 12, Coupon Code, 10QOff, Size, Large + 1, Homer, Simpson, BEER, 500, , , Flavor, Hops + 1, Homer, Simpson, COUCH, 1 + 2, Ned, Flanders, BIBLE, 7, , , Flavor, King James + 2, Ned, Flanders, LAWNMOWER, 1 + """); + + BulkLoadFileRow header = fileToRows.next(); + + TallRowsToRecord rowsToRecord = new TallRowsToRecord(); + + BulkInsertMapping mapping = new BulkInsertMapping() + .withFieldNameToHeaderNameMap(Map.of( + "orderNo", "orderNo", + "shipToName", "Ship To", + "orderLine.sku", "SKU", + "orderLine.quantity", "Quantity", + "extrinsics.key", "Extrinsic Key", + "extrinsics.value", "Extrinsic Value", + "orderLine.extrinsics.key", "Line Extrinsic Key", + "orderLine.extrinsics.value", "Line Extrinsic Value" + )) + .withFieldNameToDefaultValueMap(Map.of( + "storeId", defaultStoreId, + "orderLine.lineNumber", defaultLineNo, + "orderLine.extrinsics.source", defaultOrderLineExtraSource + )) + .withFieldNameToValueMapping(Map.of("orderLine.sku", Map.of("DONUT", "D'OH-NUT"))) + .withMappedAssociations(List.of("orderLine", "extrinsics", "orderLine.extrinsics")) + .withTableName(TestUtils.TABLE_NAME_ORDER) + .withLayout(BulkInsertMapping.Layout.TALL) + .withHasHeaderRow(true); + + List records = rowsToRecord.nextPage(fileToRows, header, mapping, Integer.MAX_VALUE); + assertEquals(2, records.size()); + + QRecord order = records.get(0); + assertEquals(1, order.getValueInteger("orderNo")); + assertEquals("Homer", order.getValueString("shipToName")); + assertEquals(defaultStoreId, order.getValue("storeId")); + assertEquals(List.of("D'OH-NUT", "BEER", "COUCH"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); + assertEquals(List.of(12, 500, 1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); + assertEquals(List.of(defaultLineNo, defaultLineNo, defaultLineNo), getValues(order.getAssociatedRecords().get("orderLine"), "lineNumber")); + assertEquals(List.of("Store Name", "Coupon Code"), getValues(order.getAssociatedRecords().get("extrinsics"), "key")); + assertEquals(List.of("QQQ Mart", "10QOff"), getValues(order.getAssociatedRecords().get("extrinsics"), "value")); + + QRecord lineItem = order.getAssociatedRecords().get("orderLine").get(0); + assertEquals(List.of("Flavor", "Size"), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "key")); + assertEquals(List.of("Chocolate", "Large"), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "value")); + assertEquals(List.of(defaultOrderLineExtraSource, defaultOrderLineExtraSource), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "source")); + + lineItem = order.getAssociatedRecords().get("orderLine").get(1); + assertEquals(List.of("Flavor"), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "key")); + assertEquals(List.of("Hops"), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "value")); + + order = records.get(1); + assertEquals(2, order.getValueInteger("orderNo")); + assertEquals("Ned", order.getValueString("shipToName")); + assertEquals(defaultStoreId, order.getValue("storeId")); + assertEquals(List.of("BIBLE", "LAWNMOWER"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); + assertEquals(List.of(7, 1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); assertEquals(List.of(defaultLineNo, defaultLineNo), getValues(order.getAssociatedRecords().get("orderLine"), "lineNumber")); assertThat(order.getAssociatedRecords().get("extrinsics")).isNullOrEmpty(); diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/ValueMapperTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/ValueMapperTest.java index 9e4108e9..7b868766 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/ValueMapperTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/ValueMapperTest.java @@ -26,8 +26,11 @@ import java.io.Serializable; import java.util.List; import java.util.Map; import com.kingsrook.qqq.backend.core.BaseTest; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.utils.CollectionUtils; +import com.kingsrook.qqq.backend.core.utils.TestUtils; import org.json.JSONArray; import org.json.JSONObject; import org.junit.jupiter.api.Test; @@ -44,7 +47,7 @@ class ValueMapperTest extends BaseTest ** *******************************************************************************/ @Test - void test() + void test() throws QException { BulkInsertMapping mapping = new BulkInsertMapping().withFieldNameToValueMapping(Map.of( "storeId", Map.of("QQQMart", 1, "Q'R'Us", 2), @@ -94,7 +97,7 @@ class ValueMapperTest extends BaseTest ); JSONObject expectedJson = recordToJson(expectedRecord); - ValueMapper.valueMapping(List.of(inputRecord), mapping); + ValueMapper.valueMapping(List.of(inputRecord), mapping, QContext.getQInstance().getTable(TestUtils.TABLE_NAME_ORDER)); JSONObject actualJson = recordToJson(inputRecord); System.out.println("Before"); diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithExplicitMappingTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithExplicitMappingTest.java new file mode 100644 index 00000000..39434f78 --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithExplicitMappingTest.java @@ -0,0 +1,269 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping; + + +import java.io.Serializable; +import java.util.List; +import java.util.Map; +import com.kingsrook.qqq.backend.core.BaseTest; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.CsvFileToRows; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow; +import com.kingsrook.qqq.backend.core.utils.TestUtils; +import org.junit.jupiter.api.Test; +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertEquals; + + +/******************************************************************************* + ** Unit test for WideRowsToRecord + *******************************************************************************/ +class WideRowsToRecordWithExplicitMappingTest extends BaseTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testOrderAndLinesWithoutDupes() throws QException + { + String csv = """ + orderNo, Ship To, lastName, SKU 1, Quantity 1, SKU 2, Quantity 2, SKU 3, Quantity 3 + 1, Homer, Simpson, DONUT, 12, BEER, 500, COUCH, 1 + 2, Ned, Flanders, BIBLE, 7, LAWNMOWER, 1 + """; + + CsvFileToRows fileToRows = CsvFileToRows.forString(csv); + BulkLoadFileRow header = fileToRows.next(); + + WideRowsToRecordWithExplicitMapping rowsToRecord = new WideRowsToRecordWithExplicitMapping(); + + BulkInsertMapping mapping = new BulkInsertMapping() + .withFieldNameToHeaderNameMap(Map.of( + "orderNo", "orderNo", + "shipToName", "Ship To" + )) + .withWideLayoutMapping(Map.of( + "orderLine", new BulkInsertWideLayoutMapping(List.of( + new BulkInsertWideLayoutMapping.ChildRecordMapping(Map.of("sku", "SKU 1", "quantity", "Quantity 1")), + new BulkInsertWideLayoutMapping.ChildRecordMapping(Map.of("sku", "SKU 2", "quantity", "Quantity 2")), + new BulkInsertWideLayoutMapping.ChildRecordMapping(Map.of("sku", "SKU 3", "quantity", "Quantity 3")) + )) + )) + .withMappedAssociations(List.of("orderLine")) + .withTableName(TestUtils.TABLE_NAME_ORDER) + .withLayout(BulkInsertMapping.Layout.WIDE) + .withHasHeaderRow(true); + + List records = rowsToRecord.nextPage(fileToRows, header, mapping, Integer.MAX_VALUE); + assertEquals(2, records.size()); + + QRecord order = records.get(0); + assertEquals(1, order.getValueInteger("orderNo")); + assertEquals("Homer", order.getValueString("shipToName")); + assertEquals(List.of("DONUT", "BEER", "COUCH"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); + assertEquals(List.of("12", "500", "1"), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); + + order = records.get(1); + assertEquals(2, order.getValueInteger("orderNo")); + assertEquals("Ned", order.getValueString("shipToName")); + assertEquals(List.of("BIBLE", "LAWNMOWER"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); + assertEquals(List.of("7", "1"), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testOrderLinesAndOrderExtrinsicWithoutDupes() throws QException + { + String csv = """ + orderNo, Ship To, lastName, SKU 1, Quantity 1, SKU 2, Quantity 2, SKU 3, Quantity 3, Extrinsic Key 1, Extrinsic Value 1, Extrinsic Key 2, Extrinsic Value 2 + 1, Homer, Simpson, DONUT, 12, BEER, 500, COUCH, 1, Store Name, QQQ Mart, Coupon Code, 10QOff + 2, Ned, Flanders, BIBLE, 7, LAWNMOWER, 1 + """; + + CsvFileToRows fileToRows = CsvFileToRows.forString(csv); + BulkLoadFileRow header = fileToRows.next(); + + WideRowsToRecordWithExplicitMapping rowsToRecord = new WideRowsToRecordWithExplicitMapping(); + + BulkInsertMapping mapping = new BulkInsertMapping() + .withFieldNameToHeaderNameMap(Map.of( + "orderNo", "orderNo", + "shipToName", "Ship To" + )) + .withWideLayoutMapping(Map.of( + "orderLine", new BulkInsertWideLayoutMapping(List.of( + new BulkInsertWideLayoutMapping.ChildRecordMapping(Map.of("sku", "SKU 1", "quantity", "Quantity 1")), + new BulkInsertWideLayoutMapping.ChildRecordMapping(Map.of("sku", "SKU 2", "quantity", "Quantity 2")), + new BulkInsertWideLayoutMapping.ChildRecordMapping(Map.of("sku", "SKU 3", "quantity", "Quantity 3")) + )), + "extrinsics", new BulkInsertWideLayoutMapping(List.of( + new BulkInsertWideLayoutMapping.ChildRecordMapping(Map.of("key", "Extrinsic Key 1", "value", "Extrinsic Value 1")), + new BulkInsertWideLayoutMapping.ChildRecordMapping(Map.of("key", "Extrinsic Key 2", "value", "Extrinsic Value 2")) + )) + )) + .withMappedAssociations(List.of("orderLine", "extrinsics")) + .withTableName(TestUtils.TABLE_NAME_ORDER) + .withLayout(BulkInsertMapping.Layout.WIDE) + .withHasHeaderRow(true); + + List records = rowsToRecord.nextPage(fileToRows, header, mapping, Integer.MAX_VALUE); + assertEquals(2, records.size()); + + QRecord order = records.get(0); + assertEquals(1, order.getValueInteger("orderNo")); + assertEquals("Homer", order.getValueString("shipToName")); + assertEquals(List.of("DONUT", "BEER", "COUCH"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); + assertEquals(List.of("12", "500", "1"), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); + assertEquals(List.of("Store Name", "Coupon Code"), getValues(order.getAssociatedRecords().get("extrinsics"), "key")); + assertEquals(List.of("QQQ Mart", "10QOff"), getValues(order.getAssociatedRecords().get("extrinsics"), "value")); + + order = records.get(1); + assertEquals(2, order.getValueInteger("orderNo")); + assertEquals("Ned", order.getValueString("shipToName")); + assertEquals(List.of("BIBLE", "LAWNMOWER"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); + assertEquals(List.of("7", "1"), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); + assertThat(order.getAssociatedRecords().get("extrinsics")).isNullOrEmpty(); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testOrderLinesWithLineExtrinsicsAndOrderExtrinsicWithoutDupes() throws QException + { + String csv = """ + orderNo, Ship To, lastName, Extrinsic Key 1, Extrinsic Value 1, Extrinsic Key 2, Extrinsic Value 2, SKU 1, Quantity 1, Line Extrinsic Key 1.1, Line Extrinsic Value 1.1, Line Extrinsic Key 1.2, Line Extrinsic Value 1.2, SKU 2, Quantity 2, Line Extrinsic Key 2.1, Line Extrinsic Value 2.1, SKU 3, Quantity 3, Line Extrinsic Key 3.1, Line Extrinsic Value 3.1, Line Extrinsic Key 3.2 + 1, Homer, Simpson, Store Name, QQQ Mart, Coupon Code, 10QOff, DONUT, 12, Flavor, Chocolate, Size, Large, BEER, 500, Flavor, Hops, COUCH, 1, Color, Brown, foo, + 2, Ned, Flanders, , , , , BIBLE, 7, Flavor, King James, Size, X-Large, LAWNMOWER, 1 + """; + + Integer defaultStoreId = 42; + Integer defaultLineNo = 47; + String defaultLineExtraValue = "bar"; + + CsvFileToRows fileToRows = CsvFileToRows.forString(csv); + BulkLoadFileRow header = fileToRows.next(); + + WideRowsToRecordWithExplicitMapping rowsToRecord = new WideRowsToRecordWithExplicitMapping(); + + BulkInsertMapping mapping = new BulkInsertMapping() + .withFieldNameToHeaderNameMap(Map.of( + "orderNo", "orderNo", + "shipToName", "Ship To" + )) + .withMappedAssociations(List.of("orderLine", "extrinsics", "orderLine.extrinsics")) + .withWideLayoutMapping(Map.of( + "orderLine", new BulkInsertWideLayoutMapping(List.of( + new BulkInsertWideLayoutMapping.ChildRecordMapping( + Map.of("sku", "SKU 1", "quantity", "Quantity 1"), + Map.of("extrinsics", new BulkInsertWideLayoutMapping(List.of( + new BulkInsertWideLayoutMapping.ChildRecordMapping(Map.of("key", "Line Extrinsic Key 1.1", "value", "Line Extrinsic Value 1.1")), + new BulkInsertWideLayoutMapping.ChildRecordMapping(Map.of("key", "Line Extrinsic Key 1.2", "value", "Line Extrinsic Value 1.2")) + )))), + new BulkInsertWideLayoutMapping.ChildRecordMapping( + Map.of("sku", "SKU 2", "quantity", "Quantity 2"), + Map.of("extrinsics", new BulkInsertWideLayoutMapping(List.of( + new BulkInsertWideLayoutMapping.ChildRecordMapping(Map.of("key", "Line Extrinsic Key 2.1", "value", "Line Extrinsic Value 2.1")), + new BulkInsertWideLayoutMapping.ChildRecordMapping(Map.of("key", "Line Extrinsic Key 2.2", "value", "Line Extrinsic Value 2.2")) + )))), + new BulkInsertWideLayoutMapping.ChildRecordMapping( + Map.of("sku", "SKU 3", "quantity", "Quantity 3"), + Map.of("extrinsics", new BulkInsertWideLayoutMapping(List.of( + new BulkInsertWideLayoutMapping.ChildRecordMapping(Map.of("key", "Line Extrinsic Key 3.1", "value", "Line Extrinsic Value 3.1")), + new BulkInsertWideLayoutMapping.ChildRecordMapping(Map.of("key", "Line Extrinsic Key 3.2", "value", "Line Extrinsic Value 3.2")) + )))) + )), + "extrinsics", new BulkInsertWideLayoutMapping(List.of( + new BulkInsertWideLayoutMapping.ChildRecordMapping(Map.of("key", "Extrinsic Key 1", "value", "Extrinsic Value 1")), + new BulkInsertWideLayoutMapping.ChildRecordMapping(Map.of("key", "Extrinsic Key 2", "value", "Extrinsic Value 2")) + )) + )) + + .withFieldNameToValueMapping(Map.of("orderLine.extrinsics.value", Map.of("Large", "L", "X-Large", "XL"))) + .withFieldNameToDefaultValueMap(Map.of( + "storeId", defaultStoreId, + "orderLine.lineNumber", defaultLineNo, + "orderLine.extrinsics.value", defaultLineExtraValue + )) + .withTableName(TestUtils.TABLE_NAME_ORDER) + .withLayout(BulkInsertMapping.Layout.WIDE) + .withHasHeaderRow(true); + + List records = rowsToRecord.nextPage(fileToRows, header, mapping, Integer.MAX_VALUE); + assertEquals(2, records.size()); + + QRecord order = records.get(0); + assertEquals(1, order.getValueInteger("orderNo")); + assertEquals("Homer", order.getValueString("shipToName")); + assertEquals(defaultStoreId, order.getValue("storeId")); + assertEquals(List.of("DONUT", "BEER", "COUCH"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); + assertEquals(List.of("12", "500", "1"), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); + assertEquals(List.of(defaultLineNo, defaultLineNo, defaultLineNo), getValues(order.getAssociatedRecords().get("orderLine"), "lineNumber")); + assertEquals(List.of("Store Name", "Coupon Code"), getValues(order.getAssociatedRecords().get("extrinsics"), "key")); + assertEquals(List.of("QQQ Mart", "10QOff"), getValues(order.getAssociatedRecords().get("extrinsics"), "value")); + + QRecord lineItem = order.getAssociatedRecords().get("orderLine").get(0); + assertEquals(List.of("Flavor", "Size"), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "key")); + assertEquals(List.of("Chocolate", "L"), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "value")); + + lineItem = order.getAssociatedRecords().get("orderLine").get(1); + assertEquals(List.of("Flavor"), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "key")); + assertEquals(List.of("Hops"), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "value")); + + lineItem = order.getAssociatedRecords().get("orderLine").get(2); + assertEquals(List.of("Color", "foo"), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "key")); + assertEquals(List.of("Brown", defaultLineExtraValue), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "value")); + + order = records.get(1); + assertEquals(2, order.getValueInteger("orderNo")); + assertEquals("Ned", order.getValueString("shipToName")); + assertEquals(defaultStoreId, order.getValue("storeId")); + assertEquals(List.of("BIBLE", "LAWNMOWER"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); + assertEquals(List.of("7", "1"), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); + assertEquals(List.of(defaultLineNo, defaultLineNo), getValues(order.getAssociatedRecords().get("orderLine"), "lineNumber")); + assertThat(order.getAssociatedRecords().get("extrinsics")).isNullOrEmpty(); + + lineItem = order.getAssociatedRecords().get("orderLine").get(0); + assertEquals(List.of("Flavor", "Size"), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "key")); + assertEquals(List.of("King James", "XL"), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "value")); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private List getValues(List records, String fieldName) + { + return (records.stream().map(r -> r.getValue(fieldName)).toList()); + } + +} \ No newline at end of file diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithSpreadMappingTest.java similarity index 97% rename from qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordTest.java rename to qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithSpreadMappingTest.java index 51a6c5f7..45bcc751 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithSpreadMappingTest.java @@ -28,8 +28,8 @@ import java.util.Map; import com.kingsrook.qqq.backend.core.BaseTest; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.data.QRecord; -import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkLoadFileRow; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.CsvFileToRows; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow; import com.kingsrook.qqq.backend.core.utils.TestUtils; import org.junit.jupiter.api.Test; import static org.assertj.core.api.Assertions.assertThat; @@ -39,7 +39,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; /******************************************************************************* ** Unit test for WideRowsToRecord *******************************************************************************/ -class WideRowsToRecordTest extends BaseTest +class WideRowsToRecordWithSpreadMappingTest extends BaseTest { /******************************************************************************* @@ -80,7 +80,7 @@ class WideRowsToRecordTest extends BaseTest CsvFileToRows fileToRows = CsvFileToRows.forString(csv); BulkLoadFileRow header = fileToRows.next(); - WideRowsToRecord rowsToRecord = new WideRowsToRecord(); + WideRowsToRecordWithSpreadMapping rowsToRecord = new WideRowsToRecordWithSpreadMapping(); BulkInsertMapping mapping = new BulkInsertMapping() .withFieldNameToHeaderNameMap(Map.of( @@ -150,7 +150,7 @@ class WideRowsToRecordTest extends BaseTest CsvFileToRows fileToRows = CsvFileToRows.forString(csv); BulkLoadFileRow header = fileToRows.next(); - WideRowsToRecord rowsToRecord = new WideRowsToRecord(); + WideRowsToRecordWithSpreadMapping rowsToRecord = new WideRowsToRecordWithSpreadMapping(); BulkInsertMapping mapping = new BulkInsertMapping() .withFieldNameToHeaderNameMap(Map.of( @@ -229,7 +229,7 @@ class WideRowsToRecordTest extends BaseTest CsvFileToRows fileToRows = CsvFileToRows.forString(csv); BulkLoadFileRow header = fileToRows.next(); - WideRowsToRecord rowsToRecord = new WideRowsToRecord(); + WideRowsToRecordWithSpreadMapping rowsToRecord = new WideRowsToRecordWithSpreadMapping(); BulkInsertMapping mapping = new BulkInsertMapping() .withFieldNameToHeaderNameMap(Map.of( diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java index ff274dc9..8ca6da8f 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java @@ -644,6 +644,7 @@ public class TestUtils .withField(new QFieldMetaData("createDate", QFieldType.DATE_TIME).withIsEditable(false)) .withField(new QFieldMetaData("modifyDate", QFieldType.DATE_TIME).withIsEditable(false)) .withField(new QFieldMetaData("orderNo", QFieldType.STRING)) + .withField(new QFieldMetaData("shipToName", QFieldType.STRING)) .withField(new QFieldMetaData("orderDate", QFieldType.DATE)) .withField(new QFieldMetaData("storeId", QFieldType.INTEGER)) .withField(new QFieldMetaData("total", QFieldType.DECIMAL).withDisplayFormat(DisplayFormat.CURRENCY).withFieldSecurityLock(new FieldSecurityLock() @@ -700,7 +701,8 @@ public class TestUtils .withField(new QFieldMetaData("modifyDate", QFieldType.DATE_TIME).withIsEditable(false)) .withField(new QFieldMetaData("lineItemId", QFieldType.INTEGER)) .withField(new QFieldMetaData("key", QFieldType.STRING)) - .withField(new QFieldMetaData("value", QFieldType.STRING)); + .withField(new QFieldMetaData("value", QFieldType.STRING)) + .withField(new QFieldMetaData("source", QFieldType.STRING)); // doesn't really make sense, but useful to have an extra field here in some bulk-load tests }