diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTransformStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTransformStep.java index 73d35070..ad531967 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTransformStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTransformStep.java @@ -56,8 +56,8 @@ import com.kingsrook.qqq.backend.core.model.metadata.tables.QFieldSection; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.UniqueKey; import com.kingsrook.qqq.backend.core.model.statusmessages.QErrorMessage; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping.AbstractBulkLoadRollableValueError; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping.BulkLoadRecordUtils; -import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping.BulkLoadValueTypeError; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkInsertMapping; import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.AbstractTransformStep; import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.LoadViaInsertStep; @@ -210,10 +210,10 @@ public class BulkInsertTransformStep extends AbstractTransformStep { for(QErrorMessage error : record.getErrors()) { - if(error instanceof BulkLoadValueTypeError blvte) + if(error instanceof AbstractBulkLoadRollableValueError rollableValueError) { - processSummaryWarningsAndErrorsRollup.addError(blvte.getMessageToUseAsProcessSummaryRollupKey(), null); - addToErrorToExampleRowValueMap(blvte, record); + processSummaryWarningsAndErrorsRollup.addError(rollableValueError.getMessageToUseAsProcessSummaryRollupKey(), null); + addToErrorToExampleRowValueMap(rollableValueError, record); } else { @@ -348,14 +348,14 @@ public class BulkInsertTransformStep extends AbstractTransformStep /*************************************************************************** ** ***************************************************************************/ - private void addToErrorToExampleRowValueMap(BulkLoadValueTypeError bulkLoadValueTypeError, QRecord record) + private void addToErrorToExampleRowValueMap(AbstractBulkLoadRollableValueError bulkLoadRollableValueError, QRecord record) { - String message = bulkLoadValueTypeError.getMessageToUseAsProcessSummaryRollupKey(); + String message = bulkLoadRollableValueError.getMessageToUseAsProcessSummaryRollupKey(); List rowValues = errorToExampleRowValueMap.computeIfAbsent(message, k -> new ArrayList<>()); if(rowValues.size() < EXAMPLE_ROW_LIMIT) { - rowValues.add(new RowValue(bulkLoadValueTypeError, record)); + rowValues.add(new RowValue(bulkLoadRollableValueError, record)); } } @@ -550,9 +550,9 @@ public class BulkInsertTransformStep extends AbstractTransformStep /*************************************************************************** ** ***************************************************************************/ - public RowValue(BulkLoadValueTypeError bulkLoadValueTypeError, QRecord record) + public RowValue(AbstractBulkLoadRollableValueError bulkLoadRollableValueError, QRecord record) { - this(BulkLoadRecordUtils.getRowNosString(record), ValueUtils.getValueAsString(bulkLoadValueTypeError.getValue())); + this(BulkLoadRecordUtils.getRowNosString(record), ValueUtils.getValueAsString(bulkLoadRollableValueError.getValue())); } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/AbstractBulkLoadRollableValueError.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/AbstractBulkLoadRollableValueError.java new file mode 100644 index 00000000..74082fd7 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/AbstractBulkLoadRollableValueError.java @@ -0,0 +1,55 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping; + + +import java.io.Serializable; +import com.kingsrook.qqq.backend.core.model.statusmessages.BadInputStatusMessage; + + +/******************************************************************************* + ** + *******************************************************************************/ +public abstract class AbstractBulkLoadRollableValueError extends BadInputStatusMessage +{ + + /******************************************************************************* + ** Constructor + ** + *******************************************************************************/ + public AbstractBulkLoadRollableValueError(String message) + { + super(message); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + public abstract String getMessageToUseAsProcessSummaryRollupKey(); + + /*************************************************************************** + ** + ***************************************************************************/ + public abstract Serializable getValue(); +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadPossibleValueError.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadPossibleValueError.java new file mode 100644 index 00000000..199d775f --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadPossibleValueError.java @@ -0,0 +1,72 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping; + + +import java.io.Serializable; + + +/******************************************************************************* + ** Specialized error for records, for bulk-load use-cases, where we want to + ** report back info to the user about the field & value. + *******************************************************************************/ +public class BulkLoadPossibleValueError extends AbstractBulkLoadRollableValueError +{ + private final String fieldLabel; + private final Serializable value; + + + + /******************************************************************************* + ** Constructor + ** + *******************************************************************************/ + public BulkLoadPossibleValueError(String fieldName, Serializable value, String fieldLabel) + { + super("Value [" + value + "] for field [" + fieldLabel + "] is not a valid option"); + this.value = value; + this.fieldLabel = fieldLabel; + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public String getMessageToUseAsProcessSummaryRollupKey() + { + return ("Unrecognized value for field [" + fieldLabel + "]"); + } + + + + /******************************************************************************* + ** Getter for value + ** + *******************************************************************************/ + @Override + public Serializable getValue() + { + return value; + } +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadValueMapper.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadValueMapper.java new file mode 100644 index 00000000..cc44e027 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadValueMapper.java @@ -0,0 +1,238 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping; + + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import com.kingsrook.qqq.backend.core.actions.values.SearchPossibleValueSourceAction; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.logging.QLogger; +import com.kingsrook.qqq.backend.core.model.actions.values.SearchPossibleValueSourceInput; +import com.kingsrook.qqq.backend.core.model.actions.values.SearchPossibleValueSourceOutput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; +import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValue; +import com.kingsrook.qqq.backend.core.model.metadata.tables.Association; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkInsertMapping; +import com.kingsrook.qqq.backend.core.utils.CollectionUtils; +import com.kingsrook.qqq.backend.core.utils.ListingHash; +import com.kingsrook.qqq.backend.core.utils.StringUtils; +import com.kingsrook.qqq.backend.core.utils.ValueUtils; +import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class BulkLoadValueMapper +{ + private static final QLogger LOG = QLogger.getLogger(BulkLoadValueMapper.class); + + + + /*************************************************************************** + ** + ***************************************************************************/ + public static void valueMapping(List records, BulkInsertMapping mapping, QTableMetaData table) throws QException + { + valueMapping(records, mapping, table, null); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private static void valueMapping(List records, BulkInsertMapping mapping, QTableMetaData table, String associationNameChain) throws QException + { + if(CollectionUtils.nullSafeIsEmpty(records)) + { + return; + } + + String associationNamePrefixForFields = StringUtils.hasContent(associationNameChain) ? associationNameChain + "." : ""; + String tableLabelPrefix = StringUtils.hasContent(associationNameChain) ? table.getLabel() + ": " : ""; + + Map> possibleValueToRecordMap = new HashMap<>(); + + Map> mappingForTable = mapping.getFieldNameToValueMappingForTable(associationNameChain); + for(QRecord record : records) + { + for(Map.Entry valueEntry : record.getValues().entrySet()) + { + QFieldMetaData field = table.getField(valueEntry.getKey()); + Serializable value = valueEntry.getValue(); + + /////////////////// + // value mappin' // + /////////////////// + if(mappingForTable.containsKey(field.getName()) && value != null) + { + Serializable mappedValue = mappingForTable.get(field.getName()).get(ValueUtils.getValueAsString(value)); + if(mappedValue != null) + { + value = mappedValue; + } + } + + ///////////////////// + // type convertin' // + ///////////////////// + if(value != null && !"".equals(value)) + { + if(StringUtils.hasContent(field.getPossibleValueSourceName())) + { + ListingHash fieldPossibleValueToRecordMap = possibleValueToRecordMap.computeIfAbsent(field.getName(), k -> new ListingHash<>()); + fieldPossibleValueToRecordMap.add(ValueUtils.getValueAsString(value), record); + } + else + { + QFieldType type = field.getType(); + try + { + value = ValueUtils.getValueAsFieldType(type, value); + } + catch(Exception e) + { + record.addError(new BulkLoadValueTypeError(associationNamePrefixForFields + field.getName(), value, type, tableLabelPrefix + field.getLabel())); + } + } + } + + record.setValue(field.getName(), value); + } + + ////////////////////////////////////// + // recursively process associations // + ////////////////////////////////////// + for(Map.Entry> entry : record.getAssociatedRecords().entrySet()) + { + String associationName = entry.getKey(); + Optional association = table.getAssociations().stream().filter(a -> a.getName().equals(associationName)).findFirst(); + if(association.isPresent()) + { + QTableMetaData associatedTable = QContext.getQInstance().getTable(association.get().getAssociatedTableName()); + valueMapping(entry.getValue(), mapping, associatedTable, StringUtils.hasContent(associationNameChain) ? associationNameChain + "." + associationName : associationName); + } + else + { + throw new QException("Missing association [" + associationName + "] on table [" + table.getName() + "]"); + } + } + } + + ////////////////////////////////////////// + // look up and validate possible values // + ////////////////////////////////////////// + for(Map.Entry> entry : possibleValueToRecordMap.entrySet()) + { + String fieldName = entry.getKey(); + QFieldMetaData field = table.getField(fieldName); + ListingHash fieldPossibleValueToRecordMap = possibleValueToRecordMap.get(fieldName); + + handlePossibleValues(field, fieldPossibleValueToRecordMap, associationNamePrefixForFields, tableLabelPrefix); + } + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private static void handlePossibleValues(QFieldMetaData field, ListingHash fieldPossibleValueToRecordMap, String associationNamePrefixForFields, String tableLabelPrefix) throws QException + { + Set values = fieldPossibleValueToRecordMap.keySet(); + Map> valuesFound = new HashMap<>(); + Set valuesNotFound = new HashSet<>(values); + + //////////////////////////////////////////////////////// + // do a search, trying to use all given values as ids // + //////////////////////////////////////////////////////// + SearchPossibleValueSourceInput searchPossibleValueSourceInput = new SearchPossibleValueSourceInput(); + searchPossibleValueSourceInput.setPossibleValueSourceName(field.getPossibleValueSourceName()); + ArrayList idList = new ArrayList<>(values); + searchPossibleValueSourceInput.setIdList(idList); + searchPossibleValueSourceInput.setLimit(values.size()); + LOG.debug("Searching possible value source by ids during bulk load mapping", logPair("pvsName", field.getPossibleValueSourceName()), logPair("noOfIds", idList.size()), logPair("firstId", () -> idList.get(0))); + SearchPossibleValueSourceOutput searchPossibleValueSourceOutput = new SearchPossibleValueSourceAction().execute(searchPossibleValueSourceInput); + + //////////////////////////////////////////////////////////////////////////////////////////////////// + // for each possible value found, remove it from the set of ones not-found, and store it as a hit // + //////////////////////////////////////////////////////////////////////////////////////////////////// + for(QPossibleValue possibleValue : searchPossibleValueSourceOutput.getResults()) + { + String valueAsString = ValueUtils.getValueAsString(possibleValue.getId()); + valuesFound.put(valueAsString, possibleValue); + valuesNotFound.remove(valueAsString); + } + + /////////////////////////////////////////////////////////////////////////// + // if there are any that weren't found, try to look them up now by label // + /////////////////////////////////////////////////////////////////////////// + if(!valuesNotFound.isEmpty()) + { + searchPossibleValueSourceInput = new SearchPossibleValueSourceInput(); + searchPossibleValueSourceInput.setPossibleValueSourceName(field.getPossibleValueSourceName()); + ArrayList labelList = new ArrayList<>(valuesNotFound); + searchPossibleValueSourceInput.setLabelList(labelList); + searchPossibleValueSourceInput.setLimit(valuesNotFound.size()); + + LOG.debug("Searching possible value source by labels during bulk load mapping", logPair("pvsName", field.getPossibleValueSourceName()), logPair("noOfLabels", labelList.size()), logPair("firstLabel", () -> labelList.get(0))); + searchPossibleValueSourceOutput = new SearchPossibleValueSourceAction().execute(searchPossibleValueSourceInput); + for(QPossibleValue possibleValue : searchPossibleValueSourceOutput.getResults()) + { + valuesFound.put(possibleValue.getLabel(), possibleValue); + valuesNotFound.remove(possibleValue.getLabel()); + } + } + + //////////////////////////////////////////////////////////////////////////////// + // for each record now, either set a usable value (e.g., a PV.id) or an error // + //////////////////////////////////////////////////////////////////////////////// + for(Map.Entry> entry : fieldPossibleValueToRecordMap.entrySet()) + { + String value = entry.getKey(); + for(QRecord record : entry.getValue()) + { + if(valuesFound.containsKey(value)) + { + record.setValue(field.getName(), valuesFound.get(value).getId()); + } + else + { + record.addError(new BulkLoadPossibleValueError(associationNamePrefixForFields + field.getName(), value, tableLabelPrefix + field.getLabel())); + } + } + } + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadValueTypeError.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadValueTypeError.java index a7e6f371..6bee0449 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadValueTypeError.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadValueTypeError.java @@ -24,17 +24,15 @@ package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.map import java.io.Serializable; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; -import com.kingsrook.qqq.backend.core.model.statusmessages.BadInputStatusMessage; /******************************************************************************* ** Specialized error for records, for bulk-load use-cases, where we want to ** report back info to the user about the field & value. *******************************************************************************/ -public class BulkLoadValueTypeError extends BadInputStatusMessage +public class BulkLoadValueTypeError extends AbstractBulkLoadRollableValueError { private final String fieldLabel; - private final String fieldName; private final Serializable value; private final QFieldType type; @@ -47,7 +45,6 @@ public class BulkLoadValueTypeError extends BadInputStatusMessage public BulkLoadValueTypeError(String fieldName, Serializable value, QFieldType type, String fieldLabel) { super("Value [" + value + "] for field [" + fieldLabel + "] could not be converted to type [" + type + "]"); - this.fieldName = fieldName; this.value = value; this.type = type; this.fieldLabel = fieldLabel; @@ -58,6 +55,7 @@ public class BulkLoadValueTypeError extends BadInputStatusMessage /*************************************************************************** ** ***************************************************************************/ + @Override public String getMessageToUseAsProcessSummaryRollupKey() { return ("Cannot convert value for field [" + fieldLabel + "] to type [" + type.getMixedCaseLabel() + "]"); @@ -69,6 +67,7 @@ public class BulkLoadValueTypeError extends BadInputStatusMessage ** Getter for value ** *******************************************************************************/ + @Override public Serializable getValue() { return value; diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/FlatRowsToRecord.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/FlatRowsToRecord.java index 243496ef..310fd8db 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/FlatRowsToRecord.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/FlatRowsToRecord.java @@ -70,7 +70,7 @@ public class FlatRowsToRecord implements RowsToRecordInterface rs.add(record); } - ValueMapper.valueMapping(rs, mapping, table); + BulkLoadValueMapper.valueMapping(rs, mapping, table); return (rs); } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/TallRowsToRecord.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/TallRowsToRecord.java index 8e0443d3..84ab1fa7 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/TallRowsToRecord.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/TallRowsToRecord.java @@ -138,7 +138,7 @@ public class TallRowsToRecord implements RowsToRecordInterface rs.add(record); } - ValueMapper.valueMapping(rs, mapping, table); + BulkLoadValueMapper.valueMapping(rs, mapping, table); return (rs); } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/ValueMapper.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/ValueMapper.java deleted file mode 100644 index 44a37f6a..00000000 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/ValueMapper.java +++ /dev/null @@ -1,134 +0,0 @@ -/* - * QQQ - Low-code Application Framework for Engineers. - * Copyright (C) 2021-2024. Kingsrook, LLC - * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States - * contact@kingsrook.com - * https://github.com/Kingsrook/ - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping; - - -import java.io.Serializable; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import com.kingsrook.qqq.backend.core.context.QContext; -import com.kingsrook.qqq.backend.core.exceptions.QException; -import com.kingsrook.qqq.backend.core.logging.QLogger; -import com.kingsrook.qqq.backend.core.model.data.QRecord; -import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; -import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; -import com.kingsrook.qqq.backend.core.model.metadata.tables.Association; -import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; -import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkInsertMapping; -import com.kingsrook.qqq.backend.core.utils.CollectionUtils; -import com.kingsrook.qqq.backend.core.utils.StringUtils; -import com.kingsrook.qqq.backend.core.utils.ValueUtils; - - -/******************************************************************************* - ** - *******************************************************************************/ -public class ValueMapper -{ - private static final QLogger LOG = QLogger.getLogger(ValueMapper.class); - - - - /*************************************************************************** - ** - ***************************************************************************/ - public static void valueMapping(List records, BulkInsertMapping mapping, QTableMetaData table) throws QException - { - valueMapping(records, mapping, table, null); - } - - - - /*************************************************************************** - ** - ***************************************************************************/ - private static void valueMapping(List records, BulkInsertMapping mapping, QTableMetaData table, String associationNameChain) throws QException - { - if(CollectionUtils.nullSafeIsEmpty(records)) - { - return; - } - - String associationNamePrefixForFields = StringUtils.hasContent(associationNameChain) ? associationNameChain + "." : ""; - String tableLabelPrefix = StringUtils.hasContent(associationNameChain) ? table.getLabel() + ": " : ""; - - Map> mappingForTable = mapping.getFieldNameToValueMappingForTable(associationNameChain); - for(QRecord record : records) - { - for(Map.Entry valueEntry : record.getValues().entrySet()) - { - QFieldMetaData field = table.getField(valueEntry.getKey()); - Serializable value = valueEntry.getValue(); - - /////////////////// - // value mappin' // - /////////////////// - if(mappingForTable.containsKey(field.getName()) && value != null) - { - Serializable mappedValue = mappingForTable.get(field.getName()).get(ValueUtils.getValueAsString(value)); - if(mappedValue != null) - { - value = mappedValue; - } - } - - ///////////////////// - // type convertin' // - ///////////////////// - if(value != null) - { - QFieldType type = field.getType(); - try - { - value = ValueUtils.getValueAsFieldType(type, value); - } - catch(Exception e) - { - record.addError(new BulkLoadValueTypeError(associationNamePrefixForFields + field.getName(), value, type, tableLabelPrefix + field.getLabel())); - } - } - - record.setValue(field.getName(), value); - } - - ////////////////////////////////////// - // recursively process associations // - ////////////////////////////////////// - for(Map.Entry> entry : record.getAssociatedRecords().entrySet()) - { - String associationName = entry.getKey(); - Optional association = table.getAssociations().stream().filter(a -> a.getName().equals(associationName)).findFirst(); - if(association.isPresent()) - { - QTableMetaData associatedTable = QContext.getQInstance().getTable(association.get().getAssociatedTableName()); - valueMapping(entry.getValue(), mapping, associatedTable, StringUtils.hasContent(associationNameChain) ? associationNameChain + "." + associationName : associationName); - } - else - { - throw new QException("Missing association [" + associationName + "] on table [" + table.getName() + "]"); - } - } - } - } - -} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMapping.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMapping.java index e79dd2c2..c4231ea7 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMapping.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMapping.java @@ -75,7 +75,7 @@ public class WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMapping implem rs.add(record); } - ValueMapper.valueMapping(rs, mapping, table); + BulkLoadValueMapper.valueMapping(rs, mapping, table); return (rs); } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithSpreadMapping.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithSpreadMapping.java index 7b215d2e..e39e95c0 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithSpreadMapping.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithSpreadMapping.java @@ -86,7 +86,7 @@ public class WideRowsToRecordWithSpreadMapping implements RowsToRecordInterface rs.add(record); } - ValueMapper.valueMapping(rs, mapping, table); + BulkLoadValueMapper.valueMapping(rs, mapping, table); return (rs); } diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/ValueMapperTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadValueMapperTest.java similarity index 96% rename from qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/ValueMapperTest.java rename to qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadValueMapperTest.java index 4348cb2b..687f1f62 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/ValueMapperTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadValueMapperTest.java @@ -41,7 +41,7 @@ import static org.assertj.core.api.Assertions.assertThat; /******************************************************************************* ** Unit test for ValueMapper *******************************************************************************/ -class ValueMapperTest extends BaseTest +class BulkLoadValueMapperTest extends BaseTest { /******************************************************************************* @@ -98,7 +98,7 @@ class ValueMapperTest extends BaseTest ); JSONObject expectedJson = recordToJson(expectedRecord); - ValueMapper.valueMapping(List.of(inputRecord), mapping, QContext.getQInstance().getTable(TestUtils.TABLE_NAME_ORDER)); + BulkLoadValueMapper.valueMapping(List.of(inputRecord), mapping, QContext.getQInstance().getTable(TestUtils.TABLE_NAME_ORDER)); JSONObject actualJson = recordToJson(inputRecord); System.out.println("Before"); diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/FlatRowsToRecordTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/FlatRowsToRecordTest.java index fd1f70c4..5237fcf5 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/FlatRowsToRecordTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/FlatRowsToRecordTest.java @@ -35,6 +35,7 @@ import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mode import com.kingsrook.qqq.backend.core.utils.TestUtils; import com.kingsrook.qqq.backend.core.utils.collections.ListBuilder; import org.junit.jupiter.api.Test; +import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -102,6 +103,8 @@ class FlatRowsToRecordTest extends BaseTest assertEquals("Row 5", records.get(0).getBackendDetail("rowNos")); } + + /******************************************************************************* ** *******************************************************************************/ @@ -205,6 +208,48 @@ class FlatRowsToRecordTest extends BaseTest + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testPossibleValueMappings() throws QException + { + TestFileToRows fileToRows = new TestFileToRows(List.of( + new Serializable[] { "id", "firstName", "Last Name", "Home State" }, + new Serializable[] { 1, "Homer", "Simpson", 1 }, + new Serializable[] { 2, "Marge", "Simpson", "MO" }, + new Serializable[] { 3, "Bart", "Simpson", null }, + new Serializable[] { 4, "Ned", "Flanders", "Not a state" }, + new Serializable[] { 5, "Mr.", "Burns", 5 } + )); + + BulkLoadFileRow header = fileToRows.next(); + + FlatRowsToRecord rowsToRecord = new FlatRowsToRecord(); + + BulkInsertMapping mapping = new BulkInsertMapping() + .withFieldNameToHeaderNameMap(Map.of( + "firstName", "firstName", + "lastName", "Last Name", + "homeStateId", "Home State" + )) + .withTableName(TestUtils.TABLE_NAME_PERSON) + .withHasHeaderRow(true); + + List records = rowsToRecord.nextPage(fileToRows, header, mapping, Integer.MAX_VALUE); + assertEquals(5, records.size()); + assertEquals(List.of("Homer", "Marge", "Bart", "Ned", "Mr."), getValues(records, "firstName")); + assertEquals(ListBuilder.of(1, 2, null, "Not a state", 5), getValues(records, "homeStateId")); + + assertThat(records.get(0).getErrors()).isNullOrEmpty(); + assertThat(records.get(1).getErrors()).isNullOrEmpty(); + assertThat(records.get(2).getErrors()).isNullOrEmpty(); + assertThat(records.get(3).getErrors()).hasSize(1).element(0).matches(e -> e.getMessage().contains("not a valid option")); + assertThat(records.get(4).getErrors()).hasSize(1).element(0).matches(e -> e.getMessage().contains("not a valid option")); + } + + + /*************************************************************************** ** ***************************************************************************/