QRunIO/qqq
2
0
Fork 0
mirror of https://github.com/Kingsrook/qqq.git synced 2025-07-17 20:50:44 +00:00
Code Issues Packages Projects Releases Wiki Activity

CE-1955 Add mapping and validation of possible-values; refactor error classes some for rollup possible value errors

Browse Source
This commit is contained in:
darin.kelkhoff
2024-12-03 08:50:05 -06:00
parent 8ec6ccd691
commit 0e93b90270
12 changed files with 428 additions and 153 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTransformStep.java
View File

@ -56,8 +56,8 @@ import com.kingsrook.qqq.backend.core.model.metadata.tables.QFieldSection;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.tables.UniqueKey;
import com.kingsrook.qqq.backend.core.model.statusmessages.QErrorMessage;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping.AbstractBulkLoadRollableValueError;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping.BulkLoadRecordUtils;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping.BulkLoadValueTypeError;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkInsertMapping;
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.AbstractTransformStep;
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.LoadViaInsertStep;
@ -210,10 +210,10 @@ public class BulkInsertTransformStep extends AbstractTransformStep
{
for(QErrorMessage error : record.getErrors())
{
if(error instanceof BulkLoadValueTypeError blvte)
if(error instanceof AbstractBulkLoadRollableValueError rollableValueError)
{
processSummaryWarningsAndErrorsRollup.addError(blvte.getMessageToUseAsProcessSummaryRollupKey(), null);
addToErrorToExampleRowValueMap(blvte, record);
processSummaryWarningsAndErrorsRollup.addError(rollableValueError.getMessageToUseAsProcessSummaryRollupKey(), null);
addToErrorToExampleRowValueMap(rollableValueError, record);
}
else
{
@ -348,14 +348,14 @@ public class BulkInsertTransformStep extends AbstractTransformStep
/***************************************************************************
**
***************************************************************************/
private void addToErrorToExampleRowValueMap(BulkLoadValueTypeError bulkLoadValueTypeError, QRecord record)
private void addToErrorToExampleRowValueMap(AbstractBulkLoadRollableValueError bulkLoadRollableValueError, QRecord record)
{
String message = bulkLoadValueTypeError.getMessageToUseAsProcessSummaryRollupKey();
String message = bulkLoadRollableValueError.getMessageToUseAsProcessSummaryRollupKey();
List<RowValue> rowValues = errorToExampleRowValueMap.computeIfAbsent(message, k -> new ArrayList<>());
if(rowValues.size() < EXAMPLE_ROW_LIMIT)
{
rowValues.add(new RowValue(bulkLoadValueTypeError, record));
rowValues.add(new RowValue(bulkLoadRollableValueError, record));
}
}
@ -550,9 +550,9 @@ public class BulkInsertTransformStep extends AbstractTransformStep
/***************************************************************************
**
***************************************************************************/
public RowValue(BulkLoadValueTypeError bulkLoadValueTypeError, QRecord record)
public RowValue(AbstractBulkLoadRollableValueError bulkLoadRollableValueError, QRecord record)
{
this(BulkLoadRecordUtils.getRowNosString(record), ValueUtils.getValueAsString(bulkLoadValueTypeError.getValue()));
this(BulkLoadRecordUtils.getRowNosString(record), ValueUtils.getValueAsString(bulkLoadRollableValueError.getValue()));
}

55
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/AbstractBulkLoadRollableValueError.java Normal file
View File

@ -0,0 +1,55 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2024. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping;
import java.io.Serializable;
import com.kingsrook.qqq.backend.core.model.statusmessages.BadInputStatusMessage;
/*******************************************************************************
**
*******************************************************************************/
public abstract class AbstractBulkLoadRollableValueError extends BadInputStatusMessage
{
/*******************************************************************************
** Constructor
**
*******************************************************************************/
public AbstractBulkLoadRollableValueError(String message)
{
super(message);
}
/***************************************************************************
**
***************************************************************************/
public abstract String getMessageToUseAsProcessSummaryRollupKey();
/***************************************************************************
**
***************************************************************************/
public abstract Serializable getValue();
}

72
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadPossibleValueError.java Normal file
View File

@ -0,0 +1,72 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2024. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping;
import java.io.Serializable;
/*******************************************************************************
** Specialized error for records, for bulk-load use-cases, where we want to
** report back info to the user about the field & value.
*******************************************************************************/
public class BulkLoadPossibleValueError extends AbstractBulkLoadRollableValueError
{
private final String fieldLabel;
private final Serializable value;
/*******************************************************************************
** Constructor
**
*******************************************************************************/
public BulkLoadPossibleValueError(String fieldName, Serializable value, String fieldLabel)
{
super("Value [" + value + "] for field [" + fieldLabel + "] is not a valid option");
this.value = value;
this.fieldLabel = fieldLabel;
}
/***************************************************************************
**
***************************************************************************/
@Override
public String getMessageToUseAsProcessSummaryRollupKey()
{
return ("Unrecognized value for field [" + fieldLabel + "]");
}
/*******************************************************************************
** Getter for value
**
*******************************************************************************/
@Override
public Serializable getValue()
{
return value;
}
}

238
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadValueMapper.java Normal file
View File

@ -0,0 +1,238 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2024. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import com.kingsrook.qqq.backend.core.actions.values.SearchPossibleValueSourceAction;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.logging.QLogger;
import com.kingsrook.qqq.backend.core.model.actions.values.SearchPossibleValueSourceInput;
import com.kingsrook.qqq.backend.core.model.actions.values.SearchPossibleValueSourceOutput;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValue;
import com.kingsrook.qqq.backend.core.model.metadata.tables.Association;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkInsertMapping;
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
import com.kingsrook.qqq.backend.core.utils.ListingHash;
import com.kingsrook.qqq.backend.core.utils.StringUtils;
import com.kingsrook.qqq.backend.core.utils.ValueUtils;
import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
/*******************************************************************************
**
*******************************************************************************/
public class BulkLoadValueMapper
{
private static final QLogger LOG = QLogger.getLogger(BulkLoadValueMapper.class);
/***************************************************************************
**
***************************************************************************/
public static void valueMapping(List<QRecord> records, BulkInsertMapping mapping, QTableMetaData table) throws QException
{
valueMapping(records, mapping, table, null);
}
/***************************************************************************
**
***************************************************************************/
private static void valueMapping(List<QRecord> records, BulkInsertMapping mapping, QTableMetaData table, String associationNameChain) throws QException
{
if(CollectionUtils.nullSafeIsEmpty(records))
{
return;
}
String associationNamePrefixForFields = StringUtils.hasContent(associationNameChain) ? associationNameChain + "." : "";
String tableLabelPrefix = StringUtils.hasContent(associationNameChain) ? table.getLabel() + ": " : "";
Map<String, ListingHash<String, QRecord>> possibleValueToRecordMap = new HashMap<>();
Map<String, Map<String, Serializable>> mappingForTable = mapping.getFieldNameToValueMappingForTable(associationNameChain);
for(QRecord record : records)
{
for(Map.Entry<String, Serializable> valueEntry : record.getValues().entrySet())
{
QFieldMetaData field = table.getField(valueEntry.getKey());
Serializable value = valueEntry.getValue();
///////////////////
// value mappin' //
///////////////////
if(mappingForTable.containsKey(field.getName()) && value != null)
{
Serializable mappedValue = mappingForTable.get(field.getName()).get(ValueUtils.getValueAsString(value));
if(mappedValue != null)
{
value = mappedValue;
}
}
/////////////////////
// type convertin' //
/////////////////////
if(value != null && !"".equals(value))
{
if(StringUtils.hasContent(field.getPossibleValueSourceName()))
{
ListingHash<String, QRecord> fieldPossibleValueToRecordMap = possibleValueToRecordMap.computeIfAbsent(field.getName(), k -> new ListingHash<>());
fieldPossibleValueToRecordMap.add(ValueUtils.getValueAsString(value), record);
}
else
{
QFieldType type = field.getType();
try
{
value = ValueUtils.getValueAsFieldType(type, value);
}
catch(Exception e)
{
record.addError(new BulkLoadValueTypeError(associationNamePrefixForFields + field.getName(), value, type, tableLabelPrefix + field.getLabel()));
}
}
}
record.setValue(field.getName(), value);
}
//////////////////////////////////////
// recursively process associations //
//////////////////////////////////////
for(Map.Entry<String, List<QRecord>> entry : record.getAssociatedRecords().entrySet())
{
String associationName = entry.getKey();
Optional<Association> association = table.getAssociations().stream().filter(a -> a.getName().equals(associationName)).findFirst();
if(association.isPresent())
{
QTableMetaData associatedTable = QContext.getQInstance().getTable(association.get().getAssociatedTableName());
valueMapping(entry.getValue(), mapping, associatedTable, StringUtils.hasContent(associationNameChain) ? associationNameChain + "." + associationName : associationName);
}
else
{
throw new QException("Missing association [" + associationName + "] on table [" + table.getName() + "]");
}
}
}
//////////////////////////////////////////
// look up and validate possible values //
//////////////////////////////////////////
for(Map.Entry<String, ListingHash<String, QRecord>> entry : possibleValueToRecordMap.entrySet())
{
String fieldName = entry.getKey();
QFieldMetaData field = table.getField(fieldName);
ListingHash<String, QRecord> fieldPossibleValueToRecordMap = possibleValueToRecordMap.get(fieldName);
handlePossibleValues(field, fieldPossibleValueToRecordMap, associationNamePrefixForFields, tableLabelPrefix);
}
}
/***************************************************************************
**
***************************************************************************/
private static void handlePossibleValues(QFieldMetaData field, ListingHash<String, QRecord> fieldPossibleValueToRecordMap, String associationNamePrefixForFields, String tableLabelPrefix) throws QException
{
Set<String> values = fieldPossibleValueToRecordMap.keySet();
Map<String, QPossibleValue<?>> valuesFound = new HashMap<>();
Set<String> valuesNotFound = new HashSet<>(values);
////////////////////////////////////////////////////////
// do a search, trying to use all given values as ids //
////////////////////////////////////////////////////////
SearchPossibleValueSourceInput searchPossibleValueSourceInput = new SearchPossibleValueSourceInput();
searchPossibleValueSourceInput.setPossibleValueSourceName(field.getPossibleValueSourceName());
ArrayList<Serializable> idList = new ArrayList<>(values);
searchPossibleValueSourceInput.setIdList(idList);
searchPossibleValueSourceInput.setLimit(values.size());
LOG.debug("Searching possible value source by ids during bulk load mapping", logPair("pvsName", field.getPossibleValueSourceName()), logPair("noOfIds", idList.size()), logPair("firstId", () -> idList.get(0)));
SearchPossibleValueSourceOutput searchPossibleValueSourceOutput = new SearchPossibleValueSourceAction().execute(searchPossibleValueSourceInput);
////////////////////////////////////////////////////////////////////////////////////////////////////
// for each possible value found, remove it from the set of ones not-found, and store it as a hit //
////////////////////////////////////////////////////////////////////////////////////////////////////
for(QPossibleValue<?> possibleValue : searchPossibleValueSourceOutput.getResults())
{
String valueAsString = ValueUtils.getValueAsString(possibleValue.getId());
valuesFound.put(valueAsString, possibleValue);
valuesNotFound.remove(valueAsString);
}
///////////////////////////////////////////////////////////////////////////
// if there are any that weren't found, try to look them up now by label //
///////////////////////////////////////////////////////////////////////////
if(!valuesNotFound.isEmpty())
{
searchPossibleValueSourceInput = new SearchPossibleValueSourceInput();
searchPossibleValueSourceInput.setPossibleValueSourceName(field.getPossibleValueSourceName());
ArrayList<String> labelList = new ArrayList<>(valuesNotFound);
searchPossibleValueSourceInput.setLabelList(labelList);
searchPossibleValueSourceInput.setLimit(valuesNotFound.size());
LOG.debug("Searching possible value source by labels during bulk load mapping", logPair("pvsName", field.getPossibleValueSourceName()), logPair("noOfLabels", labelList.size()), logPair("firstLabel", () -> labelList.get(0)));
searchPossibleValueSourceOutput = new SearchPossibleValueSourceAction().execute(searchPossibleValueSourceInput);
for(QPossibleValue<?> possibleValue : searchPossibleValueSourceOutput.getResults())
{
valuesFound.put(possibleValue.getLabel(), possibleValue);
valuesNotFound.remove(possibleValue.getLabel());
}
}
////////////////////////////////////////////////////////////////////////////////
// for each record now, either set a usable value (e.g., a PV.id) or an error //
////////////////////////////////////////////////////////////////////////////////
for(Map.Entry<String, List<QRecord>> entry : fieldPossibleValueToRecordMap.entrySet())
{
String value = entry.getKey();
for(QRecord record : entry.getValue())
{
if(valuesFound.containsKey(value))
{
record.setValue(field.getName(), valuesFound.get(value).getId());
}
else
{
record.addError(new BulkLoadPossibleValueError(associationNamePrefixForFields + field.getName(), value, tableLabelPrefix + field.getLabel()));
}
}
}
}
}

7
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadValueTypeError.java
View File

@ -24,17 +24,15 @@ package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.map
import java.io.Serializable;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
import com.kingsrook.qqq.backend.core.model.statusmessages.BadInputStatusMessage;
/*******************************************************************************
** Specialized error for records, for bulk-load use-cases, where we want to
** report back info to the user about the field & value.
*******************************************************************************/
public class BulkLoadValueTypeError extends BadInputStatusMessage
public class BulkLoadValueTypeError extends AbstractBulkLoadRollableValueError
{
private final String fieldLabel;
private final String fieldName;
private final Serializable value;
private final QFieldType type;
@ -47,7 +45,6 @@ public class BulkLoadValueTypeError extends BadInputStatusMessage
public BulkLoadValueTypeError(String fieldName, Serializable value, QFieldType type, String fieldLabel)
{
super("Value [" + value + "] for field [" + fieldLabel + "] could not be converted to type [" + type + "]");
this.fieldName = fieldName;
this.value = value;
this.type = type;
this.fieldLabel = fieldLabel;
@ -58,6 +55,7 @@ public class BulkLoadValueTypeError extends BadInputStatusMessage
/***************************************************************************
**
***************************************************************************/
@Override
public String getMessageToUseAsProcessSummaryRollupKey()
{
return ("Cannot convert value for field [" + fieldLabel + "] to type [" + type.getMixedCaseLabel() + "]");
@ -69,6 +67,7 @@ public class BulkLoadValueTypeError extends BadInputStatusMessage
** Getter for value
**
*******************************************************************************/
@Override
public Serializable getValue()
{
return value;

2
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/FlatRowsToRecord.java
View File

@ -70,7 +70,7 @@ public class FlatRowsToRecord implements RowsToRecordInterface
rs.add(record);
}
ValueMapper.valueMapping(rs, mapping, table);
BulkLoadValueMapper.valueMapping(rs, mapping, table);
return (rs);
}

2
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/TallRowsToRecord.java
View File

@ -138,7 +138,7 @@ public class TallRowsToRecord implements RowsToRecordInterface
rs.add(record);
}
ValueMapper.valueMapping(rs, mapping, table);
BulkLoadValueMapper.valueMapping(rs, mapping, table);
return (rs);
}

134
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/ValueMapper.java
View File

@ -1,134 +0,0 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2024. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping;
import java.io.Serializable;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.logging.QLogger;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
import com.kingsrook.qqq.backend.core.model.metadata.tables.Association;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkInsertMapping;
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
import com.kingsrook.qqq.backend.core.utils.StringUtils;
import com.kingsrook.qqq.backend.core.utils.ValueUtils;
/*******************************************************************************
**
*******************************************************************************/
public class ValueMapper
{
private static final QLogger LOG = QLogger.getLogger(ValueMapper.class);
/***************************************************************************
**
***************************************************************************/
public static void valueMapping(List<QRecord> records, BulkInsertMapping mapping, QTableMetaData table) throws QException
{
valueMapping(records, mapping, table, null);
}
/***************************************************************************
**
***************************************************************************/
private static void valueMapping(List<QRecord> records, BulkInsertMapping mapping, QTableMetaData table, String associationNameChain) throws QException
{
if(CollectionUtils.nullSafeIsEmpty(records))
{
return;
}
String associationNamePrefixForFields = StringUtils.hasContent(associationNameChain) ? associationNameChain + "." : "";
String tableLabelPrefix = StringUtils.hasContent(associationNameChain) ? table.getLabel() + ": " : "";
Map<String, Map<String, Serializable>> mappingForTable = mapping.getFieldNameToValueMappingForTable(associationNameChain);
for(QRecord record : records)
{
for(Map.Entry<String, Serializable> valueEntry : record.getValues().entrySet())
{
QFieldMetaData field = table.getField(valueEntry.getKey());
Serializable value = valueEntry.getValue();
///////////////////
// value mappin' //
///////////////////
if(mappingForTable.containsKey(field.getName()) && value != null)
{
Serializable mappedValue = mappingForTable.get(field.getName()).get(ValueUtils.getValueAsString(value));
if(mappedValue != null)
{
value = mappedValue;
}
}
/////////////////////
// type convertin' //
/////////////////////
if(value != null)
{
QFieldType type = field.getType();
try
{
value = ValueUtils.getValueAsFieldType(type, value);
}
catch(Exception e)
{
record.addError(new BulkLoadValueTypeError(associationNamePrefixForFields + field.getName(), value, type, tableLabelPrefix + field.getLabel()));
}
}
record.setValue(field.getName(), value);
}
//////////////////////////////////////
// recursively process associations //
//////////////////////////////////////
for(Map.Entry<String, List<QRecord>> entry : record.getAssociatedRecords().entrySet())
{
String associationName = entry.getKey();
Optional<Association> association = table.getAssociations().stream().filter(a -> a.getName().equals(associationName)).findFirst();
if(association.isPresent())
{
QTableMetaData associatedTable = QContext.getQInstance().getTable(association.get().getAssociatedTableName());
valueMapping(entry.getValue(), mapping, associatedTable, StringUtils.hasContent(associationNameChain) ? associationNameChain + "." + associationName : associationName);
}
else
{
throw new QException("Missing association [" + associationName + "] on table [" + table.getName() + "]");
}
}
}
}
}

2
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMapping.java
View File

@ -75,7 +75,7 @@ public class WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMapping implem
rs.add(record);
}
ValueMapper.valueMapping(rs, mapping, table);
BulkLoadValueMapper.valueMapping(rs, mapping, table);
return (rs);
}

2
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithSpreadMapping.java
View File

@ -86,7 +86,7 @@ public class WideRowsToRecordWithSpreadMapping implements RowsToRecordInterface
rs.add(record);
}
ValueMapper.valueMapping(rs, mapping, table);
BulkLoadValueMapper.valueMapping(rs, mapping, table);
return (rs);
}

4
qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/ValueMapperTest.java → qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadValueMapperTest.java
View File

@ -41,7 +41,7 @@ import static org.assertj.core.api.Assertions.assertThat;
/*******************************************************************************
** Unit test for ValueMapper
*******************************************************************************/
class ValueMapperTest extends BaseTest
class BulkLoadValueMapperTest extends BaseTest
{
/*******************************************************************************
@ -98,7 +98,7 @@ class ValueMapperTest extends BaseTest
);
JSONObject expectedJson = recordToJson(expectedRecord);
ValueMapper.valueMapping(List.of(inputRecord), mapping, QContext.getQInstance().getTable(TestUtils.TABLE_NAME_ORDER));
BulkLoadValueMapper.valueMapping(List.of(inputRecord), mapping, QContext.getQInstance().getTable(TestUtils.TABLE_NAME_ORDER));
JSONObject actualJson = recordToJson(inputRecord);
System.out.println("Before");

45
qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/FlatRowsToRecordTest.java
View File

@ -35,6 +35,7 @@ import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mode
import com.kingsrook.qqq.backend.core.utils.TestUtils;
import com.kingsrook.qqq.backend.core.utils.collections.ListBuilder;
import org.junit.jupiter.api.Test;
import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.jupiter.api.Assertions.assertEquals;
@ -102,6 +103,8 @@ class FlatRowsToRecordTest extends BaseTest
assertEquals("Row 5", records.get(0).getBackendDetail("rowNos"));
}
/*******************************************************************************
**
*******************************************************************************/
@ -205,6 +208,48 @@ class FlatRowsToRecordTest extends BaseTest
/*******************************************************************************
**
*******************************************************************************/
@Test
void testPossibleValueMappings() throws QException
{
TestFileToRows fileToRows = new TestFileToRows(List.of(
new Serializable[] { "id", "firstName", "Last Name", "Home State" },
new Serializable[] { 1, "Homer", "Simpson", 1 },
new Serializable[] { 2, "Marge", "Simpson", "MO" },
new Serializable[] { 3, "Bart", "Simpson", null },
new Serializable[] { 4, "Ned", "Flanders", "Not a state" },
new Serializable[] { 5, "Mr.", "Burns", 5 }
));
BulkLoadFileRow header = fileToRows.next();
FlatRowsToRecord rowsToRecord = new FlatRowsToRecord();
BulkInsertMapping mapping = new BulkInsertMapping()
.withFieldNameToHeaderNameMap(Map.of(
"firstName", "firstName",
"lastName", "Last Name",
"homeStateId", "Home State"
))
.withTableName(TestUtils.TABLE_NAME_PERSON)
.withHasHeaderRow(true);
List<QRecord> records = rowsToRecord.nextPage(fileToRows, header, mapping, Integer.MAX_VALUE);
assertEquals(5, records.size());
assertEquals(List.of("Homer", "Marge", "Bart", "Ned", "Mr."), getValues(records, "firstName"));
assertEquals(ListBuilder.of(1, 2, null, "Not a state", 5), getValues(records, "homeStateId"));
assertThat(records.get(0).getErrors()).isNullOrEmpty();
assertThat(records.get(1).getErrors()).isNullOrEmpty();
assertThat(records.get(2).getErrors()).isNullOrEmpty();
assertThat(records.get(3).getErrors()).hasSize(1).element(0).matches(e -> e.getMessage().contains("not a valid option"));
assertThat(records.get(4).getErrors()).hasSize(1).element(0).matches(e -> e.getMessage().contains("not a valid option"));
}
/***************************************************************************
**
***************************************************************************/