QRunIO/qqq
2
0
Fork 0
mirror of https://github.com/Kingsrook/qqq.git synced 2025-07-20 14:10:44 +00:00
Code Issues Packages Projects Releases Wiki Activity

CE-1955 Add handling for associations w/ some vs. all values coming from defaults instead of columns;

Browse Source
This commit is contained in:
darin.kelkhoff
2025-01-03 12:57:49 -06:00
parent 048ee2e332
commit 3fda1a1eda
5 changed files with 233 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files

21
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/TallRowsToRecord.java
View File

@ -32,6 +32,7 @@ import java.util.Optional;
import com.google.gson.reflect.TypeToken;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.logging.QLogger;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.tables.Association;
@ -43,6 +44,7 @@ import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
import com.kingsrook.qqq.backend.core.utils.Pair;
import com.kingsrook.qqq.backend.core.utils.StringUtils;
import com.kingsrook.qqq.backend.core.utils.memoization.Memoization;
import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
/*******************************************************************************
@ -50,6 +52,8 @@ import com.kingsrook.qqq.backend.core.utils.memoization.Memoization;
*******************************************************************************/
public class TallRowsToRecord implements RowsToRecordInterface
{
private static final QLogger LOG = QLogger.getLogger(TallRowsToRecord.class);
private Memoization<Pair<String, String>, Boolean> shouldProcesssAssociationMemoization = new Memoization<>();
@ -166,9 +170,9 @@ public class TallRowsToRecord implements RowsToRecordInterface
Map<String, Integer> fieldIndexes = mapping.getFieldIndexes(table, associationNameChain, headerRow);
//////////////////////////////////////////////////////
// get all rows for the main table from the 0th row //
//////////////////////////////////////////////////////
////////////////////////////////////////////////////////
// get all values for the main table from the 0th row //
////////////////////////////////////////////////////////
BulkLoadFileRow row = rows.get(0);
for(QFieldMetaData field : table.getFields().values())
{
@ -258,6 +262,17 @@ public class TallRowsToRecord implements RowsToRecordInterface
// throw (new QException("Missing group-by-index(es) for association: " + associationNameChainForRecursiveCalls));
}
if(CollectionUtils.nullSafeIsEmpty(groupByIndexes))
{
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// special case here - if there are no group-by-indexes for the row, it means there are no fields coming from columns in the file. //
// but, if any fields for this association have a default value - then - make a row using just default values. //
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
LOG.info("Handling case of an association with no fields from the file, but rather only defaults", logPair("associationName", associationName));
rs.add(makeRecordFromRows(table, associationNameChainForRecursiveCalls, mapping, headerRow, List.of(row)));
break;
}
List<Serializable> rowGroupByValues = getGroupByValues(row, groupByIndexes);
if(rowGroupByValues == null)
{

80
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMapping.java
View File

@ -22,6 +22,7 @@
package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
@ -71,7 +72,7 @@ public class WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMapping implem
while(fileToRowsInterface.hasNext() && rs.size() < limit)
{
BulkLoadFileRow row = fileToRowsInterface.next();
QRecord record = makeRecordFromRow(mapping, table, "", row, fieldIndexes, headerRow, new ArrayList<>());
QRecord record = makeRecordFromRow(mapping, table, "", row, fieldIndexes, headerRow, new ArrayList<>(), false);
rs.add(record);
}
@ -84,8 +85,42 @@ public class WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMapping implem
/***************************************************************************
** may return null, if there were no values in the row for this (sub-wide) record.
** more specifically:
**
** the param `rowOfOnlyDefaultValues` - should be false for the header table,
** and true for an association iff all mapped fields are using 'default values'
** (e.g., not values from the file).
**
** So this method will return null, indicating "no child row to build" if:
** - when doing a rowOfOnlyDefaultValues - only if there actually weren't any
** default values, which, probably never happens!
** - else (doing a row with at least 1 value from the file) - then, null is
** returned if there were NO values from the file.
**
** The goal here is to support these cases:
**
** Case A (a row of not only-default-values):
** - lineItem.sku,0 = column: sku1
** - lineItem.qty,0 = column: qty1
** - lineItem.lineNo,0 = Default: 1
** - lineItem.sku,1 = column: sku2
** - lineItem.qty,1 = column: qty2
** - lineItem.lineNo,1 = Default: 2
** then a file row with no values for sku2 & qty2 - we don't want a row
** in that case (which would only have the default value of lineNo=2)
**
** Case B (a row of only-default-values):
** - lineItem.sku,0 = column: sku1
** - lineItem.qty,0 = column: qty1
** - lineItem.lineNo,0 = Default: 1
** - lineItem.sku,1 = Default: SUPPLEMENT
** - lineItem.qty,1 = Default: 1
** - lineItem.lineNo,1 = Default: 2
** we want every parent (order) to include a 2nd line item - with 3
** default values (sku=SUPPLEMENT, qty=q, lineNo=2).
**
***************************************************************************/
private QRecord makeRecordFromRow(BulkInsertMapping mapping, QTableMetaData table, String associationNameChain, BulkLoadFileRow row, Map<String, Integer> fieldIndexes, BulkLoadFileRow headerRow, List<Integer> wideAssociationIndexes) throws QException
private QRecord makeRecordFromRow(BulkInsertMapping mapping, QTableMetaData table, String associationNameChain, BulkLoadFileRow row, Map<String, Integer> fieldIndexes, BulkLoadFileRow headerRow, List<Integer> wideAssociationIndexes, boolean rowOfOnlyDefaultValues) throws QException
{
//////////////////////////////////////////////////////
// start by building the record with its own fields //
@ -93,15 +128,35 @@ public class WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMapping implem
QRecord record = new QRecord();
BulkLoadRecordUtils.addBackendDetailsAboutFileRows(record, row);
boolean hadAnyValuesInRowFromFile = false;
boolean hadAnyValuesInRow = false;
for(QFieldMetaData field : table.getFields().values())
{
hadAnyValuesInRow = setValueOrDefault(record, field, associationNameChain, mapping, row, fieldIndexes.get(field.getName()), wideAssociationIndexes) || hadAnyValuesInRow;
hadAnyValuesInRowFromFile = setValueOrDefault(record, field, associationNameChain, mapping, row, fieldIndexes.get(field.getName()), wideAssociationIndexes) || hadAnyValuesInRowFromFile;
/////////////////////////////////////////////////////////////////////////////////////
// for wide mode (different from tall) - allow a row that only has default values. //
// e.g., Line Item (2) might be a default to add to every order //
/////////////////////////////////////////////////////////////////////////////////////
if(record.getValue(field.getName()) != null)
{
hadAnyValuesInRow = true;
}
}
if(!hadAnyValuesInRow)
if(rowOfOnlyDefaultValues)
{
return (null);
if(!hadAnyValuesInRow)
{
return (null);
}
}
else
{
if(!hadAnyValuesInRowFromFile)
{
return (null);
}
}
/////////////////////////////
@ -149,12 +204,23 @@ public class WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMapping implem
// todo - doesn't support grand-children
List<Integer> wideAssociationIndexes = List.of(i);
Map<String, Integer> fieldIndexes = mapping.getFieldIndexes(associatedTable, associationNameChainForRecursiveCalls, headerRow, wideAssociationIndexes);
boolean rowOfOnlyDefaultValues = false;
if(fieldIndexes.isEmpty())
{
break;
///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// if there aren't any field-indexes for this (i) value (e.g., no columns mapped for Line Item: X (2)), we can still build a //
// child record here if there are any default values - so check for them - and only if they are empty, then break the loop. //
///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
Map<String, Serializable> fieldDefaultValues = mapping.getFieldDefaultValues(associatedTable, associationNameChainForRecursiveCalls, wideAssociationIndexes);
if(!CollectionUtils.nullSafeHasContents(fieldDefaultValues))
{
break;
}
rowOfOnlyDefaultValues = true;
}
QRecord record = makeRecordFromRow(mapping, associatedTable, associationNameChainForRecursiveCalls, row, fieldIndexes, headerRow, wideAssociationIndexes);
QRecord record = makeRecordFromRow(mapping, associatedTable, associationNameChainForRecursiveCalls, row, fieldIndexes, headerRow, wideAssociationIndexes, rowOfOnlyDefaultValues);
if(record != null)
{
rs.add(record);

34
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/model/BulkInsertMapping.java
View File

@ -165,6 +165,40 @@ public class BulkInsertMapping implements Serializable
/***************************************************************************
** get a map of default-values for fields in a given table (at the specified
** association chain and wide-indexes). Will only include fields using a
** default value.
***************************************************************************/
@JsonIgnore
public Map<String, Serializable> getFieldDefaultValues(QTableMetaData table, String associationNameChain, List<Integer> wideAssociationIndexes) throws QException
{
Map<String, Serializable> rs = new HashMap<>();
String wideAssociationSuffix = "";
if(CollectionUtils.nullSafeHasContents(wideAssociationIndexes))
{
wideAssociationSuffix = "," + StringUtils.join(".", wideAssociationIndexes);
}
///////////////////////////////////////////////////////////////////////////
// loop over fields - adding them to the rs if they have a default value //
///////////////////////////////////////////////////////////////////////////
String fieldNamePrefix = !StringUtils.hasContent(associationNameChain) ? "" : associationNameChain + ".";
for(QFieldMetaData field : table.getFields().values())
{
Serializable defaultValue = fieldNameToDefaultValueMap.get(fieldNamePrefix + field.getName() + wideAssociationSuffix);
if(defaultValue != null)
{
rs.put(field.getName(), defaultValue);
}
}
return (rs);
}
/***************************************************************************
**
***************************************************************************/