QRunIO/qqq
2
0
Fork 0
mirror of https://github.com/Kingsrook/qqq.git synced 2025-07-18 05:01:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

CE-1955 Add handling for associations w/ some vs. all values coming from defaults instead of columns;

Browse Source
This commit is contained in:
darin.kelkhoff
2025-01-03 12:57:49 -06:00
parent 048ee2e332
commit 3fda1a1eda
5 changed files with 233 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files

21
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/TallRowsToRecord.java
View File

@ -32,6 +32,7 @@ import java.util.Optional;
import com.google.gson.reflect.TypeToken;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.logging.QLogger;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.tables.Association;
@ -43,6 +44,7 @@ import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
import com.kingsrook.qqq.backend.core.utils.Pair;
import com.kingsrook.qqq.backend.core.utils.StringUtils;
import com.kingsrook.qqq.backend.core.utils.memoization.Memoization;
import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
/*******************************************************************************
@ -50,6 +52,8 @@ import com.kingsrook.qqq.backend.core.utils.memoization.Memoization;
*******************************************************************************/
public class TallRowsToRecord implements RowsToRecordInterface
{
private static final QLogger LOG = QLogger.getLogger(TallRowsToRecord.class);
private Memoization<Pair<String, String>, Boolean> shouldProcesssAssociationMemoization = new Memoization<>();
@ -166,9 +170,9 @@ public class TallRowsToRecord implements RowsToRecordInterface
Map<String, Integer> fieldIndexes = mapping.getFieldIndexes(table, associationNameChain, headerRow);
//////////////////////////////////////////////////////
// get all rows for the main table from the 0th row //
//////////////////////////////////////////////////////
////////////////////////////////////////////////////////
// get all values for the main table from the 0th row //
////////////////////////////////////////////////////////
BulkLoadFileRow row = rows.get(0);
for(QFieldMetaData field : table.getFields().values())
{
@ -258,6 +262,17 @@ public class TallRowsToRecord implements RowsToRecordInterface
// throw (new QException("Missing group-by-index(es) for association: " + associationNameChainForRecursiveCalls));
}
if(CollectionUtils.nullSafeIsEmpty(groupByIndexes))
{
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// special case here - if there are no group-by-indexes for the row, it means there are no fields coming from columns in the file. //
// but, if any fields for this association have a default value - then - make a row using just default values. //
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
LOG.info("Handling case of an association with no fields from the file, but rather only defaults", logPair("associationName", associationName));
rs.add(makeRecordFromRows(table, associationNameChainForRecursiveCalls, mapping, headerRow, List.of(row)));
break;
}
List<Serializable> rowGroupByValues = getGroupByValues(row, groupByIndexes);
if(rowGroupByValues == null)
{

80
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMapping.java
View File

@ -22,6 +22,7 @@
package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
@ -71,7 +72,7 @@ public class WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMapping implem
while(fileToRowsInterface.hasNext() && rs.size() < limit)
{
BulkLoadFileRow row = fileToRowsInterface.next();
QRecord record = makeRecordFromRow(mapping, table, "", row, fieldIndexes, headerRow, new ArrayList<>());
QRecord record = makeRecordFromRow(mapping, table, "", row, fieldIndexes, headerRow, new ArrayList<>(), false);
rs.add(record);
}
@ -84,8 +85,42 @@ public class WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMapping implem
/***************************************************************************
** may return null, if there were no values in the row for this (sub-wide) record.
** more specifically:
**
** the param `rowOfOnlyDefaultValues` - should be false for the header table,
** and true for an association iff all mapped fields are using 'default values'
** (e.g., not values from the file).
**
** So this method will return null, indicating "no child row to build" if:
** - when doing a rowOfOnlyDefaultValues - only if there actually weren't any
** default values, which, probably never happens!
** - else (doing a row with at least 1 value from the file) - then, null is
** returned if there were NO values from the file.
**
** The goal here is to support these cases:
**
** Case A (a row of not only-default-values):
** - lineItem.sku,0 = column: sku1
** - lineItem.qty,0 = column: qty1
** - lineItem.lineNo,0 = Default: 1
** - lineItem.sku,1 = column: sku2
** - lineItem.qty,1 = column: qty2
** - lineItem.lineNo,1 = Default: 2
** then a file row with no values for sku2 & qty2 - we don't want a row
** in that case (which would only have the default value of lineNo=2)
**
** Case B (a row of only-default-values):
** - lineItem.sku,0 = column: sku1
** - lineItem.qty,0 = column: qty1
** - lineItem.lineNo,0 = Default: 1
** - lineItem.sku,1 = Default: SUPPLEMENT
** - lineItem.qty,1 = Default: 1
** - lineItem.lineNo,1 = Default: 2
** we want every parent (order) to include a 2nd line item - with 3
** default values (sku=SUPPLEMENT, qty=q, lineNo=2).
**
***************************************************************************/
private QRecord makeRecordFromRow(BulkInsertMapping mapping, QTableMetaData table, String associationNameChain, BulkLoadFileRow row, Map<String, Integer> fieldIndexes, BulkLoadFileRow headerRow, List<Integer> wideAssociationIndexes) throws QException
private QRecord makeRecordFromRow(BulkInsertMapping mapping, QTableMetaData table, String associationNameChain, BulkLoadFileRow row, Map<String, Integer> fieldIndexes, BulkLoadFileRow headerRow, List<Integer> wideAssociationIndexes, boolean rowOfOnlyDefaultValues) throws QException
{
//////////////////////////////////////////////////////
// start by building the record with its own fields //
@ -93,15 +128,35 @@ public class WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMapping implem
QRecord record = new QRecord();
BulkLoadRecordUtils.addBackendDetailsAboutFileRows(record, row);
boolean hadAnyValuesInRowFromFile = false;
boolean hadAnyValuesInRow = false;
for(QFieldMetaData field : table.getFields().values())
{
hadAnyValuesInRow = setValueOrDefault(record, field, associationNameChain, mapping, row, fieldIndexes.get(field.getName()), wideAssociationIndexes) || hadAnyValuesInRow;
hadAnyValuesInRowFromFile = setValueOrDefault(record, field, associationNameChain, mapping, row, fieldIndexes.get(field.getName()), wideAssociationIndexes) || hadAnyValuesInRowFromFile;
/////////////////////////////////////////////////////////////////////////////////////
// for wide mode (different from tall) - allow a row that only has default values. //
// e.g., Line Item (2) might be a default to add to every order //
/////////////////////////////////////////////////////////////////////////////////////
if(record.getValue(field.getName()) != null)
{
hadAnyValuesInRow = true;
}
}
if(!hadAnyValuesInRow)
if(rowOfOnlyDefaultValues)
{
return (null);
if(!hadAnyValuesInRow)
{
return (null);
}
}
else
{
if(!hadAnyValuesInRowFromFile)
{
return (null);
}
}
/////////////////////////////
@ -149,12 +204,23 @@ public class WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMapping implem
// todo - doesn't support grand-children
List<Integer> wideAssociationIndexes = List.of(i);
Map<String, Integer> fieldIndexes = mapping.getFieldIndexes(associatedTable, associationNameChainForRecursiveCalls, headerRow, wideAssociationIndexes);
boolean rowOfOnlyDefaultValues = false;
if(fieldIndexes.isEmpty())
{
break;
///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// if there aren't any field-indexes for this (i) value (e.g., no columns mapped for Line Item: X (2)), we can still build a //
// child record here if there are any default values - so check for them - and only if they are empty, then break the loop. //
///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
Map<String, Serializable> fieldDefaultValues = mapping.getFieldDefaultValues(associatedTable, associationNameChainForRecursiveCalls, wideAssociationIndexes);
if(!CollectionUtils.nullSafeHasContents(fieldDefaultValues))
{
break;
}
rowOfOnlyDefaultValues = true;
}
QRecord record = makeRecordFromRow(mapping, associatedTable, associationNameChainForRecursiveCalls, row, fieldIndexes, headerRow, wideAssociationIndexes);
QRecord record = makeRecordFromRow(mapping, associatedTable, associationNameChainForRecursiveCalls, row, fieldIndexes, headerRow, wideAssociationIndexes, rowOfOnlyDefaultValues);
if(record != null)
{
rs.add(record);

34
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/model/BulkInsertMapping.java
View File

@ -165,6 +165,40 @@ public class BulkInsertMapping implements Serializable
/***************************************************************************
** get a map of default-values for fields in a given table (at the specified
** association chain and wide-indexes). Will only include fields using a
** default value.
***************************************************************************/
@JsonIgnore
public Map<String, Serializable> getFieldDefaultValues(QTableMetaData table, String associationNameChain, List<Integer> wideAssociationIndexes) throws QException
{
Map<String, Serializable> rs = new HashMap<>();
String wideAssociationSuffix = "";
if(CollectionUtils.nullSafeHasContents(wideAssociationIndexes))
{
wideAssociationSuffix = "," + StringUtils.join(".", wideAssociationIndexes);
}
///////////////////////////////////////////////////////////////////////////
// loop over fields - adding them to the rs if they have a default value //
///////////////////////////////////////////////////////////////////////////
String fieldNamePrefix = !StringUtils.hasContent(associationNameChain) ? "" : associationNameChain + ".";
for(QFieldMetaData field : table.getFields().values())
{
Serializable defaultValue = fieldNameToDefaultValueMap.get(fieldNamePrefix + field.getName() + wideAssociationSuffix);
if(defaultValue != null)
{
rs.put(field.getName(), defaultValue);
}
}
return (rs);
}
/***************************************************************************
**
***************************************************************************/

55
qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/TallRowsToRecordTest.java
View File

@ -105,6 +105,61 @@ class TallRowsToRecordTest extends BaseTest
assertEquals(2, ((List<?>) order.getBackendDetail("fileRows")).size());
}
/*******************************************************************************
** test to show that we can do 1 default line item (child record) for each
** header record.
*******************************************************************************/
@Test
void testOrderAndLinesWithLineValuesFromDefaults() throws QException
{
CsvFileToRows fileToRows = CsvFileToRows.forString("""
orderNo, Ship To, lastName
1, Homer, Simpson
2, Ned, Flanders
""");
BulkLoadFileRow header = fileToRows.next();
TallRowsToRecord rowsToRecord = new TallRowsToRecord();
BulkInsertMapping mapping = new BulkInsertMapping()
.withFieldNameToHeaderNameMap(Map.of(
"orderNo", "orderNo",
"shipToName", "Ship To"
))
.withFieldNameToDefaultValueMap(Map.of(
"orderLine.sku", "NUCLEAR-ROD",
"orderLine.quantity", 1
))
.withMappedAssociations(List.of("orderLine"))
.withTableName(TestUtils.TABLE_NAME_ORDER)
.withLayout(BulkInsertMapping.Layout.TALL)
.withHasHeaderRow(true);
List<QRecord> records = rowsToRecord.nextPage(fileToRows, header, mapping, Integer.MAX_VALUE);
assertEquals(2, records.size());
QRecord order = records.get(0);
assertEquals(1, order.getValueInteger("orderNo"));
assertEquals("Homer", order.getValueString("shipToName"));
assertEquals(1, order.getAssociatedRecords().get("orderLine").size());
assertEquals(List.of("NUCLEAR-ROD"), getValues(order.getAssociatedRecords().get("orderLine"), "sku"));
assertEquals(List.of(1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity"));
assertEquals("Row 2", order.getBackendDetail("rowNos"));
assertEquals(1, ((List<?>) order.getBackendDetail("fileRows")).size());
order = records.get(1);
assertEquals(2, order.getValueInteger("orderNo"));
assertEquals("Ned", order.getValueString("shipToName"));
assertEquals(1, order.getAssociatedRecords().get("orderLine").size());
assertEquals(List.of("NUCLEAR-ROD"), getValues(order.getAssociatedRecords().get("orderLine"), "sku"));
assertEquals(List.of(1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity"));
assertEquals("Row 3", order.getBackendDetail("rowNos"));
assertEquals(1, ((List<?>) order.getBackendDetail("fileRows")).size());
}
/*******************************************************************************
**
*******************************************************************************/

53
qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMappingTest.java
View File

@ -98,6 +98,59 @@ class WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMappingTest extends B
assertEquals("Row 3", order.getAssociatedRecords().get("orderLine").get(0).getBackendDetail("rowNos"));
}
/*******************************************************************************
**
*******************************************************************************/
@Test
void testOrderAndLinesWithLineValuesFromDefaults() throws QException
{
String csv = """
orderNo, Ship To, lastName, SKU 1, Quantity 1
1, Homer, Simpson, DONUT, 12,
2, Ned, Flanders,
""";
CsvFileToRows fileToRows = CsvFileToRows.forString(csv);
BulkLoadFileRow header = fileToRows.next();
WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMapping rowsToRecord = new WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMapping();
BulkInsertMapping mapping = new BulkInsertMapping()
.withFieldNameToHeaderNameMap(Map.of(
"orderNo", "orderNo",
"shipToName", "Ship To",
"orderLine.sku,0", "SKU 1",
"orderLine.quantity,0", "Quantity 1"
))
.withFieldNameToDefaultValueMap(Map.of(
"orderLine.sku,1", "NUCLEAR-ROD",
"orderLine.quantity,1", 1
))
.withMappedAssociations(List.of("orderLine"))
.withTableName(TestUtils.TABLE_NAME_ORDER)
.withLayout(BulkInsertMapping.Layout.WIDE)
.withHasHeaderRow(true);
List<QRecord> records = rowsToRecord.nextPage(fileToRows, header, mapping, Integer.MAX_VALUE);
assertEquals(2, records.size());
QRecord order = records.get(0);
assertEquals(1, order.getValueInteger("orderNo"));
assertEquals("Homer", order.getValueString("shipToName"));
assertEquals(List.of("DONUT", "NUCLEAR-ROD"), getValues(order.getAssociatedRecords().get("orderLine"), "sku"));
assertEquals(List.of(12, 1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity"));
assertEquals(1, ((List<?>) order.getBackendDetail("fileRows")).size());
assertEquals("Row 2", order.getAssociatedRecords().get("orderLine").get(0).getBackendDetail("rowNos"));
order = records.get(1);
assertEquals(2, order.getValueInteger("orderNo"));
assertEquals("Ned", order.getValueString("shipToName"));
assertEquals(List.of("NUCLEAR-ROD"), getValues(order.getAssociatedRecords().get("orderLine"), "sku"));
assertEquals(List.of(1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity"));
assertEquals(1, ((List<?>) order.getBackendDetail("fileRows")).size());
assertEquals("Row 3", order.getAssociatedRecords().get("orderLine").get(0).getBackendDetail("rowNos"));
}
/*******************************************************************************
**
*******************************************************************************/