diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/TallRowsToRecord.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/TallRowsToRecord.java index 84ab1fa7..bf67522c 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/TallRowsToRecord.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/TallRowsToRecord.java @@ -32,6 +32,7 @@ import java.util.Optional; import com.google.gson.reflect.TypeToken; import com.kingsrook.qqq.backend.core.context.QContext; import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.logging.QLogger; import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.Association; @@ -43,6 +44,7 @@ import com.kingsrook.qqq.backend.core.utils.CollectionUtils; import com.kingsrook.qqq.backend.core.utils.Pair; import com.kingsrook.qqq.backend.core.utils.StringUtils; import com.kingsrook.qqq.backend.core.utils.memoization.Memoization; +import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; /******************************************************************************* @@ -50,6 +52,8 @@ import com.kingsrook.qqq.backend.core.utils.memoization.Memoization; *******************************************************************************/ public class TallRowsToRecord implements RowsToRecordInterface { + private static final QLogger LOG = QLogger.getLogger(TallRowsToRecord.class); + private Memoization, Boolean> shouldProcesssAssociationMemoization = new Memoization<>(); @@ -166,9 +170,9 @@ public class TallRowsToRecord implements RowsToRecordInterface Map fieldIndexes = mapping.getFieldIndexes(table, associationNameChain, headerRow); - ////////////////////////////////////////////////////// - // get all rows for the main table from the 0th row // - ////////////////////////////////////////////////////// + //////////////////////////////////////////////////////// + // get all values for the main table from the 0th row // + //////////////////////////////////////////////////////// BulkLoadFileRow row = rows.get(0); for(QFieldMetaData field : table.getFields().values()) { @@ -258,6 +262,17 @@ public class TallRowsToRecord implements RowsToRecordInterface // throw (new QException("Missing group-by-index(es) for association: " + associationNameChainForRecursiveCalls)); } + if(CollectionUtils.nullSafeIsEmpty(groupByIndexes)) + { + ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // special case here - if there are no group-by-indexes for the row, it means there are no fields coming from columns in the file. // + // but, if any fields for this association have a default value - then - make a row using just default values. // + ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + LOG.info("Handling case of an association with no fields from the file, but rather only defaults", logPair("associationName", associationName)); + rs.add(makeRecordFromRows(table, associationNameChainForRecursiveCalls, mapping, headerRow, List.of(row))); + break; + } + List rowGroupByValues = getGroupByValues(row, groupByIndexes); if(rowGroupByValues == null) { diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMapping.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMapping.java index c4231ea7..86a5faf2 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMapping.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMapping.java @@ -22,6 +22,7 @@ package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping; +import java.io.Serializable; import java.util.ArrayList; import java.util.Arrays; import java.util.List; @@ -71,7 +72,7 @@ public class WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMapping implem while(fileToRowsInterface.hasNext() && rs.size() < limit) { BulkLoadFileRow row = fileToRowsInterface.next(); - QRecord record = makeRecordFromRow(mapping, table, "", row, fieldIndexes, headerRow, new ArrayList<>()); + QRecord record = makeRecordFromRow(mapping, table, "", row, fieldIndexes, headerRow, new ArrayList<>(), false); rs.add(record); } @@ -84,8 +85,42 @@ public class WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMapping implem /*************************************************************************** ** may return null, if there were no values in the row for this (sub-wide) record. + ** more specifically: + ** + ** the param `rowOfOnlyDefaultValues` - should be false for the header table, + ** and true for an association iff all mapped fields are using 'default values' + ** (e.g., not values from the file). + ** + ** So this method will return null, indicating "no child row to build" if: + ** - when doing a rowOfOnlyDefaultValues - only if there actually weren't any + ** default values, which, probably never happens! + ** - else (doing a row with at least 1 value from the file) - then, null is + ** returned if there were NO values from the file. + ** + ** The goal here is to support these cases: + ** + ** Case A (a row of not only-default-values): + ** - lineItem.sku,0 = column: sku1 + ** - lineItem.qty,0 = column: qty1 + ** - lineItem.lineNo,0 = Default: 1 + ** - lineItem.sku,1 = column: sku2 + ** - lineItem.qty,1 = column: qty2 + ** - lineItem.lineNo,1 = Default: 2 + ** then a file row with no values for sku2 & qty2 - we don't want a row + ** in that case (which would only have the default value of lineNo=2) + ** + ** Case B (a row of only-default-values): + ** - lineItem.sku,0 = column: sku1 + ** - lineItem.qty,0 = column: qty1 + ** - lineItem.lineNo,0 = Default: 1 + ** - lineItem.sku,1 = Default: SUPPLEMENT + ** - lineItem.qty,1 = Default: 1 + ** - lineItem.lineNo,1 = Default: 2 + ** we want every parent (order) to include a 2nd line item - with 3 + ** default values (sku=SUPPLEMENT, qty=q, lineNo=2). + ** ***************************************************************************/ - private QRecord makeRecordFromRow(BulkInsertMapping mapping, QTableMetaData table, String associationNameChain, BulkLoadFileRow row, Map fieldIndexes, BulkLoadFileRow headerRow, List wideAssociationIndexes) throws QException + private QRecord makeRecordFromRow(BulkInsertMapping mapping, QTableMetaData table, String associationNameChain, BulkLoadFileRow row, Map fieldIndexes, BulkLoadFileRow headerRow, List wideAssociationIndexes, boolean rowOfOnlyDefaultValues) throws QException { ////////////////////////////////////////////////////// // start by building the record with its own fields // @@ -93,15 +128,35 @@ public class WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMapping implem QRecord record = new QRecord(); BulkLoadRecordUtils.addBackendDetailsAboutFileRows(record, row); + boolean hadAnyValuesInRowFromFile = false; boolean hadAnyValuesInRow = false; for(QFieldMetaData field : table.getFields().values()) { - hadAnyValuesInRow = setValueOrDefault(record, field, associationNameChain, mapping, row, fieldIndexes.get(field.getName()), wideAssociationIndexes) || hadAnyValuesInRow; + hadAnyValuesInRowFromFile = setValueOrDefault(record, field, associationNameChain, mapping, row, fieldIndexes.get(field.getName()), wideAssociationIndexes) || hadAnyValuesInRowFromFile; + + ///////////////////////////////////////////////////////////////////////////////////// + // for wide mode (different from tall) - allow a row that only has default values. // + // e.g., Line Item (2) might be a default to add to every order // + ///////////////////////////////////////////////////////////////////////////////////// + if(record.getValue(field.getName()) != null) + { + hadAnyValuesInRow = true; + } } - if(!hadAnyValuesInRow) + if(rowOfOnlyDefaultValues) { - return (null); + if(!hadAnyValuesInRow) + { + return (null); + } + } + else + { + if(!hadAnyValuesInRowFromFile) + { + return (null); + } } ///////////////////////////// @@ -149,12 +204,23 @@ public class WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMapping implem // todo - doesn't support grand-children List wideAssociationIndexes = List.of(i); Map fieldIndexes = mapping.getFieldIndexes(associatedTable, associationNameChainForRecursiveCalls, headerRow, wideAssociationIndexes); + + boolean rowOfOnlyDefaultValues = false; if(fieldIndexes.isEmpty()) { - break; + /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // if there aren't any field-indexes for this (i) value (e.g., no columns mapped for Line Item: X (2)), we can still build a // + // child record here if there are any default values - so check for them - and only if they are empty, then break the loop. // + /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + Map fieldDefaultValues = mapping.getFieldDefaultValues(associatedTable, associationNameChainForRecursiveCalls, wideAssociationIndexes); + if(!CollectionUtils.nullSafeHasContents(fieldDefaultValues)) + { + break; + } + rowOfOnlyDefaultValues = true; } - QRecord record = makeRecordFromRow(mapping, associatedTable, associationNameChainForRecursiveCalls, row, fieldIndexes, headerRow, wideAssociationIndexes); + QRecord record = makeRecordFromRow(mapping, associatedTable, associationNameChainForRecursiveCalls, row, fieldIndexes, headerRow, wideAssociationIndexes, rowOfOnlyDefaultValues); if(record != null) { rs.add(record); diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/model/BulkInsertMapping.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/model/BulkInsertMapping.java index d0164b3f..e391b5f9 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/model/BulkInsertMapping.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/model/BulkInsertMapping.java @@ -165,6 +165,40 @@ public class BulkInsertMapping implements Serializable + /*************************************************************************** + ** get a map of default-values for fields in a given table (at the specified + ** association chain and wide-indexes). Will only include fields using a + ** default value. + ***************************************************************************/ + @JsonIgnore + public Map getFieldDefaultValues(QTableMetaData table, String associationNameChain, List wideAssociationIndexes) throws QException + { + Map rs = new HashMap<>(); + + String wideAssociationSuffix = ""; + if(CollectionUtils.nullSafeHasContents(wideAssociationIndexes)) + { + wideAssociationSuffix = "," + StringUtils.join(".", wideAssociationIndexes); + } + + /////////////////////////////////////////////////////////////////////////// + // loop over fields - adding them to the rs if they have a default value // + /////////////////////////////////////////////////////////////////////////// + String fieldNamePrefix = !StringUtils.hasContent(associationNameChain) ? "" : associationNameChain + "."; + for(QFieldMetaData field : table.getFields().values()) + { + Serializable defaultValue = fieldNameToDefaultValueMap.get(fieldNamePrefix + field.getName() + wideAssociationSuffix); + if(defaultValue != null) + { + rs.put(field.getName(), defaultValue); + } + } + + return (rs); + } + + + /*************************************************************************** ** ***************************************************************************/ diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/TallRowsToRecordTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/TallRowsToRecordTest.java index 126f500b..b59f3b1d 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/TallRowsToRecordTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/TallRowsToRecordTest.java @@ -105,6 +105,61 @@ class TallRowsToRecordTest extends BaseTest assertEquals(2, ((List) order.getBackendDetail("fileRows")).size()); } + + + /******************************************************************************* + ** test to show that we can do 1 default line item (child record) for each + ** header record. + *******************************************************************************/ + @Test + void testOrderAndLinesWithLineValuesFromDefaults() throws QException + { + CsvFileToRows fileToRows = CsvFileToRows.forString(""" + orderNo, Ship To, lastName + 1, Homer, Simpson + 2, Ned, Flanders + """); + + BulkLoadFileRow header = fileToRows.next(); + + TallRowsToRecord rowsToRecord = new TallRowsToRecord(); + + BulkInsertMapping mapping = new BulkInsertMapping() + .withFieldNameToHeaderNameMap(Map.of( + "orderNo", "orderNo", + "shipToName", "Ship To" + )) + .withFieldNameToDefaultValueMap(Map.of( + "orderLine.sku", "NUCLEAR-ROD", + "orderLine.quantity", 1 + )) + .withMappedAssociations(List.of("orderLine")) + .withTableName(TestUtils.TABLE_NAME_ORDER) + .withLayout(BulkInsertMapping.Layout.TALL) + .withHasHeaderRow(true); + + List records = rowsToRecord.nextPage(fileToRows, header, mapping, Integer.MAX_VALUE); + assertEquals(2, records.size()); + + QRecord order = records.get(0); + assertEquals(1, order.getValueInteger("orderNo")); + assertEquals("Homer", order.getValueString("shipToName")); + assertEquals(1, order.getAssociatedRecords().get("orderLine").size()); + assertEquals(List.of("NUCLEAR-ROD"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); + assertEquals(List.of(1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); + assertEquals("Row 2", order.getBackendDetail("rowNos")); + assertEquals(1, ((List) order.getBackendDetail("fileRows")).size()); + + order = records.get(1); + assertEquals(2, order.getValueInteger("orderNo")); + assertEquals("Ned", order.getValueString("shipToName")); + assertEquals(1, order.getAssociatedRecords().get("orderLine").size()); + assertEquals(List.of("NUCLEAR-ROD"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); + assertEquals(List.of(1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); + assertEquals("Row 3", order.getBackendDetail("rowNos")); + assertEquals(1, ((List) order.getBackendDetail("fileRows")).size()); + } + /******************************************************************************* ** *******************************************************************************/ diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMappingTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMappingTest.java index 38488683..62eeb607 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMappingTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMappingTest.java @@ -98,6 +98,59 @@ class WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMappingTest extends B assertEquals("Row 3", order.getAssociatedRecords().get("orderLine").get(0).getBackendDetail("rowNos")); } + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testOrderAndLinesWithLineValuesFromDefaults() throws QException + { + String csv = """ + orderNo, Ship To, lastName, SKU 1, Quantity 1 + 1, Homer, Simpson, DONUT, 12, + 2, Ned, Flanders, + """; + + CsvFileToRows fileToRows = CsvFileToRows.forString(csv); + BulkLoadFileRow header = fileToRows.next(); + + WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMapping rowsToRecord = new WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMapping(); + + BulkInsertMapping mapping = new BulkInsertMapping() + .withFieldNameToHeaderNameMap(Map.of( + "orderNo", "orderNo", + "shipToName", "Ship To", + "orderLine.sku,0", "SKU 1", + "orderLine.quantity,0", "Quantity 1" + )) + .withFieldNameToDefaultValueMap(Map.of( + "orderLine.sku,1", "NUCLEAR-ROD", + "orderLine.quantity,1", 1 + )) + .withMappedAssociations(List.of("orderLine")) + .withTableName(TestUtils.TABLE_NAME_ORDER) + .withLayout(BulkInsertMapping.Layout.WIDE) + .withHasHeaderRow(true); + + List records = rowsToRecord.nextPage(fileToRows, header, mapping, Integer.MAX_VALUE); + assertEquals(2, records.size()); + + QRecord order = records.get(0); + assertEquals(1, order.getValueInteger("orderNo")); + assertEquals("Homer", order.getValueString("shipToName")); + assertEquals(List.of("DONUT", "NUCLEAR-ROD"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); + assertEquals(List.of(12, 1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); + assertEquals(1, ((List) order.getBackendDetail("fileRows")).size()); + assertEquals("Row 2", order.getAssociatedRecords().get("orderLine").get(0).getBackendDetail("rowNos")); + + order = records.get(1); + assertEquals(2, order.getValueInteger("orderNo")); + assertEquals("Ned", order.getValueString("shipToName")); + assertEquals(List.of("NUCLEAR-ROD"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); + assertEquals(List.of(1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); + assertEquals(1, ((List) order.getBackendDetail("fileRows")).size()); + assertEquals("Row 3", order.getAssociatedRecords().get("orderLine").get(0).getBackendDetail("rowNos")); + } + /******************************************************************************* ** *******************************************************************************/