CE-1955 Add handling for associations w/ some vs. all values coming from defaults instead of columns;

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/TallRowsToRecord.java

@@ -32,6 +32,7 @@ import java.util.Optional;
 import com.google.gson.reflect.TypeToken;
 import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.logging.QLogger;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.Association;
@@ -43,6 +44,7 @@ import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
 import com.kingsrook.qqq.backend.core.utils.Pair;
 import com.kingsrook.qqq.backend.core.utils.StringUtils;
 import com.kingsrook.qqq.backend.core.utils.memoization.Memoization;
+import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
 
 
 /*******************************************************************************
@@ -50,6 +52,8 @@ import com.kingsrook.qqq.backend.core.utils.memoization.Memoization;
  *******************************************************************************/
 public class TallRowsToRecord implements RowsToRecordInterface
 {
+   private static final QLogger LOG = QLogger.getLogger(TallRowsToRecord.class);
+
    private Memoization<Pair<String, String>, Boolean> shouldProcesssAssociationMemoization = new Memoization<>();
 
 
@@ -166,9 +170,9 @@ public class TallRowsToRecord implements RowsToRecordInterface
 
       Map<String, Integer> fieldIndexes = mapping.getFieldIndexes(table, associationNameChain, headerRow);
 
-      //////////////////////////////////////////////////////
+      ////////////////////////////////////////////////////////
-      // get all rows for the main table from the 0th row //
+      // get all values for the main table from the 0th row //
-      //////////////////////////////////////////////////////
+      ////////////////////////////////////////////////////////
       BulkLoadFileRow row = rows.get(0);
       for(QFieldMetaData field : table.getFields().values())
       {
@@ -258,6 +262,17 @@ public class TallRowsToRecord implements RowsToRecordInterface
             // throw (new QException("Missing group-by-index(es) for association: " + associationNameChainForRecursiveCalls));
          }
 
+         if(CollectionUtils.nullSafeIsEmpty(groupByIndexes))
+         {
+            /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+            // special case here - if there are no group-by-indexes for the row, it means there are no fields coming from columns in the file. //
+            // but, if any fields for this association have a default value - then - make a row using just default values.                     //
+            /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+            LOG.info("Handling case of an association with no fields from the file, but rather only defaults", logPair("associationName", associationName));
+            rs.add(makeRecordFromRows(table, associationNameChainForRecursiveCalls, mapping, headerRow, List.of(row)));
+            break;
+         }
+
          List<Serializable> rowGroupByValues = getGroupByValues(row, groupByIndexes);
          if(rowGroupByValues == null)
          {

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMapping.java

@@ -22,6 +22,7 @@
 package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping;
 
 
+import java.io.Serializable;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
@@ -71,7 +72,7 @@ public class WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMapping implem
       while(fileToRowsInterface.hasNext() && rs.size() < limit)
       {
          BulkLoadFileRow row    = fileToRowsInterface.next();
-         QRecord         record = makeRecordFromRow(mapping, table, "", row, fieldIndexes, headerRow, new ArrayList<>());
+         QRecord         record = makeRecordFromRow(mapping, table, "", row, fieldIndexes, headerRow, new ArrayList<>(), false);
          rs.add(record);
       }
 
@@ -84,8 +85,42 @@ public class WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMapping implem
 
    /***************************************************************************
     ** may return null, if there were no values in the row for this (sub-wide) record.
+    ** more specifically:
+    **
+    ** the param `rowOfOnlyDefaultValues` - should be false for the header table,
+    ** and true for an association iff all mapped fields are using 'default values'
+    ** (e.g., not values from the file).
+    **
+    ** So this method will return null, indicating "no child row to build" if:
+    ** - when doing a rowOfOnlyDefaultValues - only if there actually weren't any
+    ** default values, which, probably never happens!
+    ** - else (doing a row with at least 1 value from the file) - then, null is
+    ** returned if there were NO values from the file.
+    **
+    ** The goal here is to support these cases:
+    **
+    ** Case A (a row of not only-default-values):
+    ** - lineItem.sku,0 = column: sku1
+    ** - lineItem.qty,0 = column: qty1
+    ** - lineItem.lineNo,0 = Default: 1
+    ** - lineItem.sku,1 = column: sku2
+    ** - lineItem.qty,1 = column: qty2
+    ** - lineItem.lineNo,1 = Default: 2
+    ** then a file row with no values for sku2 & qty2 - we don't want a row
+    ** in that case (which would only have the default value of lineNo=2)
+    **
+    ** Case B (a row of only-default-values):
+    ** - lineItem.sku,0 = column: sku1
+    ** - lineItem.qty,0 = column: qty1
+    ** - lineItem.lineNo,0 = Default: 1
+    ** - lineItem.sku,1 = Default: SUPPLEMENT
+    ** - lineItem.qty,1 = Default: 1
+    ** - lineItem.lineNo,1 = Default: 2
+    ** we want every parent (order) to include a 2nd line item - with 3
+    ** default values (sku=SUPPLEMENT, qty=q, lineNo=2).
+    **
     ***************************************************************************/
-   private QRecord makeRecordFromRow(BulkInsertMapping mapping, QTableMetaData table, String associationNameChain, BulkLoadFileRow row, Map<String, Integer> fieldIndexes, BulkLoadFileRow headerRow, List<Integer> wideAssociationIndexes) throws QException
+   private QRecord makeRecordFromRow(BulkInsertMapping mapping, QTableMetaData table, String associationNameChain, BulkLoadFileRow row, Map<String, Integer> fieldIndexes, BulkLoadFileRow headerRow, List<Integer> wideAssociationIndexes, boolean rowOfOnlyDefaultValues) throws QException
    {
       //////////////////////////////////////////////////////
       // start by building the record with its own fields //
@@ -93,15 +128,35 @@ public class WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMapping implem
       QRecord record = new QRecord();
       BulkLoadRecordUtils.addBackendDetailsAboutFileRows(record, row);
 
+      boolean hadAnyValuesInRowFromFile = false;
       boolean hadAnyValuesInRow = false;
       for(QFieldMetaData field : table.getFields().values())
       {
-         hadAnyValuesInRow = setValueOrDefault(record, field, associationNameChain, mapping, row, fieldIndexes.get(field.getName()), wideAssociationIndexes) || hadAnyValuesInRow;
+         hadAnyValuesInRowFromFile = setValueOrDefault(record, field, associationNameChain, mapping, row, fieldIndexes.get(field.getName()), wideAssociationIndexes) || hadAnyValuesInRowFromFile;
+
+         /////////////////////////////////////////////////////////////////////////////////////
+         // for wide mode (different from tall) - allow a row that only has default values. //
+         // e.g., Line Item (2) might be a default to add to every order                    //
+         /////////////////////////////////////////////////////////////////////////////////////
+         if(record.getValue(field.getName()) != null)
+         {
+            hadAnyValuesInRow = true;
+         }
       }
 
-      if(!hadAnyValuesInRow)
+      if(rowOfOnlyDefaultValues)
       {
-         return (null);
+         if(!hadAnyValuesInRow)
+         {
+            return (null);
+         }
+      }
+      else
+      {
+         if(!hadAnyValuesInRowFromFile)
+         {
+            return (null);
+         }
       }
 
       /////////////////////////////
@@ -149,12 +204,23 @@ public class WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMapping implem
          // todo - doesn't support grand-children
          List<Integer>        wideAssociationIndexes = List.of(i);
          Map<String, Integer> fieldIndexes           = mapping.getFieldIndexes(associatedTable, associationNameChainForRecursiveCalls, headerRow, wideAssociationIndexes);
+
+         boolean rowOfOnlyDefaultValues = false;
          if(fieldIndexes.isEmpty())
          {
-            break;
+            ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+            // if there aren't any field-indexes for this (i) value (e.g., no columns mapped for Line Item: X (2)), we can still build a //
+            // child record here if there are any default values - so check for them - and only if they are empty, then break the loop.  //
+            ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+            Map<String, Serializable> fieldDefaultValues = mapping.getFieldDefaultValues(associatedTable, associationNameChainForRecursiveCalls, wideAssociationIndexes);
+            if(!CollectionUtils.nullSafeHasContents(fieldDefaultValues))
+            {
+               break;
+            }
+            rowOfOnlyDefaultValues = true;
          }
 
-         QRecord record = makeRecordFromRow(mapping, associatedTable, associationNameChainForRecursiveCalls, row, fieldIndexes, headerRow, wideAssociationIndexes);
+         QRecord record = makeRecordFromRow(mapping, associatedTable, associationNameChainForRecursiveCalls, row, fieldIndexes, headerRow, wideAssociationIndexes, rowOfOnlyDefaultValues);
          if(record != null)
          {
             rs.add(record);

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/model/BulkInsertMapping.java

@@ -165,6 +165,40 @@ public class BulkInsertMapping implements Serializable
 
 
 
+   /***************************************************************************
+    ** get a map of default-values for fields in a given table (at the specified
+    ** association chain and wide-indexes).  Will only include fields using a
+    ** default value.
+    ***************************************************************************/
+   @JsonIgnore
+   public Map<String, Serializable> getFieldDefaultValues(QTableMetaData table, String associationNameChain, List<Integer> wideAssociationIndexes) throws QException
+   {
+      Map<String, Serializable> rs = new HashMap<>();
+
+      String wideAssociationSuffix = "";
+      if(CollectionUtils.nullSafeHasContents(wideAssociationIndexes))
+      {
+         wideAssociationSuffix = "," + StringUtils.join(".", wideAssociationIndexes);
+      }
+
+      ///////////////////////////////////////////////////////////////////////////
+      // loop over fields - adding them to the rs if they have a default value //
+      ///////////////////////////////////////////////////////////////////////////
+      String fieldNamePrefix = !StringUtils.hasContent(associationNameChain) ? "" : associationNameChain + ".";
+      for(QFieldMetaData field : table.getFields().values())
+      {
+         Serializable defaultValue = fieldNameToDefaultValueMap.get(fieldNamePrefix + field.getName() + wideAssociationSuffix);
+         if(defaultValue != null)
+         {
+            rs.put(field.getName(), defaultValue);
+         }
+      }
+
+      return (rs);
+   }
+
+
+
    /***************************************************************************
     **
     ***************************************************************************/

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/TallRowsToRecordTest.java

@@ -105,6 +105,61 @@ class TallRowsToRecordTest extends BaseTest
       assertEquals(2, ((List<?>) order.getBackendDetail("fileRows")).size());
    }
 
+
+
+   /*******************************************************************************
+    ** test to show that we can do 1 default line item (child record) for each
+    ** header record.
+    *******************************************************************************/
+   @Test
+   void testOrderAndLinesWithLineValuesFromDefaults() throws QException
+   {
+      CsvFileToRows fileToRows = CsvFileToRows.forString("""
+         orderNo, Ship To, lastName
+         1,       Homer,   Simpson
+         2,       Ned,     Flanders
+         """);
+
+      BulkLoadFileRow header = fileToRows.next();
+
+      TallRowsToRecord rowsToRecord = new TallRowsToRecord();
+
+      BulkInsertMapping mapping = new BulkInsertMapping()
+         .withFieldNameToHeaderNameMap(Map.of(
+            "orderNo", "orderNo",
+            "shipToName", "Ship To"
+         ))
+         .withFieldNameToDefaultValueMap(Map.of(
+            "orderLine.sku", "NUCLEAR-ROD",
+            "orderLine.quantity", 1
+         ))
+         .withMappedAssociations(List.of("orderLine"))
+         .withTableName(TestUtils.TABLE_NAME_ORDER)
+         .withLayout(BulkInsertMapping.Layout.TALL)
+         .withHasHeaderRow(true);
+
+      List<QRecord> records = rowsToRecord.nextPage(fileToRows, header, mapping, Integer.MAX_VALUE);
+      assertEquals(2, records.size());
+
+      QRecord order = records.get(0);
+      assertEquals(1, order.getValueInteger("orderNo"));
+      assertEquals("Homer", order.getValueString("shipToName"));
+      assertEquals(1, order.getAssociatedRecords().get("orderLine").size());
+      assertEquals(List.of("NUCLEAR-ROD"), getValues(order.getAssociatedRecords().get("orderLine"), "sku"));
+      assertEquals(List.of(1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity"));
+      assertEquals("Row 2", order.getBackendDetail("rowNos"));
+      assertEquals(1, ((List<?>) order.getBackendDetail("fileRows")).size());
+
+      order = records.get(1);
+      assertEquals(2, order.getValueInteger("orderNo"));
+      assertEquals("Ned", order.getValueString("shipToName"));
+      assertEquals(1, order.getAssociatedRecords().get("orderLine").size());
+      assertEquals(List.of("NUCLEAR-ROD"), getValues(order.getAssociatedRecords().get("orderLine"), "sku"));
+      assertEquals(List.of(1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity"));
+      assertEquals("Row 3", order.getBackendDetail("rowNos"));
+      assertEquals(1, ((List<?>) order.getBackendDetail("fileRows")).size());
+   }
+
    /*******************************************************************************
     **
     *******************************************************************************/

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMappingTest.java

@@ -98,6 +98,59 @@ class WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMappingTest extends B
       assertEquals("Row 3", order.getAssociatedRecords().get("orderLine").get(0).getBackendDetail("rowNos"));
    }
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testOrderAndLinesWithLineValuesFromDefaults() throws QException
+   {
+      String csv = """
+         orderNo, Ship To, lastName, SKU 1,     Quantity 1
+         1,       Homer,   Simpson,  DONUT,     12,
+         2,       Ned,     Flanders, 
+         """;
+
+      CsvFileToRows   fileToRows = CsvFileToRows.forString(csv);
+      BulkLoadFileRow header     = fileToRows.next();
+
+      WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMapping rowsToRecord = new WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMapping();
+
+      BulkInsertMapping mapping = new BulkInsertMapping()
+         .withFieldNameToHeaderNameMap(Map.of(
+            "orderNo", "orderNo",
+            "shipToName", "Ship To",
+            "orderLine.sku,0", "SKU 1",
+            "orderLine.quantity,0", "Quantity 1"
+         ))
+         .withFieldNameToDefaultValueMap(Map.of(
+            "orderLine.sku,1", "NUCLEAR-ROD",
+            "orderLine.quantity,1", 1
+         ))
+         .withMappedAssociations(List.of("orderLine"))
+         .withTableName(TestUtils.TABLE_NAME_ORDER)
+         .withLayout(BulkInsertMapping.Layout.WIDE)
+         .withHasHeaderRow(true);
+
+      List<QRecord> records = rowsToRecord.nextPage(fileToRows, header, mapping, Integer.MAX_VALUE);
+      assertEquals(2, records.size());
+
+      QRecord order = records.get(0);
+      assertEquals(1, order.getValueInteger("orderNo"));
+      assertEquals("Homer", order.getValueString("shipToName"));
+      assertEquals(List.of("DONUT", "NUCLEAR-ROD"), getValues(order.getAssociatedRecords().get("orderLine"), "sku"));
+      assertEquals(List.of(12, 1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity"));
+      assertEquals(1, ((List<?>) order.getBackendDetail("fileRows")).size());
+      assertEquals("Row 2", order.getAssociatedRecords().get("orderLine").get(0).getBackendDetail("rowNos"));
+
+      order = records.get(1);
+      assertEquals(2, order.getValueInteger("orderNo"));
+      assertEquals("Ned", order.getValueString("shipToName"));
+      assertEquals(List.of("NUCLEAR-ROD"), getValues(order.getAssociatedRecords().get("orderLine"), "sku"));
+      assertEquals(List.of(1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity"));
+      assertEquals(1, ((List<?>) order.getBackendDetail("fileRows")).size());
+      assertEquals("Row 3", order.getAssociatedRecords().get("orderLine").get(0).getBackendDetail("rowNos"));
+   }
+
    /*******************************************************************************
     **
     *******************************************************************************/