CE-1955 - Put rows & rowNos in backend details during bulk-load. assert about those. also add tests (and fixes to mapping) for no-header use-cases

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/FlatRowsToRecordTest.java

@@ -76,21 +76,86 @@ class FlatRowsToRecordTest extends BaseTest
          .withHasHeaderRow(true);
 
       List<QRecord> records = rowsToRecord.nextPage(fileToRows, header, mapping, 1);
+      assertEquals(1, records.size());
       assertEquals(List.of("Homer"), getValues(records, "firstName"));
       assertEquals(List.of("Simpson"), getValues(records, "lastName"));
       assertEquals(List.of(2), getValues(records, "noOfShoes"));
       assertEquals(List.of(new BigDecimal("3.50")), getValues(records, "cost"));
       assertEquals(4, records.get(0).getValues().size()); // make sure no additional values were set
+      assertEquals(1, ((List<?>) records.get(0).getBackendDetail("fileRows")).size());
+      assertEquals("Row 2", records.get(0).getBackendDetail("rowNos"));
 
       records = rowsToRecord.nextPage(fileToRows, header, mapping, 2);
+      assertEquals(2, records.size());
       assertEquals(List.of("Marge", "Bart"), getValues(records, "firstName"));
       assertEquals(List.of(2, 2), getValues(records, "noOfShoes"));
       assertEquals(ListBuilder.of(null, new BigDecimal("99.95")), getValues(records, "cost"));
+      assertEquals(1, ((List<?>) records.get(0).getBackendDetail("fileRows")).size());
+      assertEquals("Row 3", records.get(0).getBackendDetail("rowNos"));
+      assertEquals("Row 4", records.get(1).getBackendDetail("rowNos"));
 
       records = rowsToRecord.nextPage(fileToRows, header, mapping, Integer.MAX_VALUE);
+      assertEquals(1, records.size());
       assertEquals(List.of("Ned"), getValues(records, "firstName"));
       assertEquals(List.of(2), getValues(records, "noOfShoes"));
       assertEquals(ListBuilder.of(new BigDecimal("1.00")), getValues(records, "cost"));
+      assertEquals("Row 5", records.get(0).getBackendDetail("rowNos"));
+   }
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testFieldNameToColumnIndexMapping() throws QException
+   {
+      TestFileToRows fileToRows = new TestFileToRows(List.of(
+         //                   0, 1,       2,         3,    4
+         new Serializable[] { 1, "Homer", "Simpson", true, "three fifty" },
+         new Serializable[] { 2, "Marge", "Simpson", false, "" },
+         new Serializable[] { 3, "Bart", "Simpson", "A", "99.95" },
+         new Serializable[] { 4, "Ned", "Flanders", 3.1, "one$" }
+      ));
+
+      FlatRowsToRecord rowsToRecord = new FlatRowsToRecord();
+
+      BulkInsertMapping mapping = new BulkInsertMapping()
+         .withFieldNameToIndexMap(Map.of(
+            "firstName", 1,
+            "lastName", 2,
+            "cost", 4
+         ))
+         .withFieldNameToDefaultValueMap(Map.of(
+            "noOfShoes", 2
+         ))
+         .withFieldNameToValueMapping(Map.of("cost", Map.of("three fifty", new BigDecimal("3.50"), "one$", new BigDecimal("1.00"))))
+         .withTableName(TestUtils.TABLE_NAME_PERSON)
+         .withHasHeaderRow(false);
+
+      List<QRecord> records = rowsToRecord.nextPage(fileToRows, null, mapping, 1);
+      assertEquals(1, records.size());
+      assertEquals(List.of("Homer"), getValues(records, "firstName"));
+      assertEquals(List.of("Simpson"), getValues(records, "lastName"));
+      assertEquals(List.of(2), getValues(records, "noOfShoes"));
+      assertEquals(List.of(new BigDecimal("3.50")), getValues(records, "cost"));
+      assertEquals(4, records.get(0).getValues().size()); // make sure no additional values were set
+      assertEquals(1, ((List<?>) records.get(0).getBackendDetail("fileRows")).size());
+      assertEquals("Row 1", records.get(0).getBackendDetail("rowNos"));
+
+      records = rowsToRecord.nextPage(fileToRows, null, mapping, 2);
+      assertEquals(2, records.size());
+      assertEquals(List.of("Marge", "Bart"), getValues(records, "firstName"));
+      assertEquals(List.of(2, 2), getValues(records, "noOfShoes"));
+      assertEquals(ListBuilder.of(null, new BigDecimal("99.95")), getValues(records, "cost"));
+      assertEquals(1, ((List<?>) records.get(0).getBackendDetail("fileRows")).size());
+      assertEquals("Row 2", records.get(0).getBackendDetail("rowNos"));
+      assertEquals("Row 3", records.get(1).getBackendDetail("rowNos"));
+
+      records = rowsToRecord.nextPage(fileToRows, null, mapping, Integer.MAX_VALUE);
+      assertEquals(1, records.size());
+      assertEquals(List.of("Ned"), getValues(records, "firstName"));
+      assertEquals(List.of(2), getValues(records, "noOfShoes"));
+      assertEquals(ListBuilder.of(new BigDecimal("1.00")), getValues(records, "cost"));
+      assertEquals("Row 4", records.get(0).getBackendDetail("rowNos"));
    }
 
 

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/TallRowsToRecordTest.java

@@ -89,6 +89,11 @@ class TallRowsToRecordTest extends BaseTest
       assertEquals(3, order.getAssociatedRecords().get("orderLine").size());
       assertEquals(List.of("DONUT", "BEER", "COUCH"), getValues(order.getAssociatedRecords().get("orderLine"), "sku"));
       assertEquals(List.of(12, 500, 1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity"));
+      assertEquals("Rows 2-4", order.getBackendDetail("rowNos"));
+      assertEquals(3, ((List<?>) order.getBackendDetail("fileRows")).size());
+      assertEquals("Row 2", order.getAssociatedRecords().get("orderLine").get(0).getBackendDetail("rowNos"));
+      assertEquals("Row 3", order.getAssociatedRecords().get("orderLine").get(1).getBackendDetail("rowNos"));
+      assertEquals("Row 4", order.getAssociatedRecords().get("orderLine").get(2).getBackendDetail("rowNos"));
 
       order = records.get(1);
       assertEquals(2, order.getValueInteger("orderNo"));
@@ -96,6 +101,68 @@ class TallRowsToRecordTest extends BaseTest
       assertEquals(2, order.getAssociatedRecords().get("orderLine").size());
       assertEquals(List.of("BIBLE", "LAWNMOWER"), getValues(order.getAssociatedRecords().get("orderLine"), "sku"));
       assertEquals(List.of(7, 1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity"));
+      assertEquals("Rows 5-6", order.getBackendDetail("rowNos"));
+      assertEquals(2, ((List<?>) order.getBackendDetail("fileRows")).size());
+   }
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testOrderAndLinesWithoutHeader() throws QException
+   {
+      // 0, 1,     2,        3,         4
+      CsvFileToRows fileToRows = CsvFileToRows.forString("""
+         1, Homer, Simpson,  DONUT,     12
+         ,  Homer, Simpson,  BEER,      500
+         ,  Homer, Simpson,  COUCH,     1
+         2, Ned,   Flanders, BIBLE,     7
+         ,  Ned,   Flanders, LAWNMOWER, 1
+         """);
+
+      BulkLoadFileRow header = null;
+
+      TallRowsToRecord rowsToRecord = new TallRowsToRecord();
+
+      BulkInsertMapping mapping = new BulkInsertMapping()
+         .withFieldNameToIndexMap(Map.of(
+            "orderNo", 0,
+            "shipToName", 1,
+            "orderLine.sku", 3,
+            "orderLine.quantity", 4
+         ))
+         .withTallLayoutGroupByIndexMap(Map.of(
+            TestUtils.TABLE_NAME_ORDER, List.of(1, 2),
+            "orderLine", List.of(3)
+         ))
+         .withMappedAssociations(List.of("orderLine"))
+         .withTableName(TestUtils.TABLE_NAME_ORDER)
+         .withLayout(BulkInsertMapping.Layout.TALL)
+         .withHasHeaderRow(false);
+
+      List<QRecord> records = rowsToRecord.nextPage(fileToRows, header, mapping, Integer.MAX_VALUE);
+      assertEquals(2, records.size());
+
+      QRecord order = records.get(0);
+      assertEquals(1, order.getValueInteger("orderNo"));
+      assertEquals("Homer", order.getValueString("shipToName"));
+      assertEquals(3, order.getAssociatedRecords().get("orderLine").size());
+      assertEquals(List.of("DONUT", "BEER", "COUCH"), getValues(order.getAssociatedRecords().get("orderLine"), "sku"));
+      assertEquals(List.of(12, 500, 1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity"));
+      assertEquals("Rows 1-3", order.getBackendDetail("rowNos"));
+      assertEquals(3, ((List<?>) order.getBackendDetail("fileRows")).size());
+      assertEquals("Row 1", order.getAssociatedRecords().get("orderLine").get(0).getBackendDetail("rowNos"));
+      assertEquals("Row 2", order.getAssociatedRecords().get("orderLine").get(1).getBackendDetail("rowNos"));
+      assertEquals("Row 3", order.getAssociatedRecords().get("orderLine").get(2).getBackendDetail("rowNos"));
+
+      order = records.get(1);
+      assertEquals(2, order.getValueInteger("orderNo"));
+      assertEquals("Ned", order.getValueString("shipToName"));
+      assertEquals(2, order.getAssociatedRecords().get("orderLine").size());
+      assertEquals(List.of("BIBLE", "LAWNMOWER"), getValues(order.getAssociatedRecords().get("orderLine"), "sku"));
+      assertEquals(List.of(7, 1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity"));
+      assertEquals("Rows 4-5", order.getBackendDetail("rowNos"));
+      assertEquals(2, ((List<?>) order.getBackendDetail("fileRows")).size());
    }
 
 
@@ -338,6 +405,113 @@ class TallRowsToRecordTest extends BaseTest
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testSingleLine() throws QException
+   {
+      Integer defaultStoreId = 101;
+
+      CsvFileToRows fileToRows = CsvFileToRows.forString("""
+         orderNo, Ship To, lastName
+         1,       Homer,   Simpson
+         """);
+
+      BulkLoadFileRow header = fileToRows.next();
+
+      TallRowsToRecord rowsToRecord = new TallRowsToRecord();
+
+      BulkInsertMapping mapping = new BulkInsertMapping()
+         .withFieldNameToHeaderNameMap(Map.of(
+            "orderNo", "orderNo",
+            "shipToName", "Ship To"
+         ))
+         .withFieldNameToDefaultValueMap(Map.of(
+            "storeId", defaultStoreId
+         ))
+         .withTableName(TestUtils.TABLE_NAME_ORDER)
+         .withLayout(BulkInsertMapping.Layout.TALL)
+         .withHasHeaderRow(true);
+
+      List<QRecord> records = rowsToRecord.nextPage(fileToRows, header, mapping, Integer.MAX_VALUE);
+      assertEquals(1, records.size());
+
+      QRecord order = records.get(0);
+      assertEquals(1, order.getValueInteger("orderNo"));
+      assertEquals("Homer", order.getValueString("shipToName"));
+      assertEquals(defaultStoreId, order.getValue("storeId"));
+      assertEquals("Row 2", order.getBackendDetail("rowNos"));
+      assertEquals(1, ((List<?>) order.getBackendDetail("fileRows")).size());
+   }
+
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testPagination() throws QException
+   {
+      CsvFileToRows fileToRows = CsvFileToRows.forString("""
+         orderNo, Ship To, lastName, SKU,       Quantity
+         1,       Homer,   Simpson,  DONUT,     12
+         2,       Ned,     Flanders, BIBLE,     7
+         2,       Ned,     Flanders, LAWNMOWER, 1
+         3,       Bart,    Simpson,  SKATEBOARD,1
+         3,       Bart,    Simpson,  SLINGSHOT, 1
+         """);
+
+      BulkLoadFileRow header = fileToRows.next();
+
+      TallRowsToRecord rowsToRecord = new TallRowsToRecord();
+
+      BulkInsertMapping mapping = new BulkInsertMapping()
+         .withFieldNameToHeaderNameMap(Map.of(
+            "orderNo", "orderNo",
+            "shipToName", "Ship To",
+            "orderLine.sku", "SKU",
+            "orderLine.quantity", "Quantity"
+         ))
+         .withTallLayoutGroupByIndexMap(Map.of(
+            TestUtils.TABLE_NAME_ORDER, List.of(1, 2),
+            "orderLine", List.of(3)
+         ))
+         .withMappedAssociations(List.of("orderLine"))
+         .withTableName(TestUtils.TABLE_NAME_ORDER)
+         .withLayout(BulkInsertMapping.Layout.TALL)
+         .withHasHeaderRow(true);
+
+      List<QRecord> records = rowsToRecord.nextPage(fileToRows, header, mapping, 2);
+      assertEquals(2, records.size());
+
+      QRecord order = records.get(0);
+      assertEquals(1, order.getValueInteger("orderNo"));
+      assertEquals("Homer", order.getValueString("shipToName"));
+      assertEquals(List.of("DONUT"), getValues(order.getAssociatedRecords().get("orderLine"), "sku"));
+      assertEquals("Row 2", order.getBackendDetail("rowNos"));
+      assertEquals(1, ((List<?>) order.getBackendDetail("fileRows")).size());
+
+      order = records.get(1);
+      assertEquals(2, order.getValueInteger("orderNo"));
+      assertEquals("Ned", order.getValueString("shipToName"));
+      assertEquals(List.of("BIBLE", "LAWNMOWER"), getValues(order.getAssociatedRecords().get("orderLine"), "sku"));
+      assertEquals("Rows 3-4", order.getBackendDetail("rowNos"));
+      assertEquals(2, ((List<?>) order.getBackendDetail("fileRows")).size());
+
+      records = rowsToRecord.nextPage(fileToRows, header, mapping, 2);
+      assertEquals(1, records.size());
+      order = records.get(0);
+      assertEquals(3, order.getValueInteger("orderNo"));
+      assertEquals("Bart", order.getValueString("shipToName"));
+      assertEquals(List.of("SKATEBOARD", "SLINGSHOT"), getValues(order.getAssociatedRecords().get("orderLine"), "sku"));
+      assertEquals("Rows 5-6", order.getBackendDetail("rowNos"));
+      assertEquals(2, ((List<?>) order.getBackendDetail("fileRows")).size());
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMappingTest.java

@@ -86,12 +86,72 @@ class WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMappingTest extends B
       assertEquals("Homer", order.getValueString("shipToName"));
       assertEquals(List.of("DONUT", "BEER", "COUCH"), getValues(order.getAssociatedRecords().get("orderLine"), "sku"));
       assertEquals(List.of(12, 500, 1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity"));
+      assertEquals(1, ((List<?>) order.getBackendDetail("fileRows")).size());
+      assertEquals("Row 2", order.getAssociatedRecords().get("orderLine").get(0).getBackendDetail("rowNos"));
 
       order = records.get(1);
       assertEquals(2, order.getValueInteger("orderNo"));
       assertEquals("Ned", order.getValueString("shipToName"));
       assertEquals(List.of("BIBLE", "LAWNMOWER"), getValues(order.getAssociatedRecords().get("orderLine"), "sku"));
       assertEquals(List.of(7, 1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity"));
+      assertEquals(1, ((List<?>) order.getBackendDetail("fileRows")).size());
+      assertEquals("Row 3", order.getAssociatedRecords().get("orderLine").get(0).getBackendDetail("rowNos"));
+   }
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testOrderAndLinesWithoutHeader() throws QException
+   {
+      // 0, 1,     2,        3,     4,  5,         6,   7,     8
+      String csv = """
+         1, Homer, Simpson,  DONUT, 12, BEER,      500, COUCH, 1
+         2, Ned,   Flanders, BIBLE, 7,  LAWNMOWER, 1
+         """;
+
+      CsvFileToRows   fileToRows = CsvFileToRows.forString(csv);
+      BulkLoadFileRow header     = null;
+
+      WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMapping rowsToRecord = new WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMapping();
+
+      BulkInsertMapping mapping = new BulkInsertMapping()
+         .withFieldNameToIndexMap(Map.of(
+            "orderNo", 0,
+            "shipToName", 1,
+            "orderLine.sku,0", 3,
+            "orderLine.quantity,0", 4,
+            "orderLine.sku,1", 5,
+            "orderLine.quantity,1", 6,
+            "orderLine.sku,2", 7,
+            "orderLine.quantity,2", 8
+         ))
+         .withMappedAssociations(List.of("orderLine"))
+         .withTableName(TestUtils.TABLE_NAME_ORDER)
+         .withLayout(BulkInsertMapping.Layout.WIDE)
+         .withHasHeaderRow(false);
+
+      List<QRecord> records = rowsToRecord.nextPage(fileToRows, header, mapping, Integer.MAX_VALUE);
+      assertEquals(2, records.size());
+
+      QRecord order = records.get(0);
+      assertEquals(1, order.getValueInteger("orderNo"));
+      assertEquals("Homer", order.getValueString("shipToName"));
+      assertEquals(List.of("DONUT", "BEER", "COUCH"), getValues(order.getAssociatedRecords().get("orderLine"), "sku"));
+      assertEquals(List.of(12, 500, 1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity"));
+      assertEquals(1, ((List<?>) order.getBackendDetail("fileRows")).size());
+      assertEquals("Row 1", order.getAssociatedRecords().get("orderLine").get(0).getBackendDetail("rowNos"));
+      assertEquals("Row 1", order.getAssociatedRecords().get("orderLine").get(0).getBackendDetail("rowNos"));
+      assertEquals("Row 1", order.getAssociatedRecords().get("orderLine").get(1).getBackendDetail("rowNos"));
+      assertEquals("Row 1", order.getAssociatedRecords().get("orderLine").get(2).getBackendDetail("rowNos"));
+
+      order = records.get(1);
+      assertEquals(2, order.getValueInteger("orderNo"));
+      assertEquals("Ned", order.getValueString("shipToName"));
+      assertEquals(List.of("BIBLE", "LAWNMOWER"), getValues(order.getAssociatedRecords().get("orderLine"), "sku"));
+      assertEquals(List.of(7, 1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity"));
+      assertEquals(1, ((List<?>) order.getBackendDetail("fileRows")).size());
+      assertEquals("Row 2", order.getAssociatedRecords().get("orderLine").get(0).getBackendDetail("rowNos"));
    }