diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadRecordUtils.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadRecordUtils.java new file mode 100644 index 00000000..0c700fe0 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadRecordUtils.java @@ -0,0 +1,112 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping; + + +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow; +import com.kingsrook.qqq.backend.core.utils.CollectionUtils; + + +/******************************************************************************* + ** Utility methods for working with records in a bulk-load. + ** + ** Originally added for working with backendDetails around the source rows. + *******************************************************************************/ +public class BulkLoadRecordUtils +{ + + /******************************************************************************* + ** Constructor + ** + *******************************************************************************/ + public static QRecord addBackendDetailsAboutFileRows(QRecord record, BulkLoadFileRow fileRow) + { + return (addBackendDetailsAboutFileRows(record, new ArrayList<>(List.of(fileRow)))); + } + + + + /******************************************************************************* + ** Constructor + ** + *******************************************************************************/ + public static QRecord addBackendDetailsAboutFileRows(QRecord record, ArrayList fileRows) + { + if(CollectionUtils.nullSafeHasContents(fileRows)) + { + Integer firstRowNo = fileRows.get(0).getRowNo(); + Integer lastRowNo = fileRows.get(fileRows.size() - 1).getRowNo(); + + if(Objects.equals(firstRowNo, lastRowNo)) + { + record.addBackendDetail("rowNos", "Row " + firstRowNo); + } + else + { + record.addBackendDetail("rowNos", "Rows " + firstRowNo + "-" + lastRowNo); + } + } + else + { + record.addBackendDetail("rowNos", "Rows ?"); + } + + record.addBackendDetail("fileRows", fileRows); + return (record); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + public static String getRowNosString(QRecord record) + { + return (record.getBackendDetailString("rowNos")); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @SuppressWarnings("unchecked") + public static ArrayList getFileRows(QRecord record) + { + return (ArrayList) record.getBackendDetail("fileRows"); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + public static List getFileRowNos(QRecord record) + { + return (getFileRows(record).stream().map(row -> row.getRowNo()).toList()); + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/FlatRowsToRecord.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/FlatRowsToRecord.java index d1a5d4c0..243496ef 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/FlatRowsToRecord.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/FlatRowsToRecord.java @@ -60,6 +60,7 @@ public class FlatRowsToRecord implements RowsToRecordInterface { BulkLoadFileRow row = fileToRowsInterface.next(); QRecord record = new QRecord(); + BulkLoadRecordUtils.addBackendDetailsAboutFileRows(record, row); for(QFieldMetaData field : table.getFields().values()) { diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/TallRowsToRecord.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/TallRowsToRecord.java index 4587336c..8e0443d3 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/TallRowsToRecord.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/TallRowsToRecord.java @@ -29,6 +29,7 @@ import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Optional; +import com.google.gson.reflect.TypeToken; import com.kingsrook.qqq.backend.core.context.QContext; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.data.QRecord; @@ -67,8 +68,8 @@ public class TallRowsToRecord implements RowsToRecordInterface List rs = new ArrayList<>(); - List rowsForCurrentRecord = new ArrayList<>(); - List recordGroupByValues = null; + ArrayList rowsForCurrentRecord = new ArrayList<>(); + List recordGroupByValues = null; String associationNameChain = ""; @@ -82,6 +83,9 @@ public class TallRowsToRecord implements RowsToRecordInterface groupByIndexes = groupByAllIndexesFromTable(mapping, table, headerRow, null); } + //////////////////////// + // this is suspect... // + //////////////////////// List rowGroupByValues = getGroupByValues(row, groupByIndexes); if(rowGroupByValues == null) { @@ -108,18 +112,19 @@ public class TallRowsToRecord implements RowsToRecordInterface ////////////////////////////////////////////////////////////// // not first, and not a match, so we can finish this record // ////////////////////////////////////////////////////////////// - rs.add(makeRecordFromRows(table, associationNameChain, mapping, headerRow, rowsForCurrentRecord)); + QRecord record = makeRecordFromRows(table, associationNameChain, mapping, headerRow, rowsForCurrentRecord); + rs.add(record); + + ////////////////////////////////////////////////////////////////////////////////////////////////////// + // we need to push this row back onto the fileToRows object, so it'll be handled in the next record // + ////////////////////////////////////////////////////////////////////////////////////////////////////// + fileToRowsInterface.unNext(); //////////////////////////////////////// // reset these record-specific values // //////////////////////////////////////// rowsForCurrentRecord = new ArrayList<>(); recordGroupByValues = null; - - ////////////////////////////////////////////////////////////////////////////////////////////////////// - // we need to push this row back onto the fileToRows object, so it'll be handled in the next record // - ////////////////////////////////////////////////////////////////////////////////////////////////////// - fileToRowsInterface.unNext(); } } @@ -129,7 +134,8 @@ public class TallRowsToRecord implements RowsToRecordInterface ///////////////////////////////////////////////////////////////////////////////////////////////// if(!rowsForCurrentRecord.isEmpty()) { - rs.add(makeRecordFromRows(table, associationNameChain, mapping, headerRow, rowsForCurrentRecord)); + QRecord record = makeRecordFromRows(table, associationNameChain, mapping, headerRow, rowsForCurrentRecord); + rs.add(record); } ValueMapper.valueMapping(rs, mapping, table); @@ -156,6 +162,7 @@ public class TallRowsToRecord implements RowsToRecordInterface private QRecord makeRecordFromRows(QTableMetaData table, String associationNameChain, BulkInsertMapping mapping, BulkLoadFileRow headerRow, List rows) throws QException { QRecord record = new QRecord(); + BulkLoadRecordUtils.addBackendDetailsAboutFileRows(record, CollectionUtils.useOrWrap(rows, new TypeToken>() {})); Map fieldIndexes = mapping.getFieldIndexes(table, associationNameChain, headerRow); diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMapping.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMapping.java index 9c85647e..e79dd2c2 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMapping.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMapping.java @@ -90,7 +90,9 @@ public class WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMapping implem ////////////////////////////////////////////////////// // start by building the record with its own fields // ////////////////////////////////////////////////////// - QRecord record = new QRecord(); + QRecord record = new QRecord(); + BulkLoadRecordUtils.addBackendDetailsAboutFileRows(record, row); + boolean hadAnyValuesInRow = false; for(QFieldMetaData field : table.getFields().values()) { diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/model/BulkInsertMapping.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/model/BulkInsertMapping.java index 86c19775..d0164b3f 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/model/BulkInsertMapping.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/model/BulkInsertMapping.java @@ -157,7 +157,7 @@ public class BulkInsertMapping implements Serializable } else if(fieldNameToIndexMap != null) { - return (fieldNameToIndexMap); + return (getFieldIndexesForNoHeaderUseCase(table, associationNameChain, wideAssociationIndexes)); } throw (new QException("Mapping was not properly configured.")); @@ -165,6 +165,38 @@ public class BulkInsertMapping implements Serializable + /*************************************************************************** + ** + ***************************************************************************/ + @JsonIgnore + private Map getFieldIndexesForNoHeaderUseCase(QTableMetaData table, String associationNameChain, List wideAssociationIndexes) + { + Map rs = new HashMap<>(); + + String wideAssociationSuffix = ""; + if(CollectionUtils.nullSafeHasContents(wideAssociationIndexes)) + { + wideAssociationSuffix = "," + StringUtils.join(".", wideAssociationIndexes); + } + + ///////////////////////////////////////////////////////////////////////////////////////////////////////// + // loop over fields - finding what header name they are mapped to - then what index that header is at. // + ///////////////////////////////////////////////////////////////////////////////////////////////////////// + String fieldNamePrefix = !StringUtils.hasContent(associationNameChain) ? "" : associationNameChain + "."; + for(QFieldMetaData field : table.getFields().values()) + { + Integer index = fieldNameToIndexMap.get(fieldNamePrefix + field.getName() + wideAssociationSuffix); + if(index != null) + { + rs.put(field.getName(), index); + } + } + + return (rs); + } + + + /*************************************************************************** ** ***************************************************************************/ diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/FlatRowsToRecordTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/FlatRowsToRecordTest.java index 7124e946..fd1f70c4 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/FlatRowsToRecordTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/FlatRowsToRecordTest.java @@ -76,21 +76,86 @@ class FlatRowsToRecordTest extends BaseTest .withHasHeaderRow(true); List records = rowsToRecord.nextPage(fileToRows, header, mapping, 1); + assertEquals(1, records.size()); assertEquals(List.of("Homer"), getValues(records, "firstName")); assertEquals(List.of("Simpson"), getValues(records, "lastName")); assertEquals(List.of(2), getValues(records, "noOfShoes")); assertEquals(List.of(new BigDecimal("3.50")), getValues(records, "cost")); assertEquals(4, records.get(0).getValues().size()); // make sure no additional values were set + assertEquals(1, ((List) records.get(0).getBackendDetail("fileRows")).size()); + assertEquals("Row 2", records.get(0).getBackendDetail("rowNos")); records = rowsToRecord.nextPage(fileToRows, header, mapping, 2); + assertEquals(2, records.size()); assertEquals(List.of("Marge", "Bart"), getValues(records, "firstName")); assertEquals(List.of(2, 2), getValues(records, "noOfShoes")); assertEquals(ListBuilder.of(null, new BigDecimal("99.95")), getValues(records, "cost")); + assertEquals(1, ((List) records.get(0).getBackendDetail("fileRows")).size()); + assertEquals("Row 3", records.get(0).getBackendDetail("rowNos")); + assertEquals("Row 4", records.get(1).getBackendDetail("rowNos")); records = rowsToRecord.nextPage(fileToRows, header, mapping, Integer.MAX_VALUE); + assertEquals(1, records.size()); assertEquals(List.of("Ned"), getValues(records, "firstName")); assertEquals(List.of(2), getValues(records, "noOfShoes")); assertEquals(ListBuilder.of(new BigDecimal("1.00")), getValues(records, "cost")); + assertEquals("Row 5", records.get(0).getBackendDetail("rowNos")); + } + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testFieldNameToColumnIndexMapping() throws QException + { + TestFileToRows fileToRows = new TestFileToRows(List.of( + // 0, 1, 2, 3, 4 + new Serializable[] { 1, "Homer", "Simpson", true, "three fifty" }, + new Serializable[] { 2, "Marge", "Simpson", false, "" }, + new Serializable[] { 3, "Bart", "Simpson", "A", "99.95" }, + new Serializable[] { 4, "Ned", "Flanders", 3.1, "one$" } + )); + + FlatRowsToRecord rowsToRecord = new FlatRowsToRecord(); + + BulkInsertMapping mapping = new BulkInsertMapping() + .withFieldNameToIndexMap(Map.of( + "firstName", 1, + "lastName", 2, + "cost", 4 + )) + .withFieldNameToDefaultValueMap(Map.of( + "noOfShoes", 2 + )) + .withFieldNameToValueMapping(Map.of("cost", Map.of("three fifty", new BigDecimal("3.50"), "one$", new BigDecimal("1.00")))) + .withTableName(TestUtils.TABLE_NAME_PERSON) + .withHasHeaderRow(false); + + List records = rowsToRecord.nextPage(fileToRows, null, mapping, 1); + assertEquals(1, records.size()); + assertEquals(List.of("Homer"), getValues(records, "firstName")); + assertEquals(List.of("Simpson"), getValues(records, "lastName")); + assertEquals(List.of(2), getValues(records, "noOfShoes")); + assertEquals(List.of(new BigDecimal("3.50")), getValues(records, "cost")); + assertEquals(4, records.get(0).getValues().size()); // make sure no additional values were set + assertEquals(1, ((List) records.get(0).getBackendDetail("fileRows")).size()); + assertEquals("Row 1", records.get(0).getBackendDetail("rowNos")); + + records = rowsToRecord.nextPage(fileToRows, null, mapping, 2); + assertEquals(2, records.size()); + assertEquals(List.of("Marge", "Bart"), getValues(records, "firstName")); + assertEquals(List.of(2, 2), getValues(records, "noOfShoes")); + assertEquals(ListBuilder.of(null, new BigDecimal("99.95")), getValues(records, "cost")); + assertEquals(1, ((List) records.get(0).getBackendDetail("fileRows")).size()); + assertEquals("Row 2", records.get(0).getBackendDetail("rowNos")); + assertEquals("Row 3", records.get(1).getBackendDetail("rowNos")); + + records = rowsToRecord.nextPage(fileToRows, null, mapping, Integer.MAX_VALUE); + assertEquals(1, records.size()); + assertEquals(List.of("Ned"), getValues(records, "firstName")); + assertEquals(List.of(2), getValues(records, "noOfShoes")); + assertEquals(ListBuilder.of(new BigDecimal("1.00")), getValues(records, "cost")); + assertEquals("Row 4", records.get(0).getBackendDetail("rowNos")); } diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/TallRowsToRecordTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/TallRowsToRecordTest.java index 1a0ef2ab..126f500b 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/TallRowsToRecordTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/TallRowsToRecordTest.java @@ -89,6 +89,11 @@ class TallRowsToRecordTest extends BaseTest assertEquals(3, order.getAssociatedRecords().get("orderLine").size()); assertEquals(List.of("DONUT", "BEER", "COUCH"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); assertEquals(List.of(12, 500, 1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); + assertEquals("Rows 2-4", order.getBackendDetail("rowNos")); + assertEquals(3, ((List) order.getBackendDetail("fileRows")).size()); + assertEquals("Row 2", order.getAssociatedRecords().get("orderLine").get(0).getBackendDetail("rowNos")); + assertEquals("Row 3", order.getAssociatedRecords().get("orderLine").get(1).getBackendDetail("rowNos")); + assertEquals("Row 4", order.getAssociatedRecords().get("orderLine").get(2).getBackendDetail("rowNos")); order = records.get(1); assertEquals(2, order.getValueInteger("orderNo")); @@ -96,6 +101,68 @@ class TallRowsToRecordTest extends BaseTest assertEquals(2, order.getAssociatedRecords().get("orderLine").size()); assertEquals(List.of("BIBLE", "LAWNMOWER"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); assertEquals(List.of(7, 1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); + assertEquals("Rows 5-6", order.getBackendDetail("rowNos")); + assertEquals(2, ((List) order.getBackendDetail("fileRows")).size()); + } + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testOrderAndLinesWithoutHeader() throws QException + { + // 0, 1, 2, 3, 4 + CsvFileToRows fileToRows = CsvFileToRows.forString(""" + 1, Homer, Simpson, DONUT, 12 + , Homer, Simpson, BEER, 500 + , Homer, Simpson, COUCH, 1 + 2, Ned, Flanders, BIBLE, 7 + , Ned, Flanders, LAWNMOWER, 1 + """); + + BulkLoadFileRow header = null; + + TallRowsToRecord rowsToRecord = new TallRowsToRecord(); + + BulkInsertMapping mapping = new BulkInsertMapping() + .withFieldNameToIndexMap(Map.of( + "orderNo", 0, + "shipToName", 1, + "orderLine.sku", 3, + "orderLine.quantity", 4 + )) + .withTallLayoutGroupByIndexMap(Map.of( + TestUtils.TABLE_NAME_ORDER, List.of(1, 2), + "orderLine", List.of(3) + )) + .withMappedAssociations(List.of("orderLine")) + .withTableName(TestUtils.TABLE_NAME_ORDER) + .withLayout(BulkInsertMapping.Layout.TALL) + .withHasHeaderRow(false); + + List records = rowsToRecord.nextPage(fileToRows, header, mapping, Integer.MAX_VALUE); + assertEquals(2, records.size()); + + QRecord order = records.get(0); + assertEquals(1, order.getValueInteger("orderNo")); + assertEquals("Homer", order.getValueString("shipToName")); + assertEquals(3, order.getAssociatedRecords().get("orderLine").size()); + assertEquals(List.of("DONUT", "BEER", "COUCH"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); + assertEquals(List.of(12, 500, 1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); + assertEquals("Rows 1-3", order.getBackendDetail("rowNos")); + assertEquals(3, ((List) order.getBackendDetail("fileRows")).size()); + assertEquals("Row 1", order.getAssociatedRecords().get("orderLine").get(0).getBackendDetail("rowNos")); + assertEquals("Row 2", order.getAssociatedRecords().get("orderLine").get(1).getBackendDetail("rowNos")); + assertEquals("Row 3", order.getAssociatedRecords().get("orderLine").get(2).getBackendDetail("rowNos")); + + order = records.get(1); + assertEquals(2, order.getValueInteger("orderNo")); + assertEquals("Ned", order.getValueString("shipToName")); + assertEquals(2, order.getAssociatedRecords().get("orderLine").size()); + assertEquals(List.of("BIBLE", "LAWNMOWER"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); + assertEquals(List.of(7, 1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); + assertEquals("Rows 4-5", order.getBackendDetail("rowNos")); + assertEquals(2, ((List) order.getBackendDetail("fileRows")).size()); } @@ -338,6 +405,113 @@ class TallRowsToRecordTest extends BaseTest + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testSingleLine() throws QException + { + Integer defaultStoreId = 101; + + CsvFileToRows fileToRows = CsvFileToRows.forString(""" + orderNo, Ship To, lastName + 1, Homer, Simpson + """); + + BulkLoadFileRow header = fileToRows.next(); + + TallRowsToRecord rowsToRecord = new TallRowsToRecord(); + + BulkInsertMapping mapping = new BulkInsertMapping() + .withFieldNameToHeaderNameMap(Map.of( + "orderNo", "orderNo", + "shipToName", "Ship To" + )) + .withFieldNameToDefaultValueMap(Map.of( + "storeId", defaultStoreId + )) + .withTableName(TestUtils.TABLE_NAME_ORDER) + .withLayout(BulkInsertMapping.Layout.TALL) + .withHasHeaderRow(true); + + List records = rowsToRecord.nextPage(fileToRows, header, mapping, Integer.MAX_VALUE); + assertEquals(1, records.size()); + + QRecord order = records.get(0); + assertEquals(1, order.getValueInteger("orderNo")); + assertEquals("Homer", order.getValueString("shipToName")); + assertEquals(defaultStoreId, order.getValue("storeId")); + assertEquals("Row 2", order.getBackendDetail("rowNos")); + assertEquals(1, ((List) order.getBackendDetail("fileRows")).size()); + } + + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testPagination() throws QException + { + CsvFileToRows fileToRows = CsvFileToRows.forString(""" + orderNo, Ship To, lastName, SKU, Quantity + 1, Homer, Simpson, DONUT, 12 + 2, Ned, Flanders, BIBLE, 7 + 2, Ned, Flanders, LAWNMOWER, 1 + 3, Bart, Simpson, SKATEBOARD,1 + 3, Bart, Simpson, SLINGSHOT, 1 + """); + + BulkLoadFileRow header = fileToRows.next(); + + TallRowsToRecord rowsToRecord = new TallRowsToRecord(); + + BulkInsertMapping mapping = new BulkInsertMapping() + .withFieldNameToHeaderNameMap(Map.of( + "orderNo", "orderNo", + "shipToName", "Ship To", + "orderLine.sku", "SKU", + "orderLine.quantity", "Quantity" + )) + .withTallLayoutGroupByIndexMap(Map.of( + TestUtils.TABLE_NAME_ORDER, List.of(1, 2), + "orderLine", List.of(3) + )) + .withMappedAssociations(List.of("orderLine")) + .withTableName(TestUtils.TABLE_NAME_ORDER) + .withLayout(BulkInsertMapping.Layout.TALL) + .withHasHeaderRow(true); + + List records = rowsToRecord.nextPage(fileToRows, header, mapping, 2); + assertEquals(2, records.size()); + + QRecord order = records.get(0); + assertEquals(1, order.getValueInteger("orderNo")); + assertEquals("Homer", order.getValueString("shipToName")); + assertEquals(List.of("DONUT"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); + assertEquals("Row 2", order.getBackendDetail("rowNos")); + assertEquals(1, ((List) order.getBackendDetail("fileRows")).size()); + + order = records.get(1); + assertEquals(2, order.getValueInteger("orderNo")); + assertEquals("Ned", order.getValueString("shipToName")); + assertEquals(List.of("BIBLE", "LAWNMOWER"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); + assertEquals("Rows 3-4", order.getBackendDetail("rowNos")); + assertEquals(2, ((List) order.getBackendDetail("fileRows")).size()); + + records = rowsToRecord.nextPage(fileToRows, header, mapping, 2); + assertEquals(1, records.size()); + order = records.get(0); + assertEquals(3, order.getValueInteger("orderNo")); + assertEquals("Bart", order.getValueString("shipToName")); + assertEquals(List.of("SKATEBOARD", "SLINGSHOT"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); + assertEquals("Rows 5-6", order.getBackendDetail("rowNos")); + assertEquals(2, ((List) order.getBackendDetail("fileRows")).size()); + } + + + /******************************************************************************* ** *******************************************************************************/ diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMappingTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMappingTest.java index 6e59c527..38488683 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMappingTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMappingTest.java @@ -86,12 +86,72 @@ class WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMappingTest extends B assertEquals("Homer", order.getValueString("shipToName")); assertEquals(List.of("DONUT", "BEER", "COUCH"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); assertEquals(List.of(12, 500, 1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); + assertEquals(1, ((List) order.getBackendDetail("fileRows")).size()); + assertEquals("Row 2", order.getAssociatedRecords().get("orderLine").get(0).getBackendDetail("rowNos")); order = records.get(1); assertEquals(2, order.getValueInteger("orderNo")); assertEquals("Ned", order.getValueString("shipToName")); assertEquals(List.of("BIBLE", "LAWNMOWER"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); assertEquals(List.of(7, 1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); + assertEquals(1, ((List) order.getBackendDetail("fileRows")).size()); + assertEquals("Row 3", order.getAssociatedRecords().get("orderLine").get(0).getBackendDetail("rowNos")); + } + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testOrderAndLinesWithoutHeader() throws QException + { + // 0, 1, 2, 3, 4, 5, 6, 7, 8 + String csv = """ + 1, Homer, Simpson, DONUT, 12, BEER, 500, COUCH, 1 + 2, Ned, Flanders, BIBLE, 7, LAWNMOWER, 1 + """; + + CsvFileToRows fileToRows = CsvFileToRows.forString(csv); + BulkLoadFileRow header = null; + + WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMapping rowsToRecord = new WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMapping(); + + BulkInsertMapping mapping = new BulkInsertMapping() + .withFieldNameToIndexMap(Map.of( + "orderNo", 0, + "shipToName", 1, + "orderLine.sku,0", 3, + "orderLine.quantity,0", 4, + "orderLine.sku,1", 5, + "orderLine.quantity,1", 6, + "orderLine.sku,2", 7, + "orderLine.quantity,2", 8 + )) + .withMappedAssociations(List.of("orderLine")) + .withTableName(TestUtils.TABLE_NAME_ORDER) + .withLayout(BulkInsertMapping.Layout.WIDE) + .withHasHeaderRow(false); + + List records = rowsToRecord.nextPage(fileToRows, header, mapping, Integer.MAX_VALUE); + assertEquals(2, records.size()); + + QRecord order = records.get(0); + assertEquals(1, order.getValueInteger("orderNo")); + assertEquals("Homer", order.getValueString("shipToName")); + assertEquals(List.of("DONUT", "BEER", "COUCH"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); + assertEquals(List.of(12, 500, 1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); + assertEquals(1, ((List) order.getBackendDetail("fileRows")).size()); + assertEquals("Row 1", order.getAssociatedRecords().get("orderLine").get(0).getBackendDetail("rowNos")); + assertEquals("Row 1", order.getAssociatedRecords().get("orderLine").get(0).getBackendDetail("rowNos")); + assertEquals("Row 1", order.getAssociatedRecords().get("orderLine").get(1).getBackendDetail("rowNos")); + assertEquals("Row 1", order.getAssociatedRecords().get("orderLine").get(2).getBackendDetail("rowNos")); + + order = records.get(1); + assertEquals(2, order.getValueInteger("orderNo")); + assertEquals("Ned", order.getValueString("shipToName")); + assertEquals(List.of("BIBLE", "LAWNMOWER"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); + assertEquals(List.of(7, 1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); + assertEquals(1, ((List) order.getBackendDetail("fileRows")).size()); + assertEquals("Row 2", order.getAssociatedRecords().get("orderLine").get(0).getBackendDetail("rowNos")); }