QRunIO/qqq
2
0
Fork 0
mirror of https://github.com/Kingsrook/qqq.git synced 2025-07-18 21:20:45 +00:00
Code Issues Packages Projects Releases Wiki Activity

CE-1955 Checkpoint on bulk-load backend

Browse Source
This commit is contained in:
darin.kelkhoff
2024-11-19 08:44:43 -06:00
parent b684f2409b
commit d8ac14a756
35 changed files with 2682 additions and 242 deletions
Download Patch File Download Diff File Expand all files Collapse all files

69
qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertPrepareFileMappingStepTest.java Normal file
View File

@ -0,0 +1,69 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2024. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert;
import com.kingsrook.qqq.backend.core.BaseTest;
import org.junit.jupiter.api.Test;
import static org.junit.jupiter.api.Assertions.assertEquals;
/*******************************************************************************
** Unit test for BulkInsertPrepareMappingStep
*******************************************************************************/
class BulkInsertPrepareFileMappingStepTest extends BaseTest
{
/*******************************************************************************
**
*******************************************************************************/
@SuppressWarnings("PointlessArithmeticExpression")
@Test
void testToHeaderLetter()
{
assertEquals("A", BulkInsertPrepareFileMappingStep.toHeaderLetter(0));
assertEquals("B", BulkInsertPrepareFileMappingStep.toHeaderLetter(1));
assertEquals("Z", BulkInsertPrepareFileMappingStep.toHeaderLetter(25));
assertEquals("AA", BulkInsertPrepareFileMappingStep.toHeaderLetter(26 + 0));
assertEquals("AB", BulkInsertPrepareFileMappingStep.toHeaderLetter(26 + 1));
assertEquals("AZ", BulkInsertPrepareFileMappingStep.toHeaderLetter(26 + 25));
assertEquals("BA", BulkInsertPrepareFileMappingStep.toHeaderLetter(2 * 26 + 0));
assertEquals("BB", BulkInsertPrepareFileMappingStep.toHeaderLetter(2 * 26 + 1));
assertEquals("BZ", BulkInsertPrepareFileMappingStep.toHeaderLetter(2 * 26 + 25));
assertEquals("ZA", BulkInsertPrepareFileMappingStep.toHeaderLetter(26 * 26 + 0));
assertEquals("ZB", BulkInsertPrepareFileMappingStep.toHeaderLetter(26 * 26 + 1));
assertEquals("ZZ", BulkInsertPrepareFileMappingStep.toHeaderLetter(26 * 26 + 25));
assertEquals("AAA", BulkInsertPrepareFileMappingStep.toHeaderLetter(27 * 26 + 0));
assertEquals("AAB", BulkInsertPrepareFileMappingStep.toHeaderLetter(27 * 26 + 1));
assertEquals("AAC", BulkInsertPrepareFileMappingStep.toHeaderLetter(27 * 26 + 2));
assertEquals("ABA", BulkInsertPrepareFileMappingStep.toHeaderLetter(28 * 26 + 0));
assertEquals("ABB", BulkInsertPrepareFileMappingStep.toHeaderLetter(28 * 26 + 1));
assertEquals("BAA", BulkInsertPrepareFileMappingStep.toHeaderLetter(2 * 26 * 26 + 26 + 0));
}
}

55
qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertPrepareValueMappingStepTest.java Normal file
View File

@ -0,0 +1,55 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2024. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert;
import com.kingsrook.qqq.backend.core.BaseTest;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.utils.TestUtils;
import org.junit.jupiter.api.Test;
import static org.junit.jupiter.api.Assertions.assertEquals;
/*******************************************************************************
** Unit test for BulkInsertPrepareValueMappingStep
*******************************************************************************/
class BulkInsertPrepareValueMappingStepTest extends BaseTest
{
/*******************************************************************************
**
*******************************************************************************/
@Test
void test() throws QException
{
assertEquals(TestUtils.TABLE_NAME_ORDER, BulkInsertPrepareValueMappingStep.getTableAndField(TestUtils.TABLE_NAME_ORDER, "orderNo").table().getName());
assertEquals("orderNo", BulkInsertPrepareValueMappingStep.getTableAndField(TestUtils.TABLE_NAME_ORDER, "orderNo").field().getName());
assertEquals(TestUtils.TABLE_NAME_LINE_ITEM, BulkInsertPrepareValueMappingStep.getTableAndField(TestUtils.TABLE_NAME_ORDER, "orderLine.sku").table().getName());
assertEquals("sku", BulkInsertPrepareValueMappingStep.getTableAndField(TestUtils.TABLE_NAME_ORDER, "orderLine.sku").field().getName());
assertEquals(TestUtils.TABLE_NAME_LINE_ITEM_EXTRINSIC, BulkInsertPrepareValueMappingStep.getTableAndField(TestUtils.TABLE_NAME_ORDER, "orderLine.extrinsics.key").table().getName());
assertEquals("key", BulkInsertPrepareValueMappingStep.getTableAndField(TestUtils.TABLE_NAME_ORDER, "orderLine.extrinsics.key").field().getName());
}
}

2
qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/CsvFileToRowsTest.java
View File

@ -25,7 +25,7 @@ package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.fil
import java.io.ByteArrayInputStream;
import com.kingsrook.qqq.backend.core.BaseTest;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkLoadFileRow;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow;
import org.junit.jupiter.api.Test;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;

2
qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/TestFileToRows.java
View File

@ -26,7 +26,7 @@ import java.io.InputStream;
import java.io.Serializable;
import java.util.List;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkLoadFileRow;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow;
/***************************************************************************

2
qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/XlsxFileToRowsTest.java
View File

@ -37,7 +37,7 @@ import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportDestination;
import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportFormat;
import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportInput;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkLoadFileRow;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow;
import org.junit.jupiter.api.Test;
import static com.kingsrook.qqq.backend.core.actions.reporting.GenerateReportActionTest.REPORT_NAME;
import static org.junit.jupiter.api.Assertions.assertEquals;

2
qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/FlatRowsToRecordTest.java
View File

@ -29,8 +29,8 @@ import java.util.Map;
import com.kingsrook.qqq.backend.core.BaseTest;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkLoadFileRow;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.TestFileToRows;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow;
import com.kingsrook.qqq.backend.core.utils.TestUtils;
import com.kingsrook.qqq.backend.core.utils.collections.ListBuilder;
import org.junit.jupiter.api.Test;

101
qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/TallRowsToRecordTest.java
View File

@ -28,8 +28,8 @@ import java.util.Map;
import com.kingsrook.qqq.backend.core.BaseTest;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkLoadFileRow;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.CsvFileToRows;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow;
import com.kingsrook.qqq.backend.core.utils.TestUtils;
import org.junit.jupiter.api.Test;
import static org.assertj.core.api.Assertions.assertThat;
@ -87,14 +87,14 @@ class TallRowsToRecordTest extends BaseTest
assertEquals("Homer", order.getValueString("shipToName"));
assertEquals(3, order.getAssociatedRecords().get("orderLine").size());
assertEquals(List.of("DONUT", "BEER", "COUCH"), getValues(order.getAssociatedRecords().get("orderLine"), "sku"));
assertEquals(List.of("12", "500", "1"), getValues(order.getAssociatedRecords().get("orderLine"), "quantity"));
assertEquals(List.of(12, 500, 1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity"));
order = records.get(1);
assertEquals(2, order.getValueInteger("orderNo"));
assertEquals("Ned", order.getValueString("shipToName"));
assertEquals(2, order.getAssociatedRecords().get("orderLine").size());
assertEquals(List.of("BIBLE", "LAWNMOWER"), getValues(order.getAssociatedRecords().get("orderLine"), "sku"));
assertEquals(List.of("7", "1"), getValues(order.getAssociatedRecords().get("orderLine"), "quantity"));
assertEquals(List.of(7, 1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity"));
}
@ -145,7 +145,7 @@ class TallRowsToRecordTest extends BaseTest
assertEquals("Homer", order.getValueString("shipToName"));
assertEquals(3, order.getAssociatedRecords().get("orderLine").size());
assertEquals(List.of("DONUT", "BEER", "COUCH"), getValues(order.getAssociatedRecords().get("orderLine"), "sku"));
assertEquals(List.of("12", "500", "1"), getValues(order.getAssociatedRecords().get("orderLine"), "quantity"));
assertEquals(List.of(12, 500, 1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity"));
assertEquals(2, order.getAssociatedRecords().get("extrinsics").size());
assertEquals(List.of("Store Name", "Coupon Code"), getValues(order.getAssociatedRecords().get("extrinsics"), "key"));
assertEquals(List.of("QQQ Mart", "10QOff"), getValues(order.getAssociatedRecords().get("extrinsics"), "value"));
@ -155,7 +155,7 @@ class TallRowsToRecordTest extends BaseTest
assertEquals("Ned", order.getValueString("shipToName"));
assertEquals(2, order.getAssociatedRecords().get("orderLine").size());
assertEquals(List.of("BIBLE", "LAWNMOWER"), getValues(order.getAssociatedRecords().get("orderLine"), "sku"));
assertEquals(List.of("7", "1"), getValues(order.getAssociatedRecords().get("orderLine"), "quantity"));
assertEquals(List.of(7, 1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity"));
assertThat(order.getAssociatedRecords().get("extrinsics")).isNullOrEmpty();
}
@ -168,7 +168,7 @@ class TallRowsToRecordTest extends BaseTest
void testOrderLinesWithLineExtrinsicsAndOrderExtrinsic() throws QException
{
Integer defaultStoreId = 101;
Integer defaultLineNo = 102;
String defaultLineNo = "102";
String defaultOrderLineExtraSource = "file";
CsvFileToRows fileToRows = CsvFileToRows.forString("""
@ -221,7 +221,7 @@ class TallRowsToRecordTest extends BaseTest
assertEquals("Homer", order.getValueString("shipToName"));
assertEquals(defaultStoreId, order.getValue("storeId"));
assertEquals(List.of("D'OH-NUT", "BEER", "COUCH"), getValues(order.getAssociatedRecords().get("orderLine"), "sku"));
assertEquals(List.of("12", "500", "1"), getValues(order.getAssociatedRecords().get("orderLine"), "quantity"));
assertEquals(List.of(12, 500, 1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity"));
assertEquals(List.of(defaultLineNo, defaultLineNo, defaultLineNo), getValues(order.getAssociatedRecords().get("orderLine"), "lineNumber"));
assertEquals(List.of("Store Name", "Coupon Code"), getValues(order.getAssociatedRecords().get("extrinsics"), "key"));
assertEquals(List.of("QQQ Mart", "10QOff"), getValues(order.getAssociatedRecords().get("extrinsics"), "value"));
@ -240,7 +240,92 @@ class TallRowsToRecordTest extends BaseTest
assertEquals("Ned", order.getValueString("shipToName"));
assertEquals(defaultStoreId, order.getValue("storeId"));
assertEquals(List.of("BIBLE", "LAWNMOWER"), getValues(order.getAssociatedRecords().get("orderLine"), "sku"));
assertEquals(List.of("7", "1"), getValues(order.getAssociatedRecords().get("orderLine"), "quantity"));
assertEquals(List.of(7, 1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity"));
assertEquals(List.of(defaultLineNo, defaultLineNo), getValues(order.getAssociatedRecords().get("orderLine"), "lineNumber"));
assertThat(order.getAssociatedRecords().get("extrinsics")).isNullOrEmpty();
lineItem = order.getAssociatedRecords().get("orderLine").get(0);
assertEquals(List.of("Flavor"), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "key"));
assertEquals(List.of("King James"), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "value"));
assertEquals(List.of(defaultOrderLineExtraSource), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "source"));
}
/*******************************************************************************
**
*******************************************************************************/
@Test
void testAutomaticGroupByAllIndexes() throws QException
{
Integer defaultStoreId = 101;
String defaultLineNo = "102";
String defaultOrderLineExtraSource = "file";
CsvFileToRows fileToRows = CsvFileToRows.forString("""
orderNo, Ship To, lastName, SKU, Quantity, Extrinsic Key, Extrinsic Value, Line Extrinsic Key, Line Extrinsic Value
1, Homer, Simpson, DONUT, 12, Store Name, QQQ Mart, Flavor, Chocolate
1, Homer, Simpson, DONUT, 12, Coupon Code, 10QOff, Size, Large
1, Homer, Simpson, BEER, 500, , , Flavor, Hops
1, Homer, Simpson, COUCH, 1
2, Ned, Flanders, BIBLE, 7, , , Flavor, King James
2, Ned, Flanders, LAWNMOWER, 1
""");
BulkLoadFileRow header = fileToRows.next();
TallRowsToRecord rowsToRecord = new TallRowsToRecord();
BulkInsertMapping mapping = new BulkInsertMapping()
.withFieldNameToHeaderNameMap(Map.of(
"orderNo", "orderNo",
"shipToName", "Ship To",
"orderLine.sku", "SKU",
"orderLine.quantity", "Quantity",
"extrinsics.key", "Extrinsic Key",
"extrinsics.value", "Extrinsic Value",
"orderLine.extrinsics.key", "Line Extrinsic Key",
"orderLine.extrinsics.value", "Line Extrinsic Value"
))
.withFieldNameToDefaultValueMap(Map.of(
"storeId", defaultStoreId,
"orderLine.lineNumber", defaultLineNo,
"orderLine.extrinsics.source", defaultOrderLineExtraSource
))
.withFieldNameToValueMapping(Map.of("orderLine.sku", Map.of("DONUT", "D'OH-NUT")))
.withMappedAssociations(List.of("orderLine", "extrinsics", "orderLine.extrinsics"))
.withTableName(TestUtils.TABLE_NAME_ORDER)
.withLayout(BulkInsertMapping.Layout.TALL)
.withHasHeaderRow(true);
List<QRecord> records = rowsToRecord.nextPage(fileToRows, header, mapping, Integer.MAX_VALUE);
assertEquals(2, records.size());
QRecord order = records.get(0);
assertEquals(1, order.getValueInteger("orderNo"));
assertEquals("Homer", order.getValueString("shipToName"));
assertEquals(defaultStoreId, order.getValue("storeId"));
assertEquals(List.of("D'OH-NUT", "BEER", "COUCH"), getValues(order.getAssociatedRecords().get("orderLine"), "sku"));
assertEquals(List.of(12, 500, 1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity"));
assertEquals(List.of(defaultLineNo, defaultLineNo, defaultLineNo), getValues(order.getAssociatedRecords().get("orderLine"), "lineNumber"));
assertEquals(List.of("Store Name", "Coupon Code"), getValues(order.getAssociatedRecords().get("extrinsics"), "key"));
assertEquals(List.of("QQQ Mart", "10QOff"), getValues(order.getAssociatedRecords().get("extrinsics"), "value"));
QRecord lineItem = order.getAssociatedRecords().get("orderLine").get(0);
assertEquals(List.of("Flavor", "Size"), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "key"));
assertEquals(List.of("Chocolate", "Large"), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "value"));
assertEquals(List.of(defaultOrderLineExtraSource, defaultOrderLineExtraSource), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "source"));
lineItem = order.getAssociatedRecords().get("orderLine").get(1);
assertEquals(List.of("Flavor"), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "key"));
assertEquals(List.of("Hops"), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "value"));
order = records.get(1);
assertEquals(2, order.getValueInteger("orderNo"));
assertEquals("Ned", order.getValueString("shipToName"));
assertEquals(defaultStoreId, order.getValue("storeId"));
assertEquals(List.of("BIBLE", "LAWNMOWER"), getValues(order.getAssociatedRecords().get("orderLine"), "sku"));
assertEquals(List.of(7, 1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity"));
assertEquals(List.of(defaultLineNo, defaultLineNo), getValues(order.getAssociatedRecords().get("orderLine"), "lineNumber"));
assertThat(order.getAssociatedRecords().get("extrinsics")).isNullOrEmpty();

7
qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/ValueMapperTest.java
View File

@ -26,8 +26,11 @@ import java.io.Serializable;
import java.util.List;
import java.util.Map;
import com.kingsrook.qqq.backend.core.BaseTest;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
import com.kingsrook.qqq.backend.core.utils.TestUtils;
import org.json.JSONArray;
import org.json.JSONObject;
import org.junit.jupiter.api.Test;
@ -44,7 +47,7 @@ class ValueMapperTest extends BaseTest
**
*******************************************************************************/
@Test
void test()
void test() throws QException
{
BulkInsertMapping mapping = new BulkInsertMapping().withFieldNameToValueMapping(Map.of(
"storeId", Map.of("QQQMart", 1, "Q'R'Us", 2),
@ -94,7 +97,7 @@ class ValueMapperTest extends BaseTest
);
JSONObject expectedJson = recordToJson(expectedRecord);
ValueMapper.valueMapping(List.of(inputRecord), mapping);
ValueMapper.valueMapping(List.of(inputRecord), mapping, QContext.getQInstance().getTable(TestUtils.TABLE_NAME_ORDER));
JSONObject actualJson = recordToJson(inputRecord);
System.out.println("Before");

269
qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithExplicitMappingTest.java Normal file
View File

@ -0,0 +1,269 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2024. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping;
import java.io.Serializable;
import java.util.List;
import java.util.Map;
import com.kingsrook.qqq.backend.core.BaseTest;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.CsvFileToRows;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow;
import com.kingsrook.qqq.backend.core.utils.TestUtils;
import org.junit.jupiter.api.Test;
import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.jupiter.api.Assertions.assertEquals;
/*******************************************************************************
** Unit test for WideRowsToRecord
*******************************************************************************/
class WideRowsToRecordWithExplicitMappingTest extends BaseTest
{
/*******************************************************************************
**
*******************************************************************************/
@Test
void testOrderAndLinesWithoutDupes() throws QException
{
String csv = """
orderNo, Ship To, lastName, SKU 1, Quantity 1, SKU 2, Quantity 2, SKU 3, Quantity 3
1, Homer, Simpson, DONUT, 12, BEER, 500, COUCH, 1
2, Ned, Flanders, BIBLE, 7, LAWNMOWER, 1
""";
CsvFileToRows fileToRows = CsvFileToRows.forString(csv);
BulkLoadFileRow header = fileToRows.next();
WideRowsToRecordWithExplicitMapping rowsToRecord = new WideRowsToRecordWithExplicitMapping();
BulkInsertMapping mapping = new BulkInsertMapping()
.withFieldNameToHeaderNameMap(Map.of(
"orderNo", "orderNo",
"shipToName", "Ship To"
))
.withWideLayoutMapping(Map.of(
"orderLine", new BulkInsertWideLayoutMapping(List.of(
new BulkInsertWideLayoutMapping.ChildRecordMapping(Map.of("sku", "SKU 1", "quantity", "Quantity 1")),
new BulkInsertWideLayoutMapping.ChildRecordMapping(Map.of("sku", "SKU 2", "quantity", "Quantity 2")),
new BulkInsertWideLayoutMapping.ChildRecordMapping(Map.of("sku", "SKU 3", "quantity", "Quantity 3"))
))
))
.withMappedAssociations(List.of("orderLine"))
.withTableName(TestUtils.TABLE_NAME_ORDER)
.withLayout(BulkInsertMapping.Layout.WIDE)
.withHasHeaderRow(true);
List<QRecord> records = rowsToRecord.nextPage(fileToRows, header, mapping, Integer.MAX_VALUE);
assertEquals(2, records.size());
QRecord order = records.get(0);
assertEquals(1, order.getValueInteger("orderNo"));
assertEquals("Homer", order.getValueString("shipToName"));
assertEquals(List.of("DONUT", "BEER", "COUCH"), getValues(order.getAssociatedRecords().get("orderLine"), "sku"));
assertEquals(List.of("12", "500", "1"), getValues(order.getAssociatedRecords().get("orderLine"), "quantity"));
order = records.get(1);
assertEquals(2, order.getValueInteger("orderNo"));
assertEquals("Ned", order.getValueString("shipToName"));
assertEquals(List.of("BIBLE", "LAWNMOWER"), getValues(order.getAssociatedRecords().get("orderLine"), "sku"));
assertEquals(List.of("7", "1"), getValues(order.getAssociatedRecords().get("orderLine"), "quantity"));
}
/*******************************************************************************
**
*******************************************************************************/
@Test
void testOrderLinesAndOrderExtrinsicWithoutDupes() throws QException
{
String csv = """
orderNo, Ship To, lastName, SKU 1, Quantity 1, SKU 2, Quantity 2, SKU 3, Quantity 3, Extrinsic Key 1, Extrinsic Value 1, Extrinsic Key 2, Extrinsic Value 2
1, Homer, Simpson, DONUT, 12, BEER, 500, COUCH, 1, Store Name, QQQ Mart, Coupon Code, 10QOff
2, Ned, Flanders, BIBLE, 7, LAWNMOWER, 1
""";
CsvFileToRows fileToRows = CsvFileToRows.forString(csv);
BulkLoadFileRow header = fileToRows.next();
WideRowsToRecordWithExplicitMapping rowsToRecord = new WideRowsToRecordWithExplicitMapping();
BulkInsertMapping mapping = new BulkInsertMapping()
.withFieldNameToHeaderNameMap(Map.of(
"orderNo", "orderNo",
"shipToName", "Ship To"
))
.withWideLayoutMapping(Map.of(
"orderLine", new BulkInsertWideLayoutMapping(List.of(
new BulkInsertWideLayoutMapping.ChildRecordMapping(Map.of("sku", "SKU 1", "quantity", "Quantity 1")),
new BulkInsertWideLayoutMapping.ChildRecordMapping(Map.of("sku", "SKU 2", "quantity", "Quantity 2")),
new BulkInsertWideLayoutMapping.ChildRecordMapping(Map.of("sku", "SKU 3", "quantity", "Quantity 3"))
)),
"extrinsics", new BulkInsertWideLayoutMapping(List.of(
new BulkInsertWideLayoutMapping.ChildRecordMapping(Map.of("key", "Extrinsic Key 1", "value", "Extrinsic Value 1")),
new BulkInsertWideLayoutMapping.ChildRecordMapping(Map.of("key", "Extrinsic Key 2", "value", "Extrinsic Value 2"))
))
))
.withMappedAssociations(List.of("orderLine", "extrinsics"))
.withTableName(TestUtils.TABLE_NAME_ORDER)
.withLayout(BulkInsertMapping.Layout.WIDE)
.withHasHeaderRow(true);
List<QRecord> records = rowsToRecord.nextPage(fileToRows, header, mapping, Integer.MAX_VALUE);
assertEquals(2, records.size());
QRecord order = records.get(0);
assertEquals(1, order.getValueInteger("orderNo"));
assertEquals("Homer", order.getValueString("shipToName"));
assertEquals(List.of("DONUT", "BEER", "COUCH"), getValues(order.getAssociatedRecords().get("orderLine"), "sku"));
assertEquals(List.of("12", "500", "1"), getValues(order.getAssociatedRecords().get("orderLine"), "quantity"));
assertEquals(List.of("Store Name", "Coupon Code"), getValues(order.getAssociatedRecords().get("extrinsics"), "key"));
assertEquals(List.of("QQQ Mart", "10QOff"), getValues(order.getAssociatedRecords().get("extrinsics"), "value"));
order = records.get(1);
assertEquals(2, order.getValueInteger("orderNo"));
assertEquals("Ned", order.getValueString("shipToName"));
assertEquals(List.of("BIBLE", "LAWNMOWER"), getValues(order.getAssociatedRecords().get("orderLine"), "sku"));
assertEquals(List.of("7", "1"), getValues(order.getAssociatedRecords().get("orderLine"), "quantity"));
assertThat(order.getAssociatedRecords().get("extrinsics")).isNullOrEmpty();
}
/*******************************************************************************
**
*******************************************************************************/
@Test
void testOrderLinesWithLineExtrinsicsAndOrderExtrinsicWithoutDupes() throws QException
{
String csv = """
orderNo, Ship To, lastName, Extrinsic Key 1, Extrinsic Value 1, Extrinsic Key 2, Extrinsic Value 2, SKU 1, Quantity 1, Line Extrinsic Key 1.1, Line Extrinsic Value 1.1, Line Extrinsic Key 1.2, Line Extrinsic Value 1.2, SKU 2, Quantity 2, Line Extrinsic Key 2.1, Line Extrinsic Value 2.1, SKU 3, Quantity 3, Line Extrinsic Key 3.1, Line Extrinsic Value 3.1, Line Extrinsic Key 3.2
1, Homer, Simpson, Store Name, QQQ Mart, Coupon Code, 10QOff, DONUT, 12, Flavor, Chocolate, Size, Large, BEER, 500, Flavor, Hops, COUCH, 1, Color, Brown, foo,
2, Ned, Flanders, , , , , BIBLE, 7, Flavor, King James, Size, X-Large, LAWNMOWER, 1
""";
Integer defaultStoreId = 42;
Integer defaultLineNo = 47;
String defaultLineExtraValue = "bar";
CsvFileToRows fileToRows = CsvFileToRows.forString(csv);
BulkLoadFileRow header = fileToRows.next();
WideRowsToRecordWithExplicitMapping rowsToRecord = new WideRowsToRecordWithExplicitMapping();
BulkInsertMapping mapping = new BulkInsertMapping()
.withFieldNameToHeaderNameMap(Map.of(
"orderNo", "orderNo",
"shipToName", "Ship To"
))
.withMappedAssociations(List.of("orderLine", "extrinsics", "orderLine.extrinsics"))
.withWideLayoutMapping(Map.of(
"orderLine", new BulkInsertWideLayoutMapping(List.of(
new BulkInsertWideLayoutMapping.ChildRecordMapping(
Map.of("sku", "SKU 1", "quantity", "Quantity 1"),
Map.of("extrinsics", new BulkInsertWideLayoutMapping(List.of(
new BulkInsertWideLayoutMapping.ChildRecordMapping(Map.of("key", "Line Extrinsic Key 1.1", "value", "Line Extrinsic Value 1.1")),
new BulkInsertWideLayoutMapping.ChildRecordMapping(Map.of("key", "Line Extrinsic Key 1.2", "value", "Line Extrinsic Value 1.2"))
)))),
new BulkInsertWideLayoutMapping.ChildRecordMapping(
Map.of("sku", "SKU 2", "quantity", "Quantity 2"),
Map.of("extrinsics", new BulkInsertWideLayoutMapping(List.of(
new BulkInsertWideLayoutMapping.ChildRecordMapping(Map.of("key", "Line Extrinsic Key 2.1", "value", "Line Extrinsic Value 2.1")),
new BulkInsertWideLayoutMapping.ChildRecordMapping(Map.of("key", "Line Extrinsic Key 2.2", "value", "Line Extrinsic Value 2.2"))
)))),
new BulkInsertWideLayoutMapping.ChildRecordMapping(
Map.of("sku", "SKU 3", "quantity", "Quantity 3"),
Map.of("extrinsics", new BulkInsertWideLayoutMapping(List.of(
new BulkInsertWideLayoutMapping.ChildRecordMapping(Map.of("key", "Line Extrinsic Key 3.1", "value", "Line Extrinsic Value 3.1")),
new BulkInsertWideLayoutMapping.ChildRecordMapping(Map.of("key", "Line Extrinsic Key 3.2", "value", "Line Extrinsic Value 3.2"))
))))
)),
"extrinsics", new BulkInsertWideLayoutMapping(List.of(
new BulkInsertWideLayoutMapping.ChildRecordMapping(Map.of("key", "Extrinsic Key 1", "value", "Extrinsic Value 1")),
new BulkInsertWideLayoutMapping.ChildRecordMapping(Map.of("key", "Extrinsic Key 2", "value", "Extrinsic Value 2"))
))
))
.withFieldNameToValueMapping(Map.of("orderLine.extrinsics.value", Map.of("Large", "L", "X-Large", "XL")))
.withFieldNameToDefaultValueMap(Map.of(
"storeId", defaultStoreId,
"orderLine.lineNumber", defaultLineNo,
"orderLine.extrinsics.value", defaultLineExtraValue
))
.withTableName(TestUtils.TABLE_NAME_ORDER)
.withLayout(BulkInsertMapping.Layout.WIDE)
.withHasHeaderRow(true);
List<QRecord> records = rowsToRecord.nextPage(fileToRows, header, mapping, Integer.MAX_VALUE);
assertEquals(2, records.size());
QRecord order = records.get(0);
assertEquals(1, order.getValueInteger("orderNo"));
assertEquals("Homer", order.getValueString("shipToName"));
assertEquals(defaultStoreId, order.getValue("storeId"));
assertEquals(List.of("DONUT", "BEER", "COUCH"), getValues(order.getAssociatedRecords().get("orderLine"), "sku"));
assertEquals(List.of("12", "500", "1"), getValues(order.getAssociatedRecords().get("orderLine"), "quantity"));
assertEquals(List.of(defaultLineNo, defaultLineNo, defaultLineNo), getValues(order.getAssociatedRecords().get("orderLine"), "lineNumber"));
assertEquals(List.of("Store Name", "Coupon Code"), getValues(order.getAssociatedRecords().get("extrinsics"), "key"));
assertEquals(List.of("QQQ Mart", "10QOff"), getValues(order.getAssociatedRecords().get("extrinsics"), "value"));
QRecord lineItem = order.getAssociatedRecords().get("orderLine").get(0);
assertEquals(List.of("Flavor", "Size"), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "key"));
assertEquals(List.of("Chocolate", "L"), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "value"));
lineItem = order.getAssociatedRecords().get("orderLine").get(1);
assertEquals(List.of("Flavor"), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "key"));
assertEquals(List.of("Hops"), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "value"));
lineItem = order.getAssociatedRecords().get("orderLine").get(2);
assertEquals(List.of("Color", "foo"), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "key"));
assertEquals(List.of("Brown", defaultLineExtraValue), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "value"));
order = records.get(1);
assertEquals(2, order.getValueInteger("orderNo"));
assertEquals("Ned", order.getValueString("shipToName"));
assertEquals(defaultStoreId, order.getValue("storeId"));
assertEquals(List.of("BIBLE", "LAWNMOWER"), getValues(order.getAssociatedRecords().get("orderLine"), "sku"));
assertEquals(List.of("7", "1"), getValues(order.getAssociatedRecords().get("orderLine"), "quantity"));
assertEquals(List.of(defaultLineNo, defaultLineNo), getValues(order.getAssociatedRecords().get("orderLine"), "lineNumber"));
assertThat(order.getAssociatedRecords().get("extrinsics")).isNullOrEmpty();
lineItem = order.getAssociatedRecords().get("orderLine").get(0);
assertEquals(List.of("Flavor", "Size"), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "key"));
assertEquals(List.of("King James", "XL"), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "value"));
}
/***************************************************************************
**
***************************************************************************/
private List<Serializable> getValues(List<QRecord> records, String fieldName)
{
return (records.stream().map(r -> r.getValue(fieldName)).toList());
}
}

10
qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordTest.java → qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithSpreadMappingTest.java
View File

@ -28,8 +28,8 @@ import java.util.Map;
import com.kingsrook.qqq.backend.core.BaseTest;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkLoadFileRow;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.CsvFileToRows;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow;
import com.kingsrook.qqq.backend.core.utils.TestUtils;
import org.junit.jupiter.api.Test;
import static org.assertj.core.api.Assertions.assertThat;
@ -39,7 +39,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals;
/*******************************************************************************
** Unit test for WideRowsToRecord
*******************************************************************************/
class WideRowsToRecordTest extends BaseTest
class WideRowsToRecordWithSpreadMappingTest extends BaseTest
{
/*******************************************************************************
@ -80,7 +80,7 @@ class WideRowsToRecordTest extends BaseTest
CsvFileToRows fileToRows = CsvFileToRows.forString(csv);
BulkLoadFileRow header = fileToRows.next();
WideRowsToRecord rowsToRecord = new WideRowsToRecord();
WideRowsToRecordWithSpreadMapping rowsToRecord = new WideRowsToRecordWithSpreadMapping();
BulkInsertMapping mapping = new BulkInsertMapping()
.withFieldNameToHeaderNameMap(Map.of(
@ -150,7 +150,7 @@ class WideRowsToRecordTest extends BaseTest
CsvFileToRows fileToRows = CsvFileToRows.forString(csv);
BulkLoadFileRow header = fileToRows.next();
WideRowsToRecord rowsToRecord = new WideRowsToRecord();
WideRowsToRecordWithSpreadMapping rowsToRecord = new WideRowsToRecordWithSpreadMapping();
BulkInsertMapping mapping = new BulkInsertMapping()
.withFieldNameToHeaderNameMap(Map.of(
@ -229,7 +229,7 @@ class WideRowsToRecordTest extends BaseTest
CsvFileToRows fileToRows = CsvFileToRows.forString(csv);
BulkLoadFileRow header = fileToRows.next();
WideRowsToRecord rowsToRecord = new WideRowsToRecord();
WideRowsToRecordWithSpreadMapping rowsToRecord = new WideRowsToRecordWithSpreadMapping();
BulkInsertMapping mapping = new BulkInsertMapping()
.withFieldNameToHeaderNameMap(Map.of(

4
qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java
View File

@ -644,6 +644,7 @@ public class TestUtils
.withField(new QFieldMetaData("createDate", QFieldType.DATE_TIME).withIsEditable(false))
.withField(new QFieldMetaData("modifyDate", QFieldType.DATE_TIME).withIsEditable(false))
.withField(new QFieldMetaData("orderNo", QFieldType.STRING))
.withField(new QFieldMetaData("shipToName", QFieldType.STRING))
.withField(new QFieldMetaData("orderDate", QFieldType.DATE))
.withField(new QFieldMetaData("storeId", QFieldType.INTEGER))
.withField(new QFieldMetaData("total", QFieldType.DECIMAL).withDisplayFormat(DisplayFormat.CURRENCY).withFieldSecurityLock(new FieldSecurityLock()
@ -700,7 +701,8 @@ public class TestUtils
.withField(new QFieldMetaData("modifyDate", QFieldType.DATE_TIME).withIsEditable(false))
.withField(new QFieldMetaData("lineItemId", QFieldType.INTEGER))
.withField(new QFieldMetaData("key", QFieldType.STRING))
.withField(new QFieldMetaData("value", QFieldType.STRING));
.withField(new QFieldMetaData("value", QFieldType.STRING))
.withField(new QFieldMetaData("source", QFieldType.STRING)); // doesn't really make sense, but useful to have an extra field here in some bulk-load tests
}