mirror of
https://github.com/Kingsrook/qqq.git
synced 2025-07-18 21:20:45 +00:00
CE-1955 - Bulk load checkpoint:
- Switch wide format to identify associations via comma-number-indexes... - Add suggested mappings - use header name instead of column index for mappings - add counts of children process summary lines - excel value/type handling
This commit is contained in:
@ -42,6 +42,7 @@ import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.frontend.QFrontendFieldMetaData;
|
||||
import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryRecordStore;
|
||||
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadProfile;
|
||||
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadProfileField;
|
||||
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess;
|
||||
import com.kingsrook.qqq.backend.core.utils.JsonUtils;
|
||||
@ -58,7 +59,7 @@ import static org.junit.jupiter.api.Assertions.assertNull;
|
||||
/*******************************************************************************
|
||||
** Unit test for full bulk insert process
|
||||
*******************************************************************************/
|
||||
class BulkInsertV2Test extends BaseTest
|
||||
class BulkInsertV2FullProcessTest extends BaseTest
|
||||
{
|
||||
|
||||
/*******************************************************************************
|
||||
@ -124,7 +125,7 @@ class BulkInsertV2Test extends BaseTest
|
||||
|
||||
QInstance qInstance = QContext.getQInstance();
|
||||
String processName = "PersonBulkInsertV2";
|
||||
new QInstanceEnricher(qInstance).defineTableBulkInsertV2(qInstance, qInstance.getTable(TestUtils.TABLE_NAME_PERSON_MEMORY), processName);
|
||||
new QInstanceEnricher(qInstance).defineTableBulkInsert(qInstance, qInstance.getTable(TestUtils.TABLE_NAME_PERSON_MEMORY), processName);
|
||||
|
||||
/////////////////////////////////////////////////////////
|
||||
// start the process - expect to go to the upload step //
|
||||
@ -159,6 +160,16 @@ class BulkInsertV2Test extends BaseTest
|
||||
runProcessOutput = new RunProcessAction().execute(runProcessInput);
|
||||
assertEquals(List.of("Id", "Create Date", "Modify Date", "First Name", "Last Name", "Birth Date", "Email", "Home State", "noOfShoes"), runProcessOutput.getValue("headerValues"));
|
||||
assertEquals(List.of("A", "B", "C", "D", "E", "F", "G", "H", "I"), runProcessOutput.getValue("headerLetters"));
|
||||
|
||||
//////////////////////////////////////////////////////
|
||||
// assert about the suggested mapping that was done //
|
||||
//////////////////////////////////////////////////////
|
||||
Serializable bulkLoadProfile = runProcessOutput.getValue("bulkLoadProfile");
|
||||
assertThat(bulkLoadProfile).isInstanceOf(BulkLoadProfile.class);
|
||||
assertThat(((BulkLoadProfile) bulkLoadProfile).getFieldList()).hasSizeGreaterThan(5);
|
||||
assertEquals("birthDate", ((BulkLoadProfile) bulkLoadProfile).getFieldList().get(0).getFieldName());
|
||||
assertEquals(5, ((BulkLoadProfile) bulkLoadProfile).getFieldList().get(0).getColumnIndex());
|
||||
|
||||
assertThat(runProcessOutput.getProcessState().getNextStepName()).isPresent().get().isEqualTo("fileMapping");
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
@ -0,0 +1,209 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2024. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* This program is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Affero General Public License as
|
||||
* published by the Free Software Foundation, either version 3 of the
|
||||
* License, or (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Affero General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping;
|
||||
|
||||
|
||||
import java.util.List;
|
||||
import com.kingsrook.qqq.backend.core.BaseTest;
|
||||
import com.kingsrook.qqq.backend.core.context.QContext;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
|
||||
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadProfile;
|
||||
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadProfileField;
|
||||
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadTableStructure;
|
||||
import com.kingsrook.qqq.backend.core.utils.TestUtils;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertNull;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Unit test for BulkLoadMappingSuggester
|
||||
*******************************************************************************/
|
||||
class BulkLoadMappingSuggesterTest extends BaseTest
|
||||
{
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testSimpleFlat()
|
||||
{
|
||||
BulkLoadTableStructure tableStructure = BulkLoadTableStructureBuilder.buildTableStructure(TestUtils.TABLE_NAME_PERSON_MEMORY);
|
||||
List<String> headerRow = List.of("Id", "First Name", "lastname", "email", "homestate");
|
||||
|
||||
BulkLoadProfile bulkLoadProfile = new BulkLoadMappingSuggester().suggestBulkLoadMappingProfile(tableStructure, headerRow);
|
||||
assertEquals("v1", bulkLoadProfile.getVersion());
|
||||
assertEquals("FLAT", bulkLoadProfile.getLayout());
|
||||
assertNull(getFieldByName(bulkLoadProfile, "id"));
|
||||
assertEquals(1, getFieldByName(bulkLoadProfile, "firstName").getColumnIndex());
|
||||
assertEquals(2, getFieldByName(bulkLoadProfile, "lastName").getColumnIndex());
|
||||
assertEquals(3, getFieldByName(bulkLoadProfile, "email").getColumnIndex());
|
||||
assertEquals(4, getFieldByName(bulkLoadProfile, "homeStateId").getColumnIndex());
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testSimpleTall()
|
||||
{
|
||||
BulkLoadTableStructure tableStructure = BulkLoadTableStructureBuilder.buildTableStructure(TestUtils.TABLE_NAME_ORDER);
|
||||
List<String> headerRow = List.of("orderNo", "shipto name", "sku", "quantity");
|
||||
|
||||
BulkLoadProfile bulkLoadProfile = new BulkLoadMappingSuggester().suggestBulkLoadMappingProfile(tableStructure, headerRow);
|
||||
assertEquals("v1", bulkLoadProfile.getVersion());
|
||||
assertEquals("TALL", bulkLoadProfile.getLayout());
|
||||
assertEquals(0, getFieldByName(bulkLoadProfile, "orderNo").getColumnIndex());
|
||||
assertEquals(1, getFieldByName(bulkLoadProfile, "shipToName").getColumnIndex());
|
||||
assertEquals(2, getFieldByName(bulkLoadProfile, "orderLine.sku").getColumnIndex());
|
||||
assertEquals(3, getFieldByName(bulkLoadProfile, "orderLine.quantity").getColumnIndex());
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testTallWithTableNamesOnAssociations()
|
||||
{
|
||||
BulkLoadTableStructure tableStructure = BulkLoadTableStructureBuilder.buildTableStructure(TestUtils.TABLE_NAME_ORDER);
|
||||
List<String> headerRow = List.of("Order No", "Ship To Name", "Order Line: SKU", "Order Line: Quantity");
|
||||
|
||||
BulkLoadProfile bulkLoadProfile = new BulkLoadMappingSuggester().suggestBulkLoadMappingProfile(tableStructure, headerRow);
|
||||
assertEquals("v1", bulkLoadProfile.getVersion());
|
||||
assertEquals("TALL", bulkLoadProfile.getLayout());
|
||||
assertEquals(0, getFieldByName(bulkLoadProfile, "orderNo").getColumnIndex());
|
||||
assertEquals(1, getFieldByName(bulkLoadProfile, "shipToName").getColumnIndex());
|
||||
assertEquals(2, getFieldByName(bulkLoadProfile, "orderLine.sku").getColumnIndex());
|
||||
assertEquals(3, getFieldByName(bulkLoadProfile, "orderLine.quantity").getColumnIndex());
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testChallengingAddress1And2()
|
||||
{
|
||||
try
|
||||
{
|
||||
{
|
||||
QTableMetaData table = QContext.getQInstance().getTable(TestUtils.TABLE_NAME_ORDER);
|
||||
table.addField(new QFieldMetaData("address1", QFieldType.STRING));
|
||||
table.addField(new QFieldMetaData("address2", QFieldType.STRING));
|
||||
|
||||
BulkLoadTableStructure tableStructure = BulkLoadTableStructureBuilder.buildTableStructure(TestUtils.TABLE_NAME_ORDER);
|
||||
List<String> headerRow = List.of("orderNo", "ship to name", "address 1", "address 2");
|
||||
|
||||
BulkLoadProfile bulkLoadProfile = new BulkLoadMappingSuggester().suggestBulkLoadMappingProfile(tableStructure, headerRow);
|
||||
assertEquals(0, getFieldByName(bulkLoadProfile, "orderNo").getColumnIndex());
|
||||
assertEquals(1, getFieldByName(bulkLoadProfile, "shipToName").getColumnIndex());
|
||||
assertEquals(2, getFieldByName(bulkLoadProfile, "address1").getColumnIndex());
|
||||
assertEquals(3, getFieldByName(bulkLoadProfile, "address2").getColumnIndex());
|
||||
reInitInstanceInContext(TestUtils.defineInstance());
|
||||
}
|
||||
|
||||
{
|
||||
QTableMetaData table = QContext.getQInstance().getTable(TestUtils.TABLE_NAME_ORDER);
|
||||
table.addField(new QFieldMetaData("address", QFieldType.STRING));
|
||||
table.addField(new QFieldMetaData("address2", QFieldType.STRING));
|
||||
|
||||
BulkLoadTableStructure tableStructure = BulkLoadTableStructureBuilder.buildTableStructure(TestUtils.TABLE_NAME_ORDER);
|
||||
List<String> headerRow = List.of("orderNo", "ship to name", "address 1", "address 2");
|
||||
|
||||
BulkLoadProfile bulkLoadProfile = new BulkLoadMappingSuggester().suggestBulkLoadMappingProfile(tableStructure, headerRow);
|
||||
assertEquals(0, getFieldByName(bulkLoadProfile, "orderNo").getColumnIndex());
|
||||
assertEquals(1, getFieldByName(bulkLoadProfile, "shipToName").getColumnIndex());
|
||||
assertEquals(2, getFieldByName(bulkLoadProfile, "address").getColumnIndex());
|
||||
assertEquals(3, getFieldByName(bulkLoadProfile, "address2").getColumnIndex());
|
||||
reInitInstanceInContext(TestUtils.defineInstance());
|
||||
}
|
||||
|
||||
{
|
||||
QTableMetaData table = QContext.getQInstance().getTable(TestUtils.TABLE_NAME_ORDER);
|
||||
table.addField(new QFieldMetaData("address1", QFieldType.STRING));
|
||||
table.addField(new QFieldMetaData("address2", QFieldType.STRING));
|
||||
|
||||
BulkLoadTableStructure tableStructure = BulkLoadTableStructureBuilder.buildTableStructure(TestUtils.TABLE_NAME_ORDER);
|
||||
List<String> headerRow = List.of("orderNo", "ship to name", "address", "address 2");
|
||||
|
||||
BulkLoadProfile bulkLoadProfile = new BulkLoadMappingSuggester().suggestBulkLoadMappingProfile(tableStructure, headerRow);
|
||||
assertEquals(0, getFieldByName(bulkLoadProfile, "orderNo").getColumnIndex());
|
||||
assertEquals(1, getFieldByName(bulkLoadProfile, "shipToName").getColumnIndex());
|
||||
assertEquals(2, getFieldByName(bulkLoadProfile, "address1").getColumnIndex());
|
||||
assertEquals(3, getFieldByName(bulkLoadProfile, "address2").getColumnIndex());
|
||||
reInitInstanceInContext(TestUtils.defineInstance());
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
reInitInstanceInContext(TestUtils.defineInstance());
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testSimpleWide()
|
||||
{
|
||||
BulkLoadTableStructure tableStructure = BulkLoadTableStructureBuilder.buildTableStructure(TestUtils.TABLE_NAME_ORDER);
|
||||
List<String> headerRow = List.of("orderNo", "ship to name", "sku", "quantity1", "sku 2", "quantity 2");
|
||||
|
||||
BulkLoadProfile bulkLoadProfile = new BulkLoadMappingSuggester().suggestBulkLoadMappingProfile(tableStructure, headerRow);
|
||||
assertEquals("v1", bulkLoadProfile.getVersion());
|
||||
assertEquals("WIDE", bulkLoadProfile.getLayout());
|
||||
assertEquals(0, getFieldByName(bulkLoadProfile, "orderNo").getColumnIndex());
|
||||
assertEquals(1, getFieldByName(bulkLoadProfile, "shipToName").getColumnIndex());
|
||||
assertEquals(2, getFieldByName(bulkLoadProfile, "orderLine.sku,0").getColumnIndex());
|
||||
assertEquals(3, getFieldByName(bulkLoadProfile, "orderLine.quantity,0").getColumnIndex());
|
||||
assertEquals(4, getFieldByName(bulkLoadProfile, "orderLine.sku,1").getColumnIndex());
|
||||
assertEquals(5, getFieldByName(bulkLoadProfile, "orderLine.quantity,1").getColumnIndex());
|
||||
|
||||
/////////////////////////////////////////////////////////////////
|
||||
// assert that the order of fields matches the file's ordering //
|
||||
/////////////////////////////////////////////////////////////////
|
||||
assertEquals(List.of("orderNo", "shipToName", "orderLine.sku,0", "orderLine.quantity,0", "orderLine.sku,1", "orderLine.quantity,1"),
|
||||
bulkLoadProfile.getFieldList().stream().map(f -> f.getFieldName()).toList());
|
||||
}
|
||||
|
||||
|
||||
|
||||
/***************************************************************************
|
||||
**
|
||||
***************************************************************************/
|
||||
private BulkLoadProfileField getFieldByName(BulkLoadProfile bulkLoadProfile, String fieldName)
|
||||
{
|
||||
return (bulkLoadProfile.getFieldList().stream()
|
||||
.filter(f -> f.getFieldName().equals(fieldName))
|
||||
.findFirst().orElse(null));
|
||||
}
|
||||
|
||||
}
|
||||
@ -30,6 +30,7 @@ import com.kingsrook.qqq.backend.core.BaseTest;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.TestFileToRows;
|
||||
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkInsertMapping;
|
||||
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow;
|
||||
import com.kingsrook.qqq.backend.core.utils.TestUtils;
|
||||
import com.kingsrook.qqq.backend.core.utils.collections.ListBuilder;
|
||||
|
||||
@ -29,6 +29,7 @@ import com.kingsrook.qqq.backend.core.BaseTest;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.CsvFileToRows;
|
||||
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkInsertMapping;
|
||||
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow;
|
||||
import com.kingsrook.qqq.backend.core.utils.TestUtils;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
@ -29,6 +29,7 @@ import com.kingsrook.qqq.backend.core.BaseTest;
|
||||
import com.kingsrook.qqq.backend.core.context.QContext;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkInsertMapping;
|
||||
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
|
||||
import com.kingsrook.qqq.backend.core.utils.TestUtils;
|
||||
import org.json.JSONArray;
|
||||
|
||||
@ -0,0 +1,175 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2024. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* This program is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Affero General Public License as
|
||||
* published by the Free Software Foundation, either version 3 of the
|
||||
* License, or (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Affero General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping;
|
||||
|
||||
|
||||
import java.io.Serializable;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import com.kingsrook.qqq.backend.core.BaseTest;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.CsvFileToRows;
|
||||
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkInsertMapping;
|
||||
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow;
|
||||
import com.kingsrook.qqq.backend.core.utils.TestUtils;
|
||||
import com.kingsrook.qqq.backend.core.utils.collections.MapBuilder;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Unit test for WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMapping
|
||||
*******************************************************************************/
|
||||
class WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMappingTest extends BaseTest
|
||||
{
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testOrderAndLinesWithoutDupes() throws QException
|
||||
{
|
||||
String csv = """
|
||||
orderNo, Ship To, lastName, SKU 1, Quantity 1, SKU 2, Quantity 2, SKU 3, Quantity 3
|
||||
1, Homer, Simpson, DONUT, 12, BEER, 500, COUCH, 1
|
||||
2, Ned, Flanders, BIBLE, 7, LAWNMOWER, 1
|
||||
""";
|
||||
|
||||
CsvFileToRows fileToRows = CsvFileToRows.forString(csv);
|
||||
BulkLoadFileRow header = fileToRows.next();
|
||||
|
||||
WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMapping rowsToRecord = new WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMapping();
|
||||
|
||||
BulkInsertMapping mapping = new BulkInsertMapping()
|
||||
.withFieldNameToHeaderNameMap(Map.of(
|
||||
"orderNo", "orderNo",
|
||||
"shipToName", "Ship To",
|
||||
"orderLine.sku,0", "SKU 1",
|
||||
"orderLine.quantity,0", "Quantity 1",
|
||||
"orderLine.sku,1", "SKU 2",
|
||||
"orderLine.quantity,1", "Quantity 2",
|
||||
"orderLine.sku,2", "SKU 3",
|
||||
"orderLine.quantity,2", "Quantity 3"
|
||||
))
|
||||
.withMappedAssociations(List.of("orderLine"))
|
||||
.withTableName(TestUtils.TABLE_NAME_ORDER)
|
||||
.withLayout(BulkInsertMapping.Layout.WIDE)
|
||||
.withHasHeaderRow(true);
|
||||
|
||||
List<QRecord> records = rowsToRecord.nextPage(fileToRows, header, mapping, Integer.MAX_VALUE);
|
||||
assertEquals(2, records.size());
|
||||
|
||||
QRecord order = records.get(0);
|
||||
assertEquals(1, order.getValueInteger("orderNo"));
|
||||
assertEquals("Homer", order.getValueString("shipToName"));
|
||||
assertEquals(List.of("DONUT", "BEER", "COUCH"), getValues(order.getAssociatedRecords().get("orderLine"), "sku"));
|
||||
assertEquals(List.of(12, 500, 1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity"));
|
||||
|
||||
order = records.get(1);
|
||||
assertEquals(2, order.getValueInteger("orderNo"));
|
||||
assertEquals("Ned", order.getValueString("shipToName"));
|
||||
assertEquals(List.of("BIBLE", "LAWNMOWER"), getValues(order.getAssociatedRecords().get("orderLine"), "sku"));
|
||||
assertEquals(List.of(7, 1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity"));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testOrderLinesAndOrderExtrinsicWithoutDupes() throws QException
|
||||
{
|
||||
String csv = """
|
||||
orderNo, Ship To, lastName, SKU 1, Quantity 1, SKU 2, Quantity 2, SKU 3, Quantity 3, Extrinsic Key 1, Extrinsic Value 1, Extrinsic Key 2, Extrinsic Value 2
|
||||
1, Homer, Simpson, DONUT, 12, BEER, 500, COUCH, 1, Store Name, QQQ Mart, Coupon Code, 10QOff
|
||||
2, Ned, Flanders, BIBLE, 7, LAWNMOWER, 1
|
||||
""";
|
||||
|
||||
CsvFileToRows fileToRows = CsvFileToRows.forString(csv);
|
||||
BulkLoadFileRow header = fileToRows.next();
|
||||
|
||||
WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMapping rowsToRecord = new WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMapping();
|
||||
|
||||
BulkInsertMapping mapping = new BulkInsertMapping()
|
||||
.withFieldNameToHeaderNameMap(MapBuilder.of(() -> new HashMap<String, String>())
|
||||
.with("orderNo", "orderNo")
|
||||
.with("shipToName", "Ship To")
|
||||
|
||||
.with("orderLine.sku,0", "SKU 1")
|
||||
.with("orderLine.quantity,0", "Quantity 1")
|
||||
.with("orderLine.sku,1", "SKU 2")
|
||||
.with("orderLine.quantity,1", "Quantity 2")
|
||||
.with("orderLine.sku,2", "SKU 3")
|
||||
.with("orderLine.quantity,2", "Quantity 3")
|
||||
|
||||
.with("extrinsics.key,0", "Extrinsic Key 1")
|
||||
.with("extrinsics.value,0", "Extrinsic Value 1")
|
||||
.with("extrinsics.key,1", "Extrinsic Key 2")
|
||||
.with("extrinsics.value,1", "Extrinsic Value 2")
|
||||
.build()
|
||||
)
|
||||
.withFieldNameToDefaultValueMap(Map.of(
|
||||
"orderLine.lineNumber,0", "1",
|
||||
"orderLine.lineNumber,1", "2",
|
||||
"orderLine.lineNumber,2", "3"
|
||||
))
|
||||
.withMappedAssociations(List.of("orderLine", "extrinsics"))
|
||||
.withTableName(TestUtils.TABLE_NAME_ORDER)
|
||||
.withLayout(BulkInsertMapping.Layout.WIDE)
|
||||
.withHasHeaderRow(true);
|
||||
|
||||
List<QRecord> records = rowsToRecord.nextPage(fileToRows, header, mapping, Integer.MAX_VALUE);
|
||||
assertEquals(2, records.size());
|
||||
|
||||
QRecord order = records.get(0);
|
||||
assertEquals(1, order.getValueInteger("orderNo"));
|
||||
assertEquals("Homer", order.getValueString("shipToName"));
|
||||
assertEquals(List.of("DONUT", "BEER", "COUCH"), getValues(order.getAssociatedRecords().get("orderLine"), "sku"));
|
||||
assertEquals(List.of(12, 500, 1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity"));
|
||||
assertEquals(List.of("1", "2", "3"), getValues(order.getAssociatedRecords().get("orderLine"), "lineNumber"));
|
||||
assertEquals(List.of("Store Name", "Coupon Code"), getValues(order.getAssociatedRecords().get("extrinsics"), "key"));
|
||||
assertEquals(List.of("QQQ Mart", "10QOff"), getValues(order.getAssociatedRecords().get("extrinsics"), "value"));
|
||||
|
||||
order = records.get(1);
|
||||
assertEquals(2, order.getValueInteger("orderNo"));
|
||||
assertEquals("Ned", order.getValueString("shipToName"));
|
||||
assertEquals(List.of("BIBLE", "LAWNMOWER"), getValues(order.getAssociatedRecords().get("orderLine"), "sku"));
|
||||
assertEquals(List.of(7, 1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity"));
|
||||
assertEquals(List.of("1", "2"), getValues(order.getAssociatedRecords().get("orderLine"), "lineNumber"));
|
||||
assertThat(order.getAssociatedRecords().get("extrinsics")).isNullOrEmpty();
|
||||
}
|
||||
|
||||
|
||||
|
||||
/***************************************************************************
|
||||
**
|
||||
***************************************************************************/
|
||||
private List<Serializable> getValues(List<QRecord> records, String fieldName)
|
||||
{
|
||||
return (records.stream().map(r -> r.getValue(fieldName)).toList());
|
||||
}
|
||||
|
||||
}
|
||||
@ -1,269 +0,0 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2024. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* This program is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Affero General Public License as
|
||||
* published by the Free Software Foundation, either version 3 of the
|
||||
* License, or (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Affero General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping;
|
||||
|
||||
|
||||
import java.io.Serializable;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import com.kingsrook.qqq.backend.core.BaseTest;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.CsvFileToRows;
|
||||
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow;
|
||||
import com.kingsrook.qqq.backend.core.utils.TestUtils;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Unit test for WideRowsToRecord
|
||||
*******************************************************************************/
|
||||
class WideRowsToRecordWithExplicitMappingTest extends BaseTest
|
||||
{
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testOrderAndLinesWithoutDupes() throws QException
|
||||
{
|
||||
String csv = """
|
||||
orderNo, Ship To, lastName, SKU 1, Quantity 1, SKU 2, Quantity 2, SKU 3, Quantity 3
|
||||
1, Homer, Simpson, DONUT, 12, BEER, 500, COUCH, 1
|
||||
2, Ned, Flanders, BIBLE, 7, LAWNMOWER, 1
|
||||
""";
|
||||
|
||||
CsvFileToRows fileToRows = CsvFileToRows.forString(csv);
|
||||
BulkLoadFileRow header = fileToRows.next();
|
||||
|
||||
WideRowsToRecordWithExplicitMapping rowsToRecord = new WideRowsToRecordWithExplicitMapping();
|
||||
|
||||
BulkInsertMapping mapping = new BulkInsertMapping()
|
||||
.withFieldNameToHeaderNameMap(Map.of(
|
||||
"orderNo", "orderNo",
|
||||
"shipToName", "Ship To"
|
||||
))
|
||||
.withWideLayoutMapping(Map.of(
|
||||
"orderLine", new BulkInsertWideLayoutMapping(List.of(
|
||||
new BulkInsertWideLayoutMapping.ChildRecordMapping(Map.of("sku", "SKU 1", "quantity", "Quantity 1")),
|
||||
new BulkInsertWideLayoutMapping.ChildRecordMapping(Map.of("sku", "SKU 2", "quantity", "Quantity 2")),
|
||||
new BulkInsertWideLayoutMapping.ChildRecordMapping(Map.of("sku", "SKU 3", "quantity", "Quantity 3"))
|
||||
))
|
||||
))
|
||||
.withMappedAssociations(List.of("orderLine"))
|
||||
.withTableName(TestUtils.TABLE_NAME_ORDER)
|
||||
.withLayout(BulkInsertMapping.Layout.WIDE)
|
||||
.withHasHeaderRow(true);
|
||||
|
||||
List<QRecord> records = rowsToRecord.nextPage(fileToRows, header, mapping, Integer.MAX_VALUE);
|
||||
assertEquals(2, records.size());
|
||||
|
||||
QRecord order = records.get(0);
|
||||
assertEquals(1, order.getValueInteger("orderNo"));
|
||||
assertEquals("Homer", order.getValueString("shipToName"));
|
||||
assertEquals(List.of("DONUT", "BEER", "COUCH"), getValues(order.getAssociatedRecords().get("orderLine"), "sku"));
|
||||
assertEquals(List.of(12, 500, 1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity"));
|
||||
|
||||
order = records.get(1);
|
||||
assertEquals(2, order.getValueInteger("orderNo"));
|
||||
assertEquals("Ned", order.getValueString("shipToName"));
|
||||
assertEquals(List.of("BIBLE", "LAWNMOWER"), getValues(order.getAssociatedRecords().get("orderLine"), "sku"));
|
||||
assertEquals(List.of(7, 1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity"));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testOrderLinesAndOrderExtrinsicWithoutDupes() throws QException
|
||||
{
|
||||
String csv = """
|
||||
orderNo, Ship To, lastName, SKU 1, Quantity 1, SKU 2, Quantity 2, SKU 3, Quantity 3, Extrinsic Key 1, Extrinsic Value 1, Extrinsic Key 2, Extrinsic Value 2
|
||||
1, Homer, Simpson, DONUT, 12, BEER, 500, COUCH, 1, Store Name, QQQ Mart, Coupon Code, 10QOff
|
||||
2, Ned, Flanders, BIBLE, 7, LAWNMOWER, 1
|
||||
""";
|
||||
|
||||
CsvFileToRows fileToRows = CsvFileToRows.forString(csv);
|
||||
BulkLoadFileRow header = fileToRows.next();
|
||||
|
||||
WideRowsToRecordWithExplicitMapping rowsToRecord = new WideRowsToRecordWithExplicitMapping();
|
||||
|
||||
BulkInsertMapping mapping = new BulkInsertMapping()
|
||||
.withFieldNameToHeaderNameMap(Map.of(
|
||||
"orderNo", "orderNo",
|
||||
"shipToName", "Ship To"
|
||||
))
|
||||
.withWideLayoutMapping(Map.of(
|
||||
"orderLine", new BulkInsertWideLayoutMapping(List.of(
|
||||
new BulkInsertWideLayoutMapping.ChildRecordMapping(Map.of("sku", "SKU 1", "quantity", "Quantity 1")),
|
||||
new BulkInsertWideLayoutMapping.ChildRecordMapping(Map.of("sku", "SKU 2", "quantity", "Quantity 2")),
|
||||
new BulkInsertWideLayoutMapping.ChildRecordMapping(Map.of("sku", "SKU 3", "quantity", "Quantity 3"))
|
||||
)),
|
||||
"extrinsics", new BulkInsertWideLayoutMapping(List.of(
|
||||
new BulkInsertWideLayoutMapping.ChildRecordMapping(Map.of("key", "Extrinsic Key 1", "value", "Extrinsic Value 1")),
|
||||
new BulkInsertWideLayoutMapping.ChildRecordMapping(Map.of("key", "Extrinsic Key 2", "value", "Extrinsic Value 2"))
|
||||
))
|
||||
))
|
||||
.withMappedAssociations(List.of("orderLine", "extrinsics"))
|
||||
.withTableName(TestUtils.TABLE_NAME_ORDER)
|
||||
.withLayout(BulkInsertMapping.Layout.WIDE)
|
||||
.withHasHeaderRow(true);
|
||||
|
||||
List<QRecord> records = rowsToRecord.nextPage(fileToRows, header, mapping, Integer.MAX_VALUE);
|
||||
assertEquals(2, records.size());
|
||||
|
||||
QRecord order = records.get(0);
|
||||
assertEquals(1, order.getValueInteger("orderNo"));
|
||||
assertEquals("Homer", order.getValueString("shipToName"));
|
||||
assertEquals(List.of("DONUT", "BEER", "COUCH"), getValues(order.getAssociatedRecords().get("orderLine"), "sku"));
|
||||
assertEquals(List.of(12, 500, 1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity"));
|
||||
assertEquals(List.of("Store Name", "Coupon Code"), getValues(order.getAssociatedRecords().get("extrinsics"), "key"));
|
||||
assertEquals(List.of("QQQ Mart", "10QOff"), getValues(order.getAssociatedRecords().get("extrinsics"), "value"));
|
||||
|
||||
order = records.get(1);
|
||||
assertEquals(2, order.getValueInteger("orderNo"));
|
||||
assertEquals("Ned", order.getValueString("shipToName"));
|
||||
assertEquals(List.of("BIBLE", "LAWNMOWER"), getValues(order.getAssociatedRecords().get("orderLine"), "sku"));
|
||||
assertEquals(List.of(7, 1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity"));
|
||||
assertThat(order.getAssociatedRecords().get("extrinsics")).isNullOrEmpty();
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testOrderLinesWithLineExtrinsicsAndOrderExtrinsicWithoutDupes() throws QException
|
||||
{
|
||||
String csv = """
|
||||
orderNo, Ship To, lastName, Extrinsic Key 1, Extrinsic Value 1, Extrinsic Key 2, Extrinsic Value 2, SKU 1, Quantity 1, Line Extrinsic Key 1.1, Line Extrinsic Value 1.1, Line Extrinsic Key 1.2, Line Extrinsic Value 1.2, SKU 2, Quantity 2, Line Extrinsic Key 2.1, Line Extrinsic Value 2.1, SKU 3, Quantity 3, Line Extrinsic Key 3.1, Line Extrinsic Value 3.1, Line Extrinsic Key 3.2
|
||||
1, Homer, Simpson, Store Name, QQQ Mart, Coupon Code, 10QOff, DONUT, 12, Flavor, Chocolate, Size, Large, BEER, 500, Flavor, Hops, COUCH, 1, Color, Brown, foo,
|
||||
2, Ned, Flanders, , , , , BIBLE, 7, Flavor, King James, Size, X-Large, LAWNMOWER, 1
|
||||
""";
|
||||
|
||||
Integer defaultStoreId = 42;
|
||||
String defaultLineNo = "47";
|
||||
String defaultLineExtraValue = "bar";
|
||||
|
||||
CsvFileToRows fileToRows = CsvFileToRows.forString(csv);
|
||||
BulkLoadFileRow header = fileToRows.next();
|
||||
|
||||
WideRowsToRecordWithExplicitMapping rowsToRecord = new WideRowsToRecordWithExplicitMapping();
|
||||
|
||||
BulkInsertMapping mapping = new BulkInsertMapping()
|
||||
.withFieldNameToHeaderNameMap(Map.of(
|
||||
"orderNo", "orderNo",
|
||||
"shipToName", "Ship To"
|
||||
))
|
||||
.withMappedAssociations(List.of("orderLine", "extrinsics", "orderLine.extrinsics"))
|
||||
.withWideLayoutMapping(Map.of(
|
||||
"orderLine", new BulkInsertWideLayoutMapping(List.of(
|
||||
new BulkInsertWideLayoutMapping.ChildRecordMapping(
|
||||
Map.of("sku", "SKU 1", "quantity", "Quantity 1"),
|
||||
Map.of("extrinsics", new BulkInsertWideLayoutMapping(List.of(
|
||||
new BulkInsertWideLayoutMapping.ChildRecordMapping(Map.of("key", "Line Extrinsic Key 1.1", "value", "Line Extrinsic Value 1.1")),
|
||||
new BulkInsertWideLayoutMapping.ChildRecordMapping(Map.of("key", "Line Extrinsic Key 1.2", "value", "Line Extrinsic Value 1.2"))
|
||||
)))),
|
||||
new BulkInsertWideLayoutMapping.ChildRecordMapping(
|
||||
Map.of("sku", "SKU 2", "quantity", "Quantity 2"),
|
||||
Map.of("extrinsics", new BulkInsertWideLayoutMapping(List.of(
|
||||
new BulkInsertWideLayoutMapping.ChildRecordMapping(Map.of("key", "Line Extrinsic Key 2.1", "value", "Line Extrinsic Value 2.1")),
|
||||
new BulkInsertWideLayoutMapping.ChildRecordMapping(Map.of("key", "Line Extrinsic Key 2.2", "value", "Line Extrinsic Value 2.2"))
|
||||
)))),
|
||||
new BulkInsertWideLayoutMapping.ChildRecordMapping(
|
||||
Map.of("sku", "SKU 3", "quantity", "Quantity 3"),
|
||||
Map.of("extrinsics", new BulkInsertWideLayoutMapping(List.of(
|
||||
new BulkInsertWideLayoutMapping.ChildRecordMapping(Map.of("key", "Line Extrinsic Key 3.1", "value", "Line Extrinsic Value 3.1")),
|
||||
new BulkInsertWideLayoutMapping.ChildRecordMapping(Map.of("key", "Line Extrinsic Key 3.2", "value", "Line Extrinsic Value 3.2"))
|
||||
))))
|
||||
)),
|
||||
"extrinsics", new BulkInsertWideLayoutMapping(List.of(
|
||||
new BulkInsertWideLayoutMapping.ChildRecordMapping(Map.of("key", "Extrinsic Key 1", "value", "Extrinsic Value 1")),
|
||||
new BulkInsertWideLayoutMapping.ChildRecordMapping(Map.of("key", "Extrinsic Key 2", "value", "Extrinsic Value 2"))
|
||||
))
|
||||
))
|
||||
|
||||
.withFieldNameToValueMapping(Map.of("orderLine.extrinsics.value", Map.of("Large", "L", "X-Large", "XL")))
|
||||
.withFieldNameToDefaultValueMap(Map.of(
|
||||
"storeId", defaultStoreId,
|
||||
"orderLine.lineNumber", defaultLineNo,
|
||||
"orderLine.extrinsics.value", defaultLineExtraValue
|
||||
))
|
||||
.withTableName(TestUtils.TABLE_NAME_ORDER)
|
||||
.withLayout(BulkInsertMapping.Layout.WIDE)
|
||||
.withHasHeaderRow(true);
|
||||
|
||||
List<QRecord> records = rowsToRecord.nextPage(fileToRows, header, mapping, Integer.MAX_VALUE);
|
||||
assertEquals(2, records.size());
|
||||
|
||||
QRecord order = records.get(0);
|
||||
assertEquals(1, order.getValueInteger("orderNo"));
|
||||
assertEquals("Homer", order.getValueString("shipToName"));
|
||||
assertEquals(defaultStoreId, order.getValue("storeId"));
|
||||
assertEquals(List.of("DONUT", "BEER", "COUCH"), getValues(order.getAssociatedRecords().get("orderLine"), "sku"));
|
||||
assertEquals(List.of(12, 500, 1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity"));
|
||||
assertEquals(List.of(defaultLineNo, defaultLineNo, defaultLineNo), getValues(order.getAssociatedRecords().get("orderLine"), "lineNumber"));
|
||||
assertEquals(List.of("Store Name", "Coupon Code"), getValues(order.getAssociatedRecords().get("extrinsics"), "key"));
|
||||
assertEquals(List.of("QQQ Mart", "10QOff"), getValues(order.getAssociatedRecords().get("extrinsics"), "value"));
|
||||
|
||||
QRecord lineItem = order.getAssociatedRecords().get("orderLine").get(0);
|
||||
assertEquals(List.of("Flavor", "Size"), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "key"));
|
||||
assertEquals(List.of("Chocolate", "L"), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "value"));
|
||||
|
||||
lineItem = order.getAssociatedRecords().get("orderLine").get(1);
|
||||
assertEquals(List.of("Flavor"), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "key"));
|
||||
assertEquals(List.of("Hops"), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "value"));
|
||||
|
||||
lineItem = order.getAssociatedRecords().get("orderLine").get(2);
|
||||
assertEquals(List.of("Color", "foo"), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "key"));
|
||||
assertEquals(List.of("Brown", defaultLineExtraValue), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "value"));
|
||||
|
||||
order = records.get(1);
|
||||
assertEquals(2, order.getValueInteger("orderNo"));
|
||||
assertEquals("Ned", order.getValueString("shipToName"));
|
||||
assertEquals(defaultStoreId, order.getValue("storeId"));
|
||||
assertEquals(List.of("BIBLE", "LAWNMOWER"), getValues(order.getAssociatedRecords().get("orderLine"), "sku"));
|
||||
assertEquals(List.of(7, 1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity"));
|
||||
assertEquals(List.of(defaultLineNo, defaultLineNo), getValues(order.getAssociatedRecords().get("orderLine"), "lineNumber"));
|
||||
assertThat(order.getAssociatedRecords().get("extrinsics")).isNullOrEmpty();
|
||||
|
||||
lineItem = order.getAssociatedRecords().get("orderLine").get(0);
|
||||
assertEquals(List.of("Flavor", "Size"), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "key"));
|
||||
assertEquals(List.of("King James", "XL"), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "value"));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/***************************************************************************
|
||||
**
|
||||
***************************************************************************/
|
||||
private List<Serializable> getValues(List<QRecord> records, String fieldName)
|
||||
{
|
||||
return (records.stream().map(r -> r.getValue(fieldName)).toList());
|
||||
}
|
||||
|
||||
}
|
||||
@ -29,6 +29,7 @@ import com.kingsrook.qqq.backend.core.BaseTest;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.CsvFileToRows;
|
||||
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkInsertMapping;
|
||||
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow;
|
||||
import com.kingsrook.qqq.backend.core.utils.TestUtils;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
Reference in New Issue
Block a user