mirror of
https://github.com/Kingsrook/qqq.git
synced 2025-07-19 05:30:43 +00:00
Add unique keys, and checking of them in bulk load; add some more validation (sqs and unique keys)
This commit is contained in:
@ -73,7 +73,7 @@ class CsvToQRecordAdapterTest
|
||||
try
|
||||
{
|
||||
CsvToQRecordAdapter csvToQRecordAdapter = new CsvToQRecordAdapter();
|
||||
List<QRecord> qRecords = csvToQRecordAdapter.buildRecordsFromCsv(csv, TestUtils.defineTablePerson(), null);
|
||||
List<QRecord> qRecords = csvToQRecordAdapter.buildRecordsFromCsv(csv, TestUtils.defineTablePerson(), null);
|
||||
System.out.println(qRecords);
|
||||
}
|
||||
catch(IllegalArgumentException iae)
|
||||
@ -94,7 +94,7 @@ class CsvToQRecordAdapterTest
|
||||
public void test_buildRecordsFromCsv_emptyList()
|
||||
{
|
||||
CsvToQRecordAdapter csvToQRecordAdapter = new CsvToQRecordAdapter();
|
||||
List<QRecord> qRecords = csvToQRecordAdapter.buildRecordsFromCsv(getPersonCsvHeader(), TestUtils.defineTablePerson(), null);
|
||||
List<QRecord> qRecords = csvToQRecordAdapter.buildRecordsFromCsv(getPersonCsvHeader(), TestUtils.defineTablePerson(), null);
|
||||
assertNotNull(qRecords);
|
||||
assertTrue(qRecords.isEmpty());
|
||||
}
|
||||
@ -144,7 +144,7 @@ class CsvToQRecordAdapterTest
|
||||
public void test_buildRecordsFromCsv_oneRowStandardHeaderNoMapping()
|
||||
{
|
||||
CsvToQRecordAdapter csvToQRecordAdapter = new CsvToQRecordAdapter();
|
||||
List<QRecord> qRecords = csvToQRecordAdapter.buildRecordsFromCsv(getPersonCsvHeader() + getPersonCsvRow1(), TestUtils.defineTablePerson(), null);
|
||||
List<QRecord> qRecords = csvToQRecordAdapter.buildRecordsFromCsv(getPersonCsvHeader() + getPersonCsvRow1(), TestUtils.defineTablePerson(), null);
|
||||
assertNotNull(qRecords);
|
||||
assertEquals(1, qRecords.size());
|
||||
QRecord qRecord = qRecords.get(0);
|
||||
@ -161,7 +161,7 @@ class CsvToQRecordAdapterTest
|
||||
public void test_buildRecordsFromCsv_twoRowsStandardHeaderNoMapping()
|
||||
{
|
||||
CsvToQRecordAdapter csvToQRecordAdapter = new CsvToQRecordAdapter();
|
||||
List<QRecord> qRecords = csvToQRecordAdapter.buildRecordsFromCsv(getPersonCsvHeader() + getPersonCsvRow1() + getPersonCsvRow2(), TestUtils.defineTablePerson(), null);
|
||||
List<QRecord> qRecords = csvToQRecordAdapter.buildRecordsFromCsv(getPersonCsvHeader() + getPersonCsvRow1() + getPersonCsvRow2(), TestUtils.defineTablePerson(), null);
|
||||
assertNotNull(qRecords);
|
||||
assertEquals(2, qRecords.size());
|
||||
QRecord qRecord1 = qRecords.get(0);
|
||||
@ -194,7 +194,7 @@ class CsvToQRecordAdapterTest
|
||||
.withMapping("email", "email");
|
||||
|
||||
CsvToQRecordAdapter csvToQRecordAdapter = new CsvToQRecordAdapter();
|
||||
List<QRecord> qRecords = csvToQRecordAdapter.buildRecordsFromCsv(csvCustomHeader + getPersonCsvRow1(), TestUtils.defineTablePerson(), mapping);
|
||||
List<QRecord> qRecords = csvToQRecordAdapter.buildRecordsFromCsv(csvCustomHeader + getPersonCsvRow1(), TestUtils.defineTablePerson(), mapping);
|
||||
assertNotNull(qRecords);
|
||||
assertEquals(1, qRecords.size());
|
||||
QRecord qRecord = qRecords.get(0);
|
||||
@ -221,7 +221,7 @@ class CsvToQRecordAdapterTest
|
||||
.withMapping("email", index++);
|
||||
|
||||
CsvToQRecordAdapter csvToQRecordAdapter = new CsvToQRecordAdapter();
|
||||
List<QRecord> qRecords = csvToQRecordAdapter.buildRecordsFromCsv(getPersonCsvRow1() + getPersonCsvRow2(), TestUtils.defineTablePerson(), mapping);
|
||||
List<QRecord> qRecords = csvToQRecordAdapter.buildRecordsFromCsv(getPersonCsvRow1() + getPersonCsvRow2(), TestUtils.defineTablePerson(), mapping);
|
||||
assertNotNull(qRecords);
|
||||
assertEquals(2, qRecords.size());
|
||||
QRecord qRecord1 = qRecords.get(0);
|
||||
@ -267,6 +267,7 @@ class CsvToQRecordAdapterTest
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
@ -323,6 +324,8 @@ class CsvToQRecordAdapterTest
|
||||
assertNull(records.get(0).getValueString("lastName"));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
@ -336,11 +339,58 @@ class CsvToQRecordAdapterTest
|
||||
.withMapping("lastName", index++);
|
||||
|
||||
CsvToQRecordAdapter csvToQRecordAdapter = new CsvToQRecordAdapter();
|
||||
List<QRecord> records = csvToQRecordAdapter.buildRecordsFromCsv("1,John", TestUtils.defineTablePerson(), mapping);
|
||||
List<QRecord> records = csvToQRecordAdapter.buildRecordsFromCsv("1,John", TestUtils.defineTablePerson(), mapping);
|
||||
|
||||
assertEquals(1, records.get(0).getValueInteger("id"));
|
||||
assertEquals("John", records.get(0).getValueString("firstName"));
|
||||
assertNull(records.get(0).getValueString("lastName"));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testCaseSensitiveHeaders()
|
||||
{
|
||||
CsvToQRecordAdapter csvToQRecordAdapter = new CsvToQRecordAdapter();
|
||||
csvToQRecordAdapter.buildRecordsFromCsv(new CsvToQRecordAdapter.InputWrapper()
|
||||
.withTable(TestUtils.defineTablePerson())
|
||||
.withCaseSensitiveHeaders(true)
|
||||
.withCsv("""
|
||||
id,FirstName,lastName
|
||||
1,John,Doe
|
||||
"""));
|
||||
List<QRecord> records = csvToQRecordAdapter.getRecordList();
|
||||
|
||||
assertEquals(1, records.get(0).getValueInteger("id"));
|
||||
assertNull(records.get(0).getValueString("firstName"));
|
||||
assertEquals("Doe", records.get(0).getValueString("lastName"));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testCaseInsensitiveHeaders()
|
||||
{
|
||||
CsvToQRecordAdapter csvToQRecordAdapter = new CsvToQRecordAdapter();
|
||||
csvToQRecordAdapter.buildRecordsFromCsv(new CsvToQRecordAdapter.InputWrapper()
|
||||
.withTable(TestUtils.defineTablePerson())
|
||||
// this is default, so don't set it: withCaseSensitiveHeaders(false)
|
||||
.withCsv("""
|
||||
id,FirstName,lastName,EMAIL
|
||||
1,John,Doe,john@doe.com
|
||||
"""));
|
||||
List<QRecord> records = csvToQRecordAdapter.getRecordList();
|
||||
|
||||
assertEquals(1, records.get(0).getValueInteger("id"));
|
||||
assertEquals("John", records.get(0).getValueString("firstName"));
|
||||
assertEquals("Doe", records.get(0).getValueString("lastName"));
|
||||
assertEquals("john@doe.com", records.get(0).getValueString("email"));
|
||||
}
|
||||
|
||||
}
|
||||
|
@ -52,9 +52,11 @@ import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValue;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSource;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.queues.SQSQueueProviderMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.tables.QFieldSection;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.tables.Tier;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.tables.UniqueKey;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.TableAutomationAction;
|
||||
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.AbstractTransformStep;
|
||||
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.ExtractViaQueryStep;
|
||||
@ -133,7 +135,7 @@ class QInstanceValidatorTest
|
||||
**
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
public void test_validateNullTables()
|
||||
public void test_validateNullTablesAndProcesses()
|
||||
{
|
||||
assertValidationFailureReasons((qInstance) ->
|
||||
{
|
||||
@ -141,7 +143,8 @@ class QInstanceValidatorTest
|
||||
qInstance.setProcesses(null);
|
||||
},
|
||||
"At least 1 table must be defined",
|
||||
"Unrecognized table shape for possibleValueSource shape");
|
||||
"Unrecognized table shape for possibleValueSource shape",
|
||||
"Unrecognized processName for queue");
|
||||
}
|
||||
|
||||
|
||||
@ -151,7 +154,7 @@ class QInstanceValidatorTest
|
||||
**
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
public void test_validateEmptyTables()
|
||||
public void test_validateEmptyTablesAndProcesses()
|
||||
{
|
||||
assertValidationFailureReasons((qInstance) ->
|
||||
{
|
||||
@ -159,7 +162,8 @@ class QInstanceValidatorTest
|
||||
qInstance.setProcesses(new HashMap<>());
|
||||
},
|
||||
"At least 1 table must be defined",
|
||||
"Unrecognized table shape for possibleValueSource shape");
|
||||
"Unrecognized table shape for possibleValueSource shape",
|
||||
"Unrecognized processName for queue");
|
||||
}
|
||||
|
||||
|
||||
@ -1066,6 +1070,187 @@ class QInstanceValidatorTest
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testUniqueKeyNoFields()
|
||||
{
|
||||
assertValidationFailureReasons((qInstance) -> qInstance.getTable(TestUtils.TABLE_NAME_PERSON_MEMORY).withUniqueKey(new UniqueKey()),
|
||||
"uniqueKey with no fields");
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testUniqueKeyDuplicatedField()
|
||||
{
|
||||
assertValidationFailureReasons((qInstance) -> qInstance.getTable(TestUtils.TABLE_NAME_PERSON_MEMORY).withUniqueKey(new UniqueKey().withFieldName("id").withFieldName("id")),
|
||||
"the same field multiple times");
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testUniqueKeyInvalidField()
|
||||
{
|
||||
assertValidationFailureReasons((qInstance) -> qInstance.getTable(TestUtils.TABLE_NAME_PERSON_MEMORY).withUniqueKey(new UniqueKey().withFieldName("notAField")),
|
||||
"unrecognized field name: notAField");
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testUniqueKeyDuplicatedUniqueKeys()
|
||||
{
|
||||
assertValidationFailureReasons((qInstance) -> qInstance.getTable(TestUtils.TABLE_NAME_PERSON_MEMORY)
|
||||
.withUniqueKey(new UniqueKey().withFieldName("id"))
|
||||
.withUniqueKey(new UniqueKey().withFieldName("id")),
|
||||
"more than one uniqueKey with the same set of fields");
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testValidUniqueKeys()
|
||||
{
|
||||
assertValidationSuccess((qInstance) -> qInstance.getTable(TestUtils.TABLE_NAME_PERSON_MEMORY)
|
||||
.withUniqueKey(new UniqueKey().withFieldName("id"))
|
||||
.withUniqueKey(new UniqueKey().withFieldName("firstName").withFieldName("lastName")));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testQueueProviderName()
|
||||
{
|
||||
assertValidationFailureReasons((qInstance) -> qInstance.getQueueProvider(TestUtils.DEFAULT_QUEUE_PROVIDER).withName(null),
|
||||
"Inconsistent naming for queueProvider");
|
||||
|
||||
assertValidationFailureReasons((qInstance) -> qInstance.getQueueProvider(TestUtils.DEFAULT_QUEUE_PROVIDER).withName(""),
|
||||
"Inconsistent naming for queueProvider");
|
||||
|
||||
assertValidationFailureReasons((qInstance) -> qInstance.getQueueProvider(TestUtils.DEFAULT_QUEUE_PROVIDER).withName("wrongName"),
|
||||
"Inconsistent naming for queueProvider");
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testQueueProviderType()
|
||||
{
|
||||
assertValidationFailureReasons((qInstance) -> qInstance.getQueueProvider(TestUtils.DEFAULT_QUEUE_PROVIDER).withType(null),
|
||||
"Missing type for queueProvider");
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testQueueProviderSQSAttributes()
|
||||
{
|
||||
assertValidationFailureReasons((qInstance) ->
|
||||
{
|
||||
SQSQueueProviderMetaData queueProvider = (SQSQueueProviderMetaData) qInstance.getQueueProvider(TestUtils.DEFAULT_QUEUE_PROVIDER);
|
||||
queueProvider.setAccessKey(null);
|
||||
queueProvider.setSecretKey("");
|
||||
queueProvider.setRegion(null);
|
||||
queueProvider.setBaseURL("");
|
||||
},
|
||||
"Missing accessKey", "Missing secretKey", "Missing region", "Missing baseURL");
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testQueueName()
|
||||
{
|
||||
assertValidationFailureReasons((qInstance) -> qInstance.getQueue("testSQSQueue").withName(null),
|
||||
"Inconsistent naming for queue");
|
||||
|
||||
assertValidationFailureReasons((qInstance) -> qInstance.getQueue("testSQSQueue").withName(""),
|
||||
"Inconsistent naming for queue");
|
||||
|
||||
assertValidationFailureReasons((qInstance) -> qInstance.getQueue("testSQSQueue").withName("wrongName"),
|
||||
"Inconsistent naming for queue");
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testQueueQueueProviderName()
|
||||
{
|
||||
assertValidationFailureReasons((qInstance) -> qInstance.getQueue("testSQSQueue").withProviderName(null),
|
||||
"Unrecognized queue providerName");
|
||||
|
||||
assertValidationFailureReasons((qInstance) -> qInstance.getQueue("testSQSQueue").withProviderName(""),
|
||||
"Unrecognized queue providerName");
|
||||
|
||||
assertValidationFailureReasons((qInstance) -> qInstance.getQueue("testSQSQueue").withProviderName("wrongName"),
|
||||
"Unrecognized queue providerName");
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testQueueQueueName()
|
||||
{
|
||||
assertValidationFailureReasons((qInstance) -> qInstance.getQueue("testSQSQueue").withQueueName(null),
|
||||
"Missing queueName for queue");
|
||||
|
||||
assertValidationFailureReasons((qInstance) -> qInstance.getQueue("testSQSQueue").withQueueName(""),
|
||||
"Missing queueName for queue");
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testQueueProcessName()
|
||||
{
|
||||
assertValidationFailureReasons((qInstance) -> qInstance.getQueue("testSQSQueue").withProcessName(null),
|
||||
"Missing processName for queue");
|
||||
|
||||
assertValidationFailureReasons((qInstance) -> qInstance.getQueue("testSQSQueue").withProcessName(""),
|
||||
"Missing processName for queue");
|
||||
|
||||
assertValidationFailureReasons((qInstance) -> qInstance.getQueue("testSQSQueue").withProcessName("notAProcess"),
|
||||
"Unrecognized processName for queue:");
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
|
@ -0,0 +1,226 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2022. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* This program is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Affero General Public License as
|
||||
* published by the Free Software Foundation, either version 3 of the
|
||||
* License, or (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Affero General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert;
|
||||
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.stream.Collectors;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLine;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLineInterface;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.processes.Status;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.tables.UniqueKey;
|
||||
import com.kingsrook.qqq.backend.core.model.session.QSession;
|
||||
import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryRecordStore;
|
||||
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess;
|
||||
import com.kingsrook.qqq.backend.core.utils.TestUtils;
|
||||
import org.junit.jupiter.api.AfterEach;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Unit test for BulkInsertTransformStep
|
||||
*******************************************************************************/
|
||||
class BulkInsertTransformStepTest
|
||||
{
|
||||
public static final String TABLE_NAME = "ukTest";
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
@BeforeEach
|
||||
@AfterEach
|
||||
void beforeAndAfterEach()
|
||||
{
|
||||
MemoryRecordStore.getInstance().reset();
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testMultipleUniqueKeys() throws Exception
|
||||
{
|
||||
QInstance instance = TestUtils.defineInstance();
|
||||
QTableMetaData table = defineTable(new QTableMetaData()
|
||||
.withName(TABLE_NAME)
|
||||
.withBackendName(TestUtils.MEMORY_BACKEND_NAME)
|
||||
.withUniqueKey(new UniqueKey().withFieldName("uuid"))
|
||||
.withUniqueKey(new UniqueKey().withFieldName("sku").withFieldName("storeId")), instance);
|
||||
|
||||
////////////////////////////////////////////////////////////
|
||||
// insert some records that will cause some UK violations //
|
||||
////////////////////////////////////////////////////////////
|
||||
TestUtils.insertRecords(instance, table, List.of(
|
||||
newQRecord("uuid-A", "SKU-1", 1),
|
||||
newQRecord("uuid-B", "SKU-2", 1),
|
||||
newQRecord("uuid-C", "SKU-2", 2)
|
||||
));
|
||||
|
||||
///////////////////////////////////////////
|
||||
// setup & run the bulk insert transform //
|
||||
///////////////////////////////////////////
|
||||
BulkInsertTransformStep bulkInsertTransformStep = new BulkInsertTransformStep();
|
||||
RunBackendStepInput input = new RunBackendStepInput(instance);
|
||||
RunBackendStepOutput output = new RunBackendStepOutput();
|
||||
|
||||
input.setSession(new QSession());
|
||||
input.setTableName(TABLE_NAME);
|
||||
input.setStepName(StreamedETLWithFrontendProcess.STEP_NAME_VALIDATE);
|
||||
input.setRecords(List.of(
|
||||
newQRecord("uuid-1", "SKU-A", 1), // OK.
|
||||
newQRecord("uuid-1", "SKU-B", 1), // violate uuid UK in this set
|
||||
newQRecord("uuid-2", "SKU-C", 1), // OK.
|
||||
newQRecord("uuid-3", "SKU-C", 2), // OK.
|
||||
newQRecord("uuid-4", "SKU-C", 1), // violate sku/storeId UK in this set
|
||||
newQRecord("uuid-A", "SKU-X", 1), // violate uuid UK from pre-existing records
|
||||
newQRecord("uuid-D", "SKU-2", 1) // violate sku/storeId UK from pre-existing records
|
||||
));
|
||||
bulkInsertTransformStep.preRun(input, output);
|
||||
bulkInsertTransformStep.run(input, output);
|
||||
|
||||
///////////////////////////////////////////////////////
|
||||
// assert about the records that passed successfully //
|
||||
///////////////////////////////////////////////////////
|
||||
assertEquals(3, output.getRecords().size());
|
||||
assertThat(output.getRecords())
|
||||
.anyMatch(r -> recordEquals(r, "uuid-1", "SKU-A", 1))
|
||||
.anyMatch(r -> recordEquals(r, "uuid-2", "SKU-C", 1))
|
||||
.anyMatch(r -> recordEquals(r, "uuid-3", "SKU-C", 2));
|
||||
|
||||
/////////////////////////////
|
||||
// assert about the errors //
|
||||
/////////////////////////////
|
||||
ArrayList<ProcessSummaryLineInterface> processSummary = bulkInsertTransformStep.doGetProcessSummary(output, false);
|
||||
List<ProcessSummaryLine> errorLines = processSummary.stream()
|
||||
.filter(pl -> pl instanceof ProcessSummaryLine psl && psl.getStatus().equals(Status.ERROR))
|
||||
.map(pl -> ((ProcessSummaryLine) pl))
|
||||
.collect(Collectors.toList());
|
||||
assertEquals(2, errorLines.size());
|
||||
assertThat(errorLines)
|
||||
.anyMatch(psl -> psl.getMessage().contains("Uuid") && psl.getCount().equals(2))
|
||||
.anyMatch(psl -> psl.getMessage().contains("Sku and Store Id") && psl.getCount().equals(2));
|
||||
}
|
||||
|
||||
|
||||
|
||||
private QTableMetaData defineTable(QTableMetaData TABLE_NAME, QInstance instance)
|
||||
{
|
||||
QTableMetaData table = TABLE_NAME
|
||||
.withPrimaryKeyField("id")
|
||||
.withField(new QFieldMetaData("id", QFieldType.INTEGER))
|
||||
.withField(new QFieldMetaData("uuid", QFieldType.STRING))
|
||||
.withField(new QFieldMetaData("sku", QFieldType.STRING))
|
||||
.withField(new QFieldMetaData("storeId", QFieldType.INTEGER))
|
||||
.withField(new QFieldMetaData("name", QFieldType.STRING));
|
||||
instance.addTable(table);
|
||||
return table;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testNoUniqueKeys() throws Exception
|
||||
{
|
||||
QInstance instance = TestUtils.defineInstance();
|
||||
QTableMetaData table = defineTable(new QTableMetaData()
|
||||
.withName(TABLE_NAME)
|
||||
.withBackendName(TestUtils.MEMORY_BACKEND_NAME), instance);
|
||||
|
||||
////////////////////////////////////////////////////////////
|
||||
// insert some records that will cause some UK violations //
|
||||
////////////////////////////////////////////////////////////
|
||||
TestUtils.insertRecords(instance, table, List.of(
|
||||
newQRecord("uuid-A", "SKU-1", 1),
|
||||
newQRecord("uuid-B", "SKU-2", 1),
|
||||
newQRecord("uuid-C", "SKU-2", 2)
|
||||
));
|
||||
|
||||
///////////////////////////////////////////
|
||||
// setup & run the bulk insert transform //
|
||||
///////////////////////////////////////////
|
||||
BulkInsertTransformStep bulkInsertTransformStep = new BulkInsertTransformStep();
|
||||
RunBackendStepInput input = new RunBackendStepInput(instance);
|
||||
RunBackendStepOutput output = new RunBackendStepOutput();
|
||||
|
||||
input.setSession(new QSession());
|
||||
input.setTableName(TABLE_NAME);
|
||||
input.setStepName(StreamedETLWithFrontendProcess.STEP_NAME_VALIDATE);
|
||||
input.setRecords(List.of(
|
||||
newQRecord("uuid-1", "SKU-A", 1), // OK.
|
||||
newQRecord("uuid-1", "SKU-B", 1), // violate uuid UK in this set
|
||||
newQRecord("uuid-2", "SKU-C", 1), // OK.
|
||||
newQRecord("uuid-3", "SKU-C", 2), // OK.
|
||||
newQRecord("uuid-4", "SKU-C", 1), // violate sku/storeId UK in this set
|
||||
newQRecord("uuid-A", "SKU-X", 1), // violate uuid UK from pre-existing records
|
||||
newQRecord("uuid-D", "SKU-2", 1) // violate sku/storeId UK from pre-existing records
|
||||
));
|
||||
bulkInsertTransformStep.preRun(input, output);
|
||||
bulkInsertTransformStep.run(input, output);
|
||||
|
||||
///////////////////////////////////////////////////////
|
||||
// assert that all records pass.
|
||||
///////////////////////////////////////////////////////
|
||||
assertEquals(7, output.getRecords().size());
|
||||
}
|
||||
|
||||
|
||||
|
||||
private boolean recordEquals(QRecord record, String uuid, String sku, Integer storeId)
|
||||
{
|
||||
return (record.getValue("uuid").equals(uuid)
|
||||
&& record.getValue("sku").equals(sku)
|
||||
&& record.getValue("storeId").equals(storeId));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
private QRecord newQRecord(String uuid, String sku, int storeId)
|
||||
{
|
||||
return new QRecord()
|
||||
.withValue("uuid", uuid)
|
||||
.withValue("sku", sku)
|
||||
.withValue("storeId", storeId)
|
||||
.withValue("name", "Some Item");
|
||||
}
|
||||
|
||||
}
|
@ -859,10 +859,10 @@ public class TestUtils
|
||||
{
|
||||
QMetaDataVariableInterpreter interpreter = new QMetaDataVariableInterpreter();
|
||||
|
||||
String accessKey = interpreter.interpret("${env.SQS_ACCESS_KEY}");
|
||||
String secretKey = interpreter.interpret("${env.SQS_SECRET_KEY}");
|
||||
String region = interpreter.interpret("${env.SQS_REGION}");
|
||||
String baseURL = interpreter.interpret("${env.SQS_BASE_URL}");
|
||||
String accessKey = "MOCK"; // interpreter.interpret("${env.SQS_ACCESS_KEY}");
|
||||
String secretKey = "MOCK"; // interpreter.interpret("${env.SQS_SECRET_KEY}");
|
||||
String region = "MOCK"; // interpreter.interpret("${env.SQS_REGION}");
|
||||
String baseURL = "MOCK"; // interpreter.interpret("${env.SQS_BASE_URL}");
|
||||
|
||||
return (new SQSQueueProviderMetaData()
|
||||
.withName(DEFAULT_QUEUE_PROVIDER)
|
||||
@ -883,7 +883,7 @@ public class TestUtils
|
||||
.withName("testSQSQueue")
|
||||
.withProviderName(DEFAULT_QUEUE_PROVIDER)
|
||||
.withQueueName("test-queue")
|
||||
.withProcessName("receiveEasypostTrackerWebhook"));
|
||||
.withProcessName(PROCESS_NAME_INCREASE_BIRTHDATE));
|
||||
}
|
||||
|
||||
}
|
||||
|
Reference in New Issue
Block a user