From fa2ab18e30ce5fd7e6884aa82e69e6850f6c6cdf Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Fri, 21 Oct 2022 11:21:07 -0500 Subject: [PATCH] Add unique keys, and checking of them in bulk load; add some more validation (sqs and unique keys) --- .../core/adapters/CsvToQRecordAdapter.java | 68 ++++- .../core/instances/QInstanceValidator.java | 76 ++++++ .../actions/tables/query/QQueryFilter.java | 15 ++ .../model/metadata/tables/QTableMetaData.java | 51 ++++ .../core/model/metadata/tables/UniqueKey.java | 120 +++++++++ .../bulk/insert/BulkInsertTransformStep.java | 242 ++++++++++++++++-- .../adapters/CsvToQRecordAdapterTest.java | 64 ++++- .../instances/QInstanceValidatorTest.java | 193 +++++++++++++- .../insert/BulkInsertTransformStepTest.java | 226 ++++++++++++++++ .../qqq/backend/core/utils/TestUtils.java | 10 +- 10 files changed, 1015 insertions(+), 50 deletions(-) create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/UniqueKey.java create mode 100644 qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTransformStepTest.java diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/adapters/CsvToQRecordAdapter.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/adapters/CsvToQRecordAdapter.java index 77b1e85d..48dffd16 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/adapters/CsvToQRecordAdapter.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/adapters/CsvToQRecordAdapter.java @@ -89,11 +89,11 @@ public class CsvToQRecordAdapter *******************************************************************************/ public void buildRecordsFromCsv(InputWrapper inputWrapper) { - String csv = inputWrapper.getCsv(); - AbstractQFieldMapping mapping = inputWrapper.getMapping(); - Consumer recordCustomizer = inputWrapper.getRecordCustomizer(); - QTableMetaData table = inputWrapper.getTable(); - Integer limit = inputWrapper.getLimit(); + String csv = inputWrapper.getCsv(); + AbstractQFieldMapping mapping = inputWrapper.getMapping(); + Consumer recordCustomizer = inputWrapper.getRecordCustomizer(); + QTableMetaData table = inputWrapper.getTable(); + Integer limit = inputWrapper.getLimit(); if(!StringUtils.hasContent(csv)) { @@ -136,7 +136,7 @@ public class CsvToQRecordAdapter headers = makeHeadersUnique(headers); Iterator csvIterator = csvParser.iterator(); - int recordCount = 0; + int recordCount = 0; while(csvIterator.hasNext()) { CSVRecord csvRecord = csvIterator.next(); @@ -147,7 +147,8 @@ public class CsvToQRecordAdapter Map csvValues = new HashMap<>(); for(int i = 0; i < headers.size() && i < csvRecord.size(); i++) { - csvValues.put(headers.get(i), csvRecord.get(i)); + String header = adjustHeaderCase(headers.get(i), inputWrapper); + csvValues.put(header, csvRecord.get(i)); } ////////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -157,6 +158,7 @@ public class CsvToQRecordAdapter for(QFieldMetaData field : table.getFields().values()) { String fieldSource = mapping == null ? field.getName() : String.valueOf(mapping.getFieldSource(field.getName())); + fieldSource = adjustHeaderCase(fieldSource, inputWrapper); qRecord.setValue(field.getName(), csvValues.get(fieldSource)); } @@ -180,7 +182,7 @@ public class CsvToQRecordAdapter .withTrim()); Iterator csvIterator = csvParser.iterator(); - int recordCount = 0; + int recordCount = 0; while(csvIterator.hasNext()) { CSVRecord csvRecord = csvIterator.next(); @@ -228,6 +230,20 @@ public class CsvToQRecordAdapter + /******************************************************************************* + ** + *******************************************************************************/ + private String adjustHeaderCase(String s, InputWrapper inputWrapper) + { + if(inputWrapper.caseSensitiveHeaders) + { + return (s); + } + return (s.toLowerCase()); + } + + + /******************************************************************************* ** *******************************************************************************/ @@ -325,6 +341,8 @@ public class CsvToQRecordAdapter private Consumer recordCustomizer; private Integer limit; + private boolean caseSensitiveHeaders = false; + /******************************************************************************* @@ -529,6 +547,40 @@ public class CsvToQRecordAdapter return (this); } + + + /******************************************************************************* + ** Getter for caseSensitiveHeaders + ** + *******************************************************************************/ + public boolean getCaseSensitiveHeaders() + { + return caseSensitiveHeaders; + } + + + + /******************************************************************************* + ** Setter for caseSensitiveHeaders + ** + *******************************************************************************/ + public void setCaseSensitiveHeaders(boolean caseSensitiveHeaders) + { + this.caseSensitiveHeaders = caseSensitiveHeaders; + } + + + + /******************************************************************************* + ** Fluent setter for caseSensitiveHeaders + ** + *******************************************************************************/ + public InputWrapper withCaseSensitiveHeaders(boolean caseSensitiveHeaders) + { + this.caseSensitiveHeaders = caseSensitiveHeaders; + return (this); + } + } } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidator.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidator.java index 70074957..6f344f10 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidator.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidator.java @@ -49,9 +49,11 @@ import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppSection; import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData; import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; import com.kingsrook.qqq.backend.core.model.metadata.processes.QStepMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.queues.SQSQueueProviderMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.QFieldSection; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.Tier; +import com.kingsrook.qqq.backend.core.model.metadata.tables.UniqueKey; import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.AutomationStatusTracking; import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.AutomationStatusTrackingType; import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.QTableAutomationDetails; @@ -118,6 +120,7 @@ public class QInstanceValidator validateProcesses(qInstance); validateApps(qInstance); validatePossibleValueSources(qInstance); + validateQueuesAndProviders(qInstance); } catch(Exception e) { @@ -134,6 +137,45 @@ public class QInstanceValidator + /******************************************************************************* + ** + *******************************************************************************/ + private void validateQueuesAndProviders(QInstance qInstance) + { + if(CollectionUtils.nullSafeHasContents(qInstance.getQueueProviders())) + { + qInstance.getQueueProviders().forEach((name, queueProvider) -> + { + assertCondition(Objects.equals(name, queueProvider.getName()), "Inconsistent naming for queueProvider: " + name + "/" + queueProvider.getName() + "."); + assertCondition(queueProvider.getType() != null, "Missing type for queueProvider: " + name); + + if(queueProvider instanceof SQSQueueProviderMetaData sqsQueueProvider) + { + assertCondition(StringUtils.hasContent(sqsQueueProvider.getAccessKey()), "Missing accessKey for SQSQueueProvider: " + name); + assertCondition(StringUtils.hasContent(sqsQueueProvider.getSecretKey()), "Missing secretKey for SQSQueueProvider: " + name); + assertCondition(StringUtils.hasContent(sqsQueueProvider.getBaseURL()), "Missing baseURL for SQSQueueProvider: " + name); + assertCondition(StringUtils.hasContent(sqsQueueProvider.getRegion()), "Missing region for SQSQueueProvider: " + name); + } + }); + } + + if(CollectionUtils.nullSafeHasContents(qInstance.getQueues())) + { + qInstance.getQueues().forEach((name, queue) -> + { + assertCondition(Objects.equals(name, queue.getName()), "Inconsistent naming for queue: " + name + "/" + queue.getName() + "."); + assertCondition(qInstance.getQueueProvider(queue.getProviderName()) != null, "Unrecognized queue providerName for queue: " + name); + assertCondition(StringUtils.hasContent(queue.getQueueName()), "Missing queueName for queue: " + name); + if(assertCondition(StringUtils.hasContent(queue.getProcessName()), "Missing processName for queue: " + name)) + { + assertCondition(qInstance.getProcesses() != null && qInstance.getProcess(queue.getProcessName()) != null, "Unrecognized processName for queue: " + name); + } + }); + } + } + + + /******************************************************************************* ** *******************************************************************************/ @@ -263,12 +305,46 @@ public class QInstanceValidator { validateTableAutomationDetails(qInstance, table); } + + ////////////////////////////////////// + // validate the table's unique keys // + ////////////////////////////////////// + if(table.getUniqueKeys() != null) + { + validateTableUniqueKeys(table); + } }); } } + /******************************************************************************* + ** + *******************************************************************************/ + private void validateTableUniqueKeys(QTableMetaData table) + { + Set> ukSets = new HashSet<>(); + for(UniqueKey uniqueKey : table.getUniqueKeys()) + { + if(assertCondition(CollectionUtils.nullSafeHasContents(uniqueKey.getFieldNames()), table.getName() + " has a uniqueKey with no fields")) + { + Set fieldNamesInThisUK = new HashSet<>(); + for(String fieldName : uniqueKey.getFieldNames()) + { + assertNoException(() -> table.getField(fieldName), table.getName() + " has a uniqueKey with an unrecognized field name: " + fieldName); + assertCondition(!fieldNamesInThisUK.contains(fieldName), table.getName() + " has a uniqueKey with the same field multiple times: " + fieldName); + fieldNamesInThisUK.add(fieldName); + } + + assertCondition(!ukSets.contains(fieldNamesInThisUK), table.getName() + " has more than one uniqueKey with the same set of fields: " + fieldNamesInThisUK); + ukSets.add(fieldNamesInThisUK); + } + } + } + + + /******************************************************************************* ** *******************************************************************************/ diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QQueryFilter.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QQueryFilter.java index ac6a99ce..f39403de 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QQueryFilter.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QQueryFilter.java @@ -286,4 +286,19 @@ public class QQueryFilter implements Serializable, Cloneable return (this); } + + + /******************************************************************************* + ** + *******************************************************************************/ + public void addSubFilter(QQueryFilter subFilter) + { + if(this.subFilters == null) + { + subFilters = new ArrayList<>(); + } + + subFilters.add(subFilter); + } + } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/QTableMetaData.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/QTableMetaData.java index bfbf1ed6..886fc5bc 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/QTableMetaData.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/QTableMetaData.java @@ -63,6 +63,7 @@ public class QTableMetaData implements QAppChildMetaData, Serializable private boolean isHidden = false; private Map fields; + private List uniqueKeys; private QTableBackendDetails backendDetails; private QTableAutomationDetails automationDetails; @@ -751,4 +752,54 @@ public class QTableMetaData implements QAppChildMetaData, Serializable return (this); } + + + /******************************************************************************* + ** Getter for uniqueKeys + ** + *******************************************************************************/ + public List getUniqueKeys() + { + return uniqueKeys; + } + + + + /******************************************************************************* + ** Setter for uniqueKeys + ** + *******************************************************************************/ + public void setUniqueKeys(List uniqueKeys) + { + this.uniqueKeys = uniqueKeys; + } + + + + /******************************************************************************* + ** Fluent setter for uniqueKeys + ** + *******************************************************************************/ + public QTableMetaData withUniqueKeys(List uniqueKeys) + { + this.uniqueKeys = uniqueKeys; + return (this); + } + + + + /******************************************************************************* + ** Fluent setter for uniqueKeys + ** + *******************************************************************************/ + public QTableMetaData withUniqueKey(UniqueKey uniqueKey) + { + if(this.uniqueKeys == null) + { + this.uniqueKeys = new ArrayList<>(); + } + this.uniqueKeys.add(uniqueKey); + return (this); + } + } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/UniqueKey.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/UniqueKey.java new file mode 100644 index 00000000..698e7417 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/UniqueKey.java @@ -0,0 +1,120 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.metadata.tables; + + +import java.util.ArrayList; +import java.util.List; + + +/******************************************************************************* + ** Definition of a Unique Key (or "Constraint", if you wanna use fancy words) + ** on a QTable. + *******************************************************************************/ +public class UniqueKey +{ + private List fieldNames; + private String label; + + + + /******************************************************************************* + ** Getter for fieldNames + ** + *******************************************************************************/ + public List getFieldNames() + { + return fieldNames; + } + + + + /******************************************************************************* + ** Setter for fieldNames + ** + *******************************************************************************/ + public void setFieldNames(List fieldNames) + { + this.fieldNames = fieldNames; + } + + + + /******************************************************************************* + ** Fluent setter for fieldNames + ** + *******************************************************************************/ + public UniqueKey withFieldNames(List fieldNames) + { + this.fieldNames = fieldNames; + return (this); + } + + + + /******************************************************************************* + ** Getter for label + ** + *******************************************************************************/ + public String getLabel() + { + return label; + } + + + + /******************************************************************************* + ** Setter for label + ** + *******************************************************************************/ + public void setLabel(String label) + { + this.label = label; + } + + + + /******************************************************************************* + ** Fluent setter for label + ** + *******************************************************************************/ + public UniqueKey withLabel(String label) + { + this.label = label; + return (this); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public UniqueKey withFieldName(String fieldName) + { + if(this.fieldNames == null) + { + this.fieldNames = new ArrayList<>(); + } + this.fieldNames.add(fieldName); + return (this); + } +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTransformStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTransformStep.java index 073d4ade..ba7912d5 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTransformStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTransformStep.java @@ -22,16 +22,33 @@ package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert; +import java.io.Serializable; import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; +import com.kingsrook.qqq.backend.core.actions.tables.QueryAction; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLine; import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLineInterface; import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; import com.kingsrook.qqq.backend.core.model.actions.processes.Status; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.tables.UniqueKey; import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.AbstractTransformStep; import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess; +import com.kingsrook.qqq.backend.core.utils.CollectionUtils; +import com.kingsrook.qqq.backend.core.utils.StringUtils; /******************************************************************************* @@ -39,9 +56,12 @@ import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwith *******************************************************************************/ public class BulkInsertTransformStep extends AbstractTransformStep { - private ProcessSummaryLine okSummary = new ProcessSummaryLine(Status.OK); + private ProcessSummaryLine okSummary = new ProcessSummaryLine(Status.OK); + private Map ukErrorSummaries = new HashMap<>(); - private String tableLabel; + private QTableMetaData table; + + private Map>> keysInThisFile = new HashMap<>(); @@ -51,14 +71,7 @@ public class BulkInsertTransformStep extends AbstractTransformStep @Override public void preRun(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException { - /////////////////////////////////////////////////////// - // capture the table label - for the process summary // - /////////////////////////////////////////////////////// - QTableMetaData table = runBackendStepInput.getInstance().getTable(runBackendStepInput.getTableName()); - if(table != null) - { - tableLabel = table.getLabel(); - } + this.table = runBackendStepInput.getInstance().getTable(runBackendStepInput.getTableName()); } @@ -69,6 +82,16 @@ public class BulkInsertTransformStep extends AbstractTransformStep @Override public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException { + QTableMetaData table = runBackendStepInput.getInstance().getTable(runBackendStepInput.getTableName()); + + Map>> existingKeys = new HashMap<>(); + List uniqueKeys = CollectionUtils.nonNullList(table.getUniqueKeys()); + for(UniqueKey uniqueKey : uniqueKeys) + { + existingKeys.put(uniqueKey, getExistingKeys(runBackendStepInput, uniqueKey)); + ukErrorSummaries.computeIfAbsent(uniqueKey, x -> new ProcessSummaryLine(Status.ERROR)); + } + ///////////////////////////////////////////////////////////////////////////////////////////////////////////////// // on the validate step, we haven't read the full file, so we don't know how many rows there are - thus // // record count is null, and the ValidateStep won't be setting status counters - so - do it here in that case. // @@ -81,19 +104,153 @@ public class BulkInsertTransformStep extends AbstractTransformStep { if(runBackendStepInput.getValue(StreamedETLWithFrontendProcess.FIELD_RECORD_COUNT) == null) { - runBackendStepInput.getAsyncJobCallback().updateStatus("Inserting " + tableLabel + " record " + "%,d".formatted(okSummary.getCount())); + runBackendStepInput.getAsyncJobCallback().updateStatus("Inserting " + table.getLabel() + " record " + "%,d".formatted(okSummary.getCount())); } else { - runBackendStepInput.getAsyncJobCallback().updateStatus("Inserting " + tableLabel + " records"); + runBackendStepInput.getAsyncJobCallback().updateStatus("Inserting " + table.getLabel() + " records"); } } - ////////////////////////////////////////////////////////////////////////////////////////////////////////// - // no transformation needs done - just pass records through from input to output, and assume all are OK // - ////////////////////////////////////////////////////////////////////////////////////////////////////////// - runBackendStepOutput.setRecords(runBackendStepInput.getRecords()); - okSummary.incrementCount(runBackendStepInput.getRecords().size()); + ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // no transformation needs to be done - just pass records through from input to output, if they don't violate any UK's // + ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + + /////////////////////////////////////////////////// + // if there are no UK's, just output all records // + /////////////////////////////////////////////////// + if(existingKeys.isEmpty()) + { + runBackendStepOutput.setRecords(runBackendStepInput.getRecords()); + okSummary.incrementCount(runBackendStepInput.getRecords().size()); + } + else + { + for(UniqueKey uniqueKey : uniqueKeys) + { + keysInThisFile.computeIfAbsent(uniqueKey, x -> new HashSet<>()); + } + + /////////////////////////////////////////////////////////////////////////// + // else, get each records keys and see if it already exists or not // + // also, build a set of keys we've seen (within this page (or overall?)) // + /////////////////////////////////////////////////////////////////////////// + for(QRecord record : runBackendStepInput.getRecords()) + { + ////////////////////////////////////////////////////////// + // check if this record violates any of the unique keys // + ////////////////////////////////////////////////////////// + boolean foundDupe = false; + for(UniqueKey uniqueKey : uniqueKeys) + { + List keyValues = getKeyValues(uniqueKey, record); + if(existingKeys.get(uniqueKey).contains(keyValues) || keysInThisFile.get(uniqueKey).contains(keyValues)) + { + ukErrorSummaries.get(uniqueKey).incrementCount(); + foundDupe = true; + break; + } + } + + /////////////////////////////////////////////////////////////////////////////// + // if this record doesn't violate any uk's, then we can add it to the output // + /////////////////////////////////////////////////////////////////////////////// + if(!foundDupe) + { + for(UniqueKey uniqueKey : uniqueKeys) + { + List keyValues = getKeyValues(uniqueKey, record); + keysInThisFile.get(uniqueKey).add(keyValues); + } + okSummary.incrementCount(); + runBackendStepOutput.addRecord(record); + } + } + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private Set> getExistingKeys(RunBackendStepInput runBackendStepInput, UniqueKey uniqueKey) throws QException + { + return (getExistingKeys(runBackendStepInput, uniqueKey.getFieldNames())); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private Set> getExistingKeys(RunBackendStepInput runBackendStepInput, List ukFieldNames) throws QException + { + Set> existingRecords = new HashSet<>(); + if(ukFieldNames != null) + { + QueryInput queryInput = new QueryInput(runBackendStepInput.getInstance()); + queryInput.setSession(runBackendStepInput.getSession()); + queryInput.setTableName(runBackendStepInput.getTableName()); + getTransaction().ifPresent(queryInput::setTransaction); + + QQueryFilter filter = new QQueryFilter(); + if(ukFieldNames.size() == 1) + { + List values = runBackendStepInput.getRecords().stream() + .map(r -> r.getValue(ukFieldNames.get(0))) + .collect(Collectors.toList()); + filter.addCriteria(new QFilterCriteria(ukFieldNames.get(0), QCriteriaOperator.IN, values)); + } + else + { + filter.setBooleanOperator(QQueryFilter.BooleanOperator.OR); + for(QRecord record : runBackendStepInput.getRecords()) + { + QQueryFilter subFilter = new QQueryFilter(); + filter.addSubFilter(subFilter); + for(String fieldName : ukFieldNames) + { + subFilter.addCriteria(new QFilterCriteria(fieldName, QCriteriaOperator.EQUALS, List.of(record.getValue(fieldName)))); + } + } + } + + queryInput.setFilter(filter); + QueryOutput queryOutput = new QueryAction().execute(queryInput); + for(QRecord record : queryOutput.getRecords()) + { + List keyValues = getKeyValues(ukFieldNames, record); + existingRecords.add(keyValues); + } + } + + return (existingRecords); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private List getKeyValues(UniqueKey uniqueKey, QRecord record) + { + return (getKeyValues(uniqueKey.getFieldNames(), record)); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private List getKeyValues(List fieldNames, QRecord record) + { + List keyValues = new ArrayList<>(); + for(String fieldName : fieldNames) + { + keyValues.add(record.getValue(fieldName)); + } + return keyValues; } @@ -104,17 +261,50 @@ public class BulkInsertTransformStep extends AbstractTransformStep @Override public ArrayList getProcessSummary(RunBackendStepOutput runBackendStepOutput, boolean isForResultScreen) { - if(isForResultScreen) - { - okSummary.setMessage(tableLabel + " records were inserted."); - } - else - { - okSummary.setMessage(tableLabel + " records will be inserted."); - } + String tableLabel = table == null ? "" : table.getLabel(); + + okSummary + .withSingularFutureMessage(tableLabel + " record will be inserted") + .withPluralFutureMessage(tableLabel + " records will be inserted") + .withSingularPastMessage(tableLabel + " record was inserted") + .withPluralPastMessage(tableLabel + " records were inserted"); ArrayList rs = new ArrayList<>(); - rs.add(okSummary); + okSummary.addSelfToListIfAnyCount(rs); + + for(Map.Entry entry : ukErrorSummaries.entrySet()) + { + UniqueKey uniqueKey = entry.getKey(); + ProcessSummaryLine ukErrorSummary = entry.getValue(); + String ukErrorSuffix = " inserted, because they contain a duplicate key (" + getUkDescription(uniqueKey.getFieldNames()) + ")"; + + ukErrorSummary + .withSingularFutureMessage(tableLabel + " record will not be" + ukErrorSuffix) + .withPluralFutureMessage(tableLabel + " records will not be" + ukErrorSuffix) + .withSingularPastMessage(tableLabel + " record was not" + ukErrorSuffix) + .withPluralPastMessage(tableLabel + " records were not" + ukErrorSuffix); + + ukErrorSummary.addSelfToListIfAnyCount(rs); + } + return (rs); } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private String getUkDescription(List ukFieldNames) + { + List fieldLabels = new ArrayList<>(); + + for(String fieldName : ukFieldNames) + { + fieldLabels.add(table.getField(fieldName).getLabel()); + } + + return (StringUtils.joinWithCommasAndAnd(fieldLabels)); + } + } diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/adapters/CsvToQRecordAdapterTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/adapters/CsvToQRecordAdapterTest.java index 7dff6bad..87b4bbb4 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/adapters/CsvToQRecordAdapterTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/adapters/CsvToQRecordAdapterTest.java @@ -73,7 +73,7 @@ class CsvToQRecordAdapterTest try { CsvToQRecordAdapter csvToQRecordAdapter = new CsvToQRecordAdapter(); - List qRecords = csvToQRecordAdapter.buildRecordsFromCsv(csv, TestUtils.defineTablePerson(), null); + List qRecords = csvToQRecordAdapter.buildRecordsFromCsv(csv, TestUtils.defineTablePerson(), null); System.out.println(qRecords); } catch(IllegalArgumentException iae) @@ -94,7 +94,7 @@ class CsvToQRecordAdapterTest public void test_buildRecordsFromCsv_emptyList() { CsvToQRecordAdapter csvToQRecordAdapter = new CsvToQRecordAdapter(); - List qRecords = csvToQRecordAdapter.buildRecordsFromCsv(getPersonCsvHeader(), TestUtils.defineTablePerson(), null); + List qRecords = csvToQRecordAdapter.buildRecordsFromCsv(getPersonCsvHeader(), TestUtils.defineTablePerson(), null); assertNotNull(qRecords); assertTrue(qRecords.isEmpty()); } @@ -144,7 +144,7 @@ class CsvToQRecordAdapterTest public void test_buildRecordsFromCsv_oneRowStandardHeaderNoMapping() { CsvToQRecordAdapter csvToQRecordAdapter = new CsvToQRecordAdapter(); - List qRecords = csvToQRecordAdapter.buildRecordsFromCsv(getPersonCsvHeader() + getPersonCsvRow1(), TestUtils.defineTablePerson(), null); + List qRecords = csvToQRecordAdapter.buildRecordsFromCsv(getPersonCsvHeader() + getPersonCsvRow1(), TestUtils.defineTablePerson(), null); assertNotNull(qRecords); assertEquals(1, qRecords.size()); QRecord qRecord = qRecords.get(0); @@ -161,7 +161,7 @@ class CsvToQRecordAdapterTest public void test_buildRecordsFromCsv_twoRowsStandardHeaderNoMapping() { CsvToQRecordAdapter csvToQRecordAdapter = new CsvToQRecordAdapter(); - List qRecords = csvToQRecordAdapter.buildRecordsFromCsv(getPersonCsvHeader() + getPersonCsvRow1() + getPersonCsvRow2(), TestUtils.defineTablePerson(), null); + List qRecords = csvToQRecordAdapter.buildRecordsFromCsv(getPersonCsvHeader() + getPersonCsvRow1() + getPersonCsvRow2(), TestUtils.defineTablePerson(), null); assertNotNull(qRecords); assertEquals(2, qRecords.size()); QRecord qRecord1 = qRecords.get(0); @@ -194,7 +194,7 @@ class CsvToQRecordAdapterTest .withMapping("email", "email"); CsvToQRecordAdapter csvToQRecordAdapter = new CsvToQRecordAdapter(); - List qRecords = csvToQRecordAdapter.buildRecordsFromCsv(csvCustomHeader + getPersonCsvRow1(), TestUtils.defineTablePerson(), mapping); + List qRecords = csvToQRecordAdapter.buildRecordsFromCsv(csvCustomHeader + getPersonCsvRow1(), TestUtils.defineTablePerson(), mapping); assertNotNull(qRecords); assertEquals(1, qRecords.size()); QRecord qRecord = qRecords.get(0); @@ -221,7 +221,7 @@ class CsvToQRecordAdapterTest .withMapping("email", index++); CsvToQRecordAdapter csvToQRecordAdapter = new CsvToQRecordAdapter(); - List qRecords = csvToQRecordAdapter.buildRecordsFromCsv(getPersonCsvRow1() + getPersonCsvRow2(), TestUtils.defineTablePerson(), mapping); + List qRecords = csvToQRecordAdapter.buildRecordsFromCsv(getPersonCsvRow1() + getPersonCsvRow2(), TestUtils.defineTablePerson(), mapping); assertNotNull(qRecords); assertEquals(2, qRecords.size()); QRecord qRecord1 = qRecords.get(0); @@ -267,6 +267,7 @@ class CsvToQRecordAdapterTest } + /******************************************************************************* ** *******************************************************************************/ @@ -323,6 +324,8 @@ class CsvToQRecordAdapterTest assertNull(records.get(0).getValueString("lastName")); } + + /******************************************************************************* ** *******************************************************************************/ @@ -336,11 +339,58 @@ class CsvToQRecordAdapterTest .withMapping("lastName", index++); CsvToQRecordAdapter csvToQRecordAdapter = new CsvToQRecordAdapter(); - List records = csvToQRecordAdapter.buildRecordsFromCsv("1,John", TestUtils.defineTablePerson(), mapping); + List records = csvToQRecordAdapter.buildRecordsFromCsv("1,John", TestUtils.defineTablePerson(), mapping); assertEquals(1, records.get(0).getValueInteger("id")); assertEquals("John", records.get(0).getValueString("firstName")); assertNull(records.get(0).getValueString("lastName")); } + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testCaseSensitiveHeaders() + { + CsvToQRecordAdapter csvToQRecordAdapter = new CsvToQRecordAdapter(); + csvToQRecordAdapter.buildRecordsFromCsv(new CsvToQRecordAdapter.InputWrapper() + .withTable(TestUtils.defineTablePerson()) + .withCaseSensitiveHeaders(true) + .withCsv(""" + id,FirstName,lastName + 1,John,Doe + """)); + List records = csvToQRecordAdapter.getRecordList(); + + assertEquals(1, records.get(0).getValueInteger("id")); + assertNull(records.get(0).getValueString("firstName")); + assertEquals("Doe", records.get(0).getValueString("lastName")); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testCaseInsensitiveHeaders() + { + CsvToQRecordAdapter csvToQRecordAdapter = new CsvToQRecordAdapter(); + csvToQRecordAdapter.buildRecordsFromCsv(new CsvToQRecordAdapter.InputWrapper() + .withTable(TestUtils.defineTablePerson()) + // this is default, so don't set it: withCaseSensitiveHeaders(false) + .withCsv(""" + id,FirstName,lastName,EMAIL + 1,John,Doe,john@doe.com + """)); + List records = csvToQRecordAdapter.getRecordList(); + + assertEquals(1, records.get(0).getValueInteger("id")); + assertEquals("John", records.get(0).getValueString("firstName")); + assertEquals("Doe", records.get(0).getValueString("lastName")); + assertEquals("john@doe.com", records.get(0).getValueString("email")); + } + } diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidatorTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidatorTest.java index 87a79fa7..d6da4b05 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidatorTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidatorTest.java @@ -52,9 +52,11 @@ import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon; import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValue; import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSource; import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.queues.SQSQueueProviderMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.QFieldSection; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.Tier; +import com.kingsrook.qqq.backend.core.model.metadata.tables.UniqueKey; import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.TableAutomationAction; import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.AbstractTransformStep; import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.ExtractViaQueryStep; @@ -133,7 +135,7 @@ class QInstanceValidatorTest ** *******************************************************************************/ @Test - public void test_validateNullTables() + public void test_validateNullTablesAndProcesses() { assertValidationFailureReasons((qInstance) -> { @@ -141,7 +143,8 @@ class QInstanceValidatorTest qInstance.setProcesses(null); }, "At least 1 table must be defined", - "Unrecognized table shape for possibleValueSource shape"); + "Unrecognized table shape for possibleValueSource shape", + "Unrecognized processName for queue"); } @@ -151,7 +154,7 @@ class QInstanceValidatorTest ** *******************************************************************************/ @Test - public void test_validateEmptyTables() + public void test_validateEmptyTablesAndProcesses() { assertValidationFailureReasons((qInstance) -> { @@ -159,7 +162,8 @@ class QInstanceValidatorTest qInstance.setProcesses(new HashMap<>()); }, "At least 1 table must be defined", - "Unrecognized table shape for possibleValueSource shape"); + "Unrecognized table shape for possibleValueSource shape", + "Unrecognized processName for queue"); } @@ -1066,6 +1070,187 @@ class QInstanceValidatorTest + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testUniqueKeyNoFields() + { + assertValidationFailureReasons((qInstance) -> qInstance.getTable(TestUtils.TABLE_NAME_PERSON_MEMORY).withUniqueKey(new UniqueKey()), + "uniqueKey with no fields"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testUniqueKeyDuplicatedField() + { + assertValidationFailureReasons((qInstance) -> qInstance.getTable(TestUtils.TABLE_NAME_PERSON_MEMORY).withUniqueKey(new UniqueKey().withFieldName("id").withFieldName("id")), + "the same field multiple times"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testUniqueKeyInvalidField() + { + assertValidationFailureReasons((qInstance) -> qInstance.getTable(TestUtils.TABLE_NAME_PERSON_MEMORY).withUniqueKey(new UniqueKey().withFieldName("notAField")), + "unrecognized field name: notAField"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testUniqueKeyDuplicatedUniqueKeys() + { + assertValidationFailureReasons((qInstance) -> qInstance.getTable(TestUtils.TABLE_NAME_PERSON_MEMORY) + .withUniqueKey(new UniqueKey().withFieldName("id")) + .withUniqueKey(new UniqueKey().withFieldName("id")), + "more than one uniqueKey with the same set of fields"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testValidUniqueKeys() + { + assertValidationSuccess((qInstance) -> qInstance.getTable(TestUtils.TABLE_NAME_PERSON_MEMORY) + .withUniqueKey(new UniqueKey().withFieldName("id")) + .withUniqueKey(new UniqueKey().withFieldName("firstName").withFieldName("lastName"))); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testQueueProviderName() + { + assertValidationFailureReasons((qInstance) -> qInstance.getQueueProvider(TestUtils.DEFAULT_QUEUE_PROVIDER).withName(null), + "Inconsistent naming for queueProvider"); + + assertValidationFailureReasons((qInstance) -> qInstance.getQueueProvider(TestUtils.DEFAULT_QUEUE_PROVIDER).withName(""), + "Inconsistent naming for queueProvider"); + + assertValidationFailureReasons((qInstance) -> qInstance.getQueueProvider(TestUtils.DEFAULT_QUEUE_PROVIDER).withName("wrongName"), + "Inconsistent naming for queueProvider"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testQueueProviderType() + { + assertValidationFailureReasons((qInstance) -> qInstance.getQueueProvider(TestUtils.DEFAULT_QUEUE_PROVIDER).withType(null), + "Missing type for queueProvider"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testQueueProviderSQSAttributes() + { + assertValidationFailureReasons((qInstance) -> + { + SQSQueueProviderMetaData queueProvider = (SQSQueueProviderMetaData) qInstance.getQueueProvider(TestUtils.DEFAULT_QUEUE_PROVIDER); + queueProvider.setAccessKey(null); + queueProvider.setSecretKey(""); + queueProvider.setRegion(null); + queueProvider.setBaseURL(""); + }, + "Missing accessKey", "Missing secretKey", "Missing region", "Missing baseURL"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testQueueName() + { + assertValidationFailureReasons((qInstance) -> qInstance.getQueue("testSQSQueue").withName(null), + "Inconsistent naming for queue"); + + assertValidationFailureReasons((qInstance) -> qInstance.getQueue("testSQSQueue").withName(""), + "Inconsistent naming for queue"); + + assertValidationFailureReasons((qInstance) -> qInstance.getQueue("testSQSQueue").withName("wrongName"), + "Inconsistent naming for queue"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testQueueQueueProviderName() + { + assertValidationFailureReasons((qInstance) -> qInstance.getQueue("testSQSQueue").withProviderName(null), + "Unrecognized queue providerName"); + + assertValidationFailureReasons((qInstance) -> qInstance.getQueue("testSQSQueue").withProviderName(""), + "Unrecognized queue providerName"); + + assertValidationFailureReasons((qInstance) -> qInstance.getQueue("testSQSQueue").withProviderName("wrongName"), + "Unrecognized queue providerName"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testQueueQueueName() + { + assertValidationFailureReasons((qInstance) -> qInstance.getQueue("testSQSQueue").withQueueName(null), + "Missing queueName for queue"); + + assertValidationFailureReasons((qInstance) -> qInstance.getQueue("testSQSQueue").withQueueName(""), + "Missing queueName for queue"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testQueueProcessName() + { + assertValidationFailureReasons((qInstance) -> qInstance.getQueue("testSQSQueue").withProcessName(null), + "Missing processName for queue"); + + assertValidationFailureReasons((qInstance) -> qInstance.getQueue("testSQSQueue").withProcessName(""), + "Missing processName for queue"); + + assertValidationFailureReasons((qInstance) -> qInstance.getQueue("testSQSQueue").withProcessName("notAProcess"), + "Unrecognized processName for queue:"); + } + + + /******************************************************************************* ** *******************************************************************************/ diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTransformStepTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTransformStepTest.java new file mode 100644 index 00000000..480e4795 --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTransformStepTest.java @@ -0,0 +1,226 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert; + + +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; +import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLine; +import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLineInterface; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; +import com.kingsrook.qqq.backend.core.model.actions.processes.Status; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.tables.UniqueKey; +import com.kingsrook.qqq.backend.core.model.session.QSession; +import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryRecordStore; +import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess; +import com.kingsrook.qqq.backend.core.utils.TestUtils; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertEquals; + + +/******************************************************************************* + ** Unit test for BulkInsertTransformStep + *******************************************************************************/ +class BulkInsertTransformStepTest +{ + public static final String TABLE_NAME = "ukTest"; + + + + /******************************************************************************* + ** + *******************************************************************************/ + @BeforeEach + @AfterEach + void beforeAndAfterEach() + { + MemoryRecordStore.getInstance().reset(); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testMultipleUniqueKeys() throws Exception + { + QInstance instance = TestUtils.defineInstance(); + QTableMetaData table = defineTable(new QTableMetaData() + .withName(TABLE_NAME) + .withBackendName(TestUtils.MEMORY_BACKEND_NAME) + .withUniqueKey(new UniqueKey().withFieldName("uuid")) + .withUniqueKey(new UniqueKey().withFieldName("sku").withFieldName("storeId")), instance); + + //////////////////////////////////////////////////////////// + // insert some records that will cause some UK violations // + //////////////////////////////////////////////////////////// + TestUtils.insertRecords(instance, table, List.of( + newQRecord("uuid-A", "SKU-1", 1), + newQRecord("uuid-B", "SKU-2", 1), + newQRecord("uuid-C", "SKU-2", 2) + )); + + /////////////////////////////////////////// + // setup & run the bulk insert transform // + /////////////////////////////////////////// + BulkInsertTransformStep bulkInsertTransformStep = new BulkInsertTransformStep(); + RunBackendStepInput input = new RunBackendStepInput(instance); + RunBackendStepOutput output = new RunBackendStepOutput(); + + input.setSession(new QSession()); + input.setTableName(TABLE_NAME); + input.setStepName(StreamedETLWithFrontendProcess.STEP_NAME_VALIDATE); + input.setRecords(List.of( + newQRecord("uuid-1", "SKU-A", 1), // OK. + newQRecord("uuid-1", "SKU-B", 1), // violate uuid UK in this set + newQRecord("uuid-2", "SKU-C", 1), // OK. + newQRecord("uuid-3", "SKU-C", 2), // OK. + newQRecord("uuid-4", "SKU-C", 1), // violate sku/storeId UK in this set + newQRecord("uuid-A", "SKU-X", 1), // violate uuid UK from pre-existing records + newQRecord("uuid-D", "SKU-2", 1) // violate sku/storeId UK from pre-existing records + )); + bulkInsertTransformStep.preRun(input, output); + bulkInsertTransformStep.run(input, output); + + /////////////////////////////////////////////////////// + // assert about the records that passed successfully // + /////////////////////////////////////////////////////// + assertEquals(3, output.getRecords().size()); + assertThat(output.getRecords()) + .anyMatch(r -> recordEquals(r, "uuid-1", "SKU-A", 1)) + .anyMatch(r -> recordEquals(r, "uuid-2", "SKU-C", 1)) + .anyMatch(r -> recordEquals(r, "uuid-3", "SKU-C", 2)); + + ///////////////////////////// + // assert about the errors // + ///////////////////////////// + ArrayList processSummary = bulkInsertTransformStep.doGetProcessSummary(output, false); + List errorLines = processSummary.stream() + .filter(pl -> pl instanceof ProcessSummaryLine psl && psl.getStatus().equals(Status.ERROR)) + .map(pl -> ((ProcessSummaryLine) pl)) + .collect(Collectors.toList()); + assertEquals(2, errorLines.size()); + assertThat(errorLines) + .anyMatch(psl -> psl.getMessage().contains("Uuid") && psl.getCount().equals(2)) + .anyMatch(psl -> psl.getMessage().contains("Sku and Store Id") && psl.getCount().equals(2)); + } + + + + private QTableMetaData defineTable(QTableMetaData TABLE_NAME, QInstance instance) + { + QTableMetaData table = TABLE_NAME + .withPrimaryKeyField("id") + .withField(new QFieldMetaData("id", QFieldType.INTEGER)) + .withField(new QFieldMetaData("uuid", QFieldType.STRING)) + .withField(new QFieldMetaData("sku", QFieldType.STRING)) + .withField(new QFieldMetaData("storeId", QFieldType.INTEGER)) + .withField(new QFieldMetaData("name", QFieldType.STRING)); + instance.addTable(table); + return table; + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testNoUniqueKeys() throws Exception + { + QInstance instance = TestUtils.defineInstance(); + QTableMetaData table = defineTable(new QTableMetaData() + .withName(TABLE_NAME) + .withBackendName(TestUtils.MEMORY_BACKEND_NAME), instance); + + //////////////////////////////////////////////////////////// + // insert some records that will cause some UK violations // + //////////////////////////////////////////////////////////// + TestUtils.insertRecords(instance, table, List.of( + newQRecord("uuid-A", "SKU-1", 1), + newQRecord("uuid-B", "SKU-2", 1), + newQRecord("uuid-C", "SKU-2", 2) + )); + + /////////////////////////////////////////// + // setup & run the bulk insert transform // + /////////////////////////////////////////// + BulkInsertTransformStep bulkInsertTransformStep = new BulkInsertTransformStep(); + RunBackendStepInput input = new RunBackendStepInput(instance); + RunBackendStepOutput output = new RunBackendStepOutput(); + + input.setSession(new QSession()); + input.setTableName(TABLE_NAME); + input.setStepName(StreamedETLWithFrontendProcess.STEP_NAME_VALIDATE); + input.setRecords(List.of( + newQRecord("uuid-1", "SKU-A", 1), // OK. + newQRecord("uuid-1", "SKU-B", 1), // violate uuid UK in this set + newQRecord("uuid-2", "SKU-C", 1), // OK. + newQRecord("uuid-3", "SKU-C", 2), // OK. + newQRecord("uuid-4", "SKU-C", 1), // violate sku/storeId UK in this set + newQRecord("uuid-A", "SKU-X", 1), // violate uuid UK from pre-existing records + newQRecord("uuid-D", "SKU-2", 1) // violate sku/storeId UK from pre-existing records + )); + bulkInsertTransformStep.preRun(input, output); + bulkInsertTransformStep.run(input, output); + + /////////////////////////////////////////////////////// + // assert that all records pass. + /////////////////////////////////////////////////////// + assertEquals(7, output.getRecords().size()); + } + + + + private boolean recordEquals(QRecord record, String uuid, String sku, Integer storeId) + { + return (record.getValue("uuid").equals(uuid) + && record.getValue("sku").equals(sku) + && record.getValue("storeId").equals(storeId)); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private QRecord newQRecord(String uuid, String sku, int storeId) + { + return new QRecord() + .withValue("uuid", uuid) + .withValue("sku", sku) + .withValue("storeId", storeId) + .withValue("name", "Some Item"); + } + +} \ No newline at end of file diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java index d9cbd9d2..618da13e 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java @@ -859,10 +859,10 @@ public class TestUtils { QMetaDataVariableInterpreter interpreter = new QMetaDataVariableInterpreter(); - String accessKey = interpreter.interpret("${env.SQS_ACCESS_KEY}"); - String secretKey = interpreter.interpret("${env.SQS_SECRET_KEY}"); - String region = interpreter.interpret("${env.SQS_REGION}"); - String baseURL = interpreter.interpret("${env.SQS_BASE_URL}"); + String accessKey = "MOCK"; // interpreter.interpret("${env.SQS_ACCESS_KEY}"); + String secretKey = "MOCK"; // interpreter.interpret("${env.SQS_SECRET_KEY}"); + String region = "MOCK"; // interpreter.interpret("${env.SQS_REGION}"); + String baseURL = "MOCK"; // interpreter.interpret("${env.SQS_BASE_URL}"); return (new SQSQueueProviderMetaData() .withName(DEFAULT_QUEUE_PROVIDER) @@ -883,7 +883,7 @@ public class TestUtils .withName("testSQSQueue") .withProviderName(DEFAULT_QUEUE_PROVIDER) .withQueueName("test-queue") - .withProcessName("receiveEasypostTrackerWebhook")); + .withProcessName(PROCESS_NAME_INCREASE_BIRTHDATE)); } }