diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTransformStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTransformStep.java index 36b29c04..9344fdf1 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTransformStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTransformStep.java @@ -30,6 +30,10 @@ import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Set; +import com.kingsrook.qqq.backend.core.actions.customizers.AbstractPreInsertCustomizer; +import com.kingsrook.qqq.backend.core.actions.customizers.AbstractPreInsertCustomizer.WhenToRun; +import com.kingsrook.qqq.backend.core.actions.customizers.QCodeLoader; +import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizers; import com.kingsrook.qqq.backend.core.actions.tables.InsertAction; import com.kingsrook.qqq.backend.core.actions.tables.helpers.UniqueKeyHelper; import com.kingsrook.qqq.backend.core.exceptions.QException; @@ -48,6 +52,7 @@ import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwith import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess; import com.kingsrook.qqq.backend.core.processes.implementations.general.ProcessSummaryWarningsAndErrorsRollup; import com.kingsrook.qqq.backend.core.utils.CollectionUtils; +import com.kingsrook.qqq.backend.core.utils.StringUtils; /******************************************************************************* @@ -59,12 +64,34 @@ public class BulkInsertTransformStep extends AbstractTransformStep private ProcessSummaryWarningsAndErrorsRollup processSummaryWarningsAndErrorsRollup = ProcessSummaryWarningsAndErrorsRollup.build("inserted"); - private Map ukErrorSummaries = new HashMap<>(); + private Map ukErrorSummaries = new HashMap<>(); private QTableMetaData table; private Map>> keysInThisFile = new HashMap<>(); + private int rowsProcessed = 0; + + + + /******************************************************************************* + ** + *******************************************************************************/ + private static class ProcessSummaryLineWithUKSampleValues extends ProcessSummaryLine + { + private List sampleValues = new ArrayList<>(); + + + + /******************************************************************************* + ** + *******************************************************************************/ + public ProcessSummaryLineWithUKSampleValues(Status status) + { + super(status); + } + } + /******************************************************************************* @@ -89,14 +116,48 @@ public class BulkInsertTransformStep extends AbstractTransformStep @Override public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException { + int rowsInThisPage = runBackendStepInput.getRecords().size(); QTableMetaData table = runBackendStepInput.getInstance().getTable(runBackendStepInput.getTableName()); + ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // set up an insert-input, which will be used as input to the pre-customizer as well as for additional validations // + ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + InsertInput insertInput = new InsertInput(); + insertInput.setInputSource(QInputSource.USER); + insertInput.setTableName(runBackendStepInput.getTableName()); + insertInput.setRecords(runBackendStepInput.getRecords()); + insertInput.setSkipUniqueKeyCheck(true); + + ////////////////////////////////////////////////////////////////////// + // load the pre-insert customizer and set it up, if there is one // + // then we'll run it based on its WhenToRun value // + // we do this, in case it needs to, for example, adjust values that // + // are part of a unique key // + ////////////////////////////////////////////////////////////////////// + Optional preInsertCustomizer = QCodeLoader.getTableCustomizer(AbstractPreInsertCustomizer.class, table, TableCustomizers.PRE_INSERT_RECORD.getRole()); + if(preInsertCustomizer.isPresent()) + { + preInsertCustomizer.get().setInsertInput(insertInput); + preInsertCustomizer.get().setIsPreview(true); + AbstractPreInsertCustomizer.WhenToRun whenToRun = preInsertCustomizer.get().getWhenToRun(); + if(WhenToRun.BEFORE_ALL_VALIDATIONS.equals(whenToRun) || WhenToRun.BEFORE_UNIQUE_KEY_CHECKS.equals(whenToRun)) + { + List recordsAfterCustomizer = preInsertCustomizer.get().apply(runBackendStepInput.getRecords()); + runBackendStepInput.setRecords(recordsAfterCustomizer); + + /////////////////////////////////////////////////////////////////////////////////////// + // todo - do we care if the customizer runs both now, and in the validation below? // + // right now we'll let it run both times, but maybe that should be protected against // + /////////////////////////////////////////////////////////////////////////////////////// + } + } + Map>> existingKeys = new HashMap<>(); List uniqueKeys = CollectionUtils.nonNullList(table.getUniqueKeys()); for(UniqueKey uniqueKey : uniqueKeys) { existingKeys.put(uniqueKey, UniqueKeyHelper.getExistingKeys(null, table, runBackendStepInput.getRecords(), uniqueKey).keySet()); - ukErrorSummaries.computeIfAbsent(uniqueKey, x -> new ProcessSummaryLine(Status.ERROR)); + ukErrorSummaries.computeIfAbsent(uniqueKey, x -> new ProcessSummaryLineWithUKSampleValues(Status.ERROR)); } ///////////////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -105,7 +166,7 @@ public class BulkInsertTransformStep extends AbstractTransformStep ///////////////////////////////////////////////////////////////////////////////////////////////////////////////// if(runBackendStepInput.getStepName().equals(StreamedETLWithFrontendProcess.STEP_NAME_VALIDATE)) { - runBackendStepInput.getAsyncJobCallback().updateStatus("Processing row " + "%,d".formatted(okSummary.getCount())); + runBackendStepInput.getAsyncJobCallback().updateStatus("Processing row " + "%,d".formatted(rowsProcessed + 1)); } else if(runBackendStepInput.getStepName().equals(StreamedETLWithFrontendProcess.STEP_NAME_EXECUTE)) { @@ -123,70 +184,13 @@ public class BulkInsertTransformStep extends AbstractTransformStep // Note, we want to do our own UK checking here, even though InsertAction also tries to do it, because InsertAction // // will only be getting the records in pages, but in here, we'll track UK's across pages!! // ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - - //////////////////////////////////////////////////// - // if there are no UK's, proceed with all records // - //////////////////////////////////////////////////// - List recordsWithoutUkErrors = new ArrayList<>(); - if(existingKeys.isEmpty()) - { - recordsWithoutUkErrors.addAll(runBackendStepInput.getRecords()); - } - else - { - ///////////////////////////////////////////////////////////// - // else, only proceed with records that don't violate a UK // - ///////////////////////////////////////////////////////////// - for(UniqueKey uniqueKey : uniqueKeys) - { - keysInThisFile.computeIfAbsent(uniqueKey, x -> new HashSet<>()); - } - - /////////////////////////////////////////////////////////////////////////// - // else, get each records keys and see if it already exists or not // - // also, build a set of keys we've seen (within this page (or overall?)) // - /////////////////////////////////////////////////////////////////////////// - for(QRecord record : runBackendStepInput.getRecords()) - { - ////////////////////////////////////////////////////////// - // check if this record violates any of the unique keys // - ////////////////////////////////////////////////////////// - boolean foundDupe = false; - for(UniqueKey uniqueKey : uniqueKeys) - { - Optional> keyValues = UniqueKeyHelper.getKeyValues(table, uniqueKey, record); - if(keyValues.isPresent() && (existingKeys.get(uniqueKey).contains(keyValues.get()) || keysInThisFile.get(uniqueKey).contains(keyValues.get()))) - { - ukErrorSummaries.get(uniqueKey).incrementCount(); - foundDupe = true; - break; - } - } - - /////////////////////////////////////////////////////////////////////////////// - // if this record doesn't violate any uk's, then we can add it to the output // - /////////////////////////////////////////////////////////////////////////////// - if(!foundDupe) - { - for(UniqueKey uniqueKey : uniqueKeys) - { - Optional> keyValues = UniqueKeyHelper.getKeyValues(table, uniqueKey, record); - keyValues.ifPresent(kv -> keysInThisFile.get(uniqueKey).add(kv)); - } - recordsWithoutUkErrors.add(record); - } - } - } + List recordsWithoutUkErrors = getRecordsWithoutUniqueKeyErrors(runBackendStepInput, existingKeys, uniqueKeys, table); ///////////////////////////////////////////////////////////////////////////////// // run all validation from the insert action - in Preview mode (boolean param) // ///////////////////////////////////////////////////////////////////////////////// - InsertAction insertAction = new InsertAction(); - InsertInput insertInput = new InsertInput(); - insertInput.setInputSource(QInputSource.USER); - insertInput.setTableName(runBackendStepInput.getTableName()); insertInput.setRecords(recordsWithoutUkErrors); - insertInput.setSkipUniqueKeyCheck(true); + InsertAction insertAction = new InsertAction(); insertAction.performValidations(insertInput, true); List validationResultRecords = insertInput.getRecords(); @@ -215,6 +219,85 @@ public class BulkInsertTransformStep extends AbstractTransformStep } runBackendStepOutput.setRecords(outputRecords); + + this.rowsProcessed += rowsInThisPage; + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private List getRecordsWithoutUniqueKeyErrors(RunBackendStepInput runBackendStepInput, Map>> existingKeys, List uniqueKeys, QTableMetaData table) + { + //////////////////////////////////////////////////// + // if there are no UK's, proceed with all records // + //////////////////////////////////////////////////// + List recordsWithoutUkErrors = new ArrayList<>(); + if(existingKeys.isEmpty()) + { + recordsWithoutUkErrors.addAll(runBackendStepInput.getRecords()); + } + else + { + ///////////////////////////////////////////////////////////// + // else, only proceed with records that don't violate a UK // + ///////////////////////////////////////////////////////////// + for(UniqueKey uniqueKey : uniqueKeys) + { + keysInThisFile.computeIfAbsent(uniqueKey, x -> new HashSet<>()); + } + + /////////////////////////////////////////////////////////////////////////// + // else, get each records keys and see if it already exists or not // + // also, build a set of keys we've seen (within this page (or overall?)) // + /////////////////////////////////////////////////////////////////////////// + for(QRecord record : runBackendStepInput.getRecords()) + { + if(CollectionUtils.nullSafeHasContents(record.getErrors())) + { + /////////////////////////////////////////////////// + // skip any records that may already be in error // + /////////////////////////////////////////////////// + recordsWithoutUkErrors.add(record); + continue; + } + + ////////////////////////////////////////////////////////// + // check if this record violates any of the unique keys // + ////////////////////////////////////////////////////////// + boolean foundDupe = false; + for(UniqueKey uniqueKey : uniqueKeys) + { + Optional> keyValues = UniqueKeyHelper.getKeyValues(table, uniqueKey, record); + if(keyValues.isPresent() && (existingKeys.get(uniqueKey).contains(keyValues.get()) || keysInThisFile.get(uniqueKey).contains(keyValues.get()))) + { + ProcessSummaryLineWithUKSampleValues processSummaryLineWithUKSampleValues = ukErrorSummaries.get(uniqueKey); + processSummaryLineWithUKSampleValues.incrementCount(); + if(processSummaryLineWithUKSampleValues.sampleValues.size() < 3) + { + processSummaryLineWithUKSampleValues.sampleValues.add(keyValues.get().toString()); + } + foundDupe = true; + break; + } + } + + /////////////////////////////////////////////////////////////////////////////// + // if this record doesn't violate any uk's, then we can add it to the output // + /////////////////////////////////////////////////////////////////////////////// + if(!foundDupe) + { + for(UniqueKey uniqueKey : uniqueKeys) + { + Optional> keyValues = UniqueKeyHelper.getKeyValues(table, uniqueKey, record); + keyValues.ifPresent(kv -> keysInThisFile.get(uniqueKey).add(kv)); + } + recordsWithoutUkErrors.add(record); + } + } + } + return recordsWithoutUkErrors; } @@ -236,17 +319,19 @@ public class BulkInsertTransformStep extends AbstractTransformStep okSummary.pickMessage(isForResultScreen); okSummary.addSelfToListIfAnyCount(rs); - for(Map.Entry entry : ukErrorSummaries.entrySet()) + for(Map.Entry entry : ukErrorSummaries.entrySet()) { - UniqueKey uniqueKey = entry.getKey(); - ProcessSummaryLine ukErrorSummary = entry.getValue(); - String ukErrorSuffix = " inserted, because of duplicate values in a unique key (" + uniqueKey.getDescription(table) + ")"; + UniqueKey uniqueKey = entry.getKey(); + ProcessSummaryLineWithUKSampleValues ukErrorSummary = entry.getValue(); ukErrorSummary - .withSingularFutureMessage(tableLabel + " record will not be" + ukErrorSuffix) - .withPluralFutureMessage(tableLabel + " records will not be" + ukErrorSuffix) - .withSingularPastMessage(tableLabel + " record was not" + ukErrorSuffix) - .withPluralPastMessage(tableLabel + " records were not" + ukErrorSuffix); + .withMessageSuffix(" inserted, because of duplicate values in a unique key on the fields (" + uniqueKey.getDescription(table) + "), with values such as: " + + StringUtils.joinWithCommasAndAnd(ukErrorSummary.sampleValues)) + + .withSingularFutureMessage(tableLabel + " record will not be") + .withPluralFutureMessage(tableLabel + " records will not be") + .withSingularPastMessage(tableLabel + " record was not") + .withPluralPastMessage(tableLabel + " records were not"); ukErrorSummary.addSelfToListIfAnyCount(rs); }