From c88fd5b7d41389c293a45aef749f125dae974ab2 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Wed, 27 Nov 2024 12:35:54 -0600 Subject: [PATCH] CE-1955 - Summarize with some examples (including rows nos) for value mapping and other validation errors --- .../model/metadata/fields/QFieldType.java | 11 + .../bulk/insert/BulkInsertTransformStep.java | 169 +++++++++++++- .../mapping/BulkLoadValueTypeError.java | 76 +++++++ .../bulk/insert/mapping/ValueMapper.java | 6 +- ...ProcessSummaryWarningsAndErrorsRollup.java | 9 + .../processes/ProcessSummaryAssert.java | 208 ++++++++++++++++++ .../ProcessSummaryLineInterfaceAssert.java | 189 ++++++++++++++++ .../insert/BulkInsertTransformStepTest.java | 190 +++++++++++++--- .../qqq/backend/core/utils/TestUtils.java | 3 +- 9 files changed, 823 insertions(+), 38 deletions(-) create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadValueTypeError.java create mode 100644 qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/actions/processes/ProcessSummaryAssert.java create mode 100644 qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/actions/processes/ProcessSummaryLineInterfaceAssert.java diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/QFieldType.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/QFieldType.java index b0e8f8e7..a8313116 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/QFieldType.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/QFieldType.java @@ -27,6 +27,7 @@ import java.time.Instant; import java.time.LocalDate; import java.time.LocalTime; import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.utils.StringUtils; /******************************************************************************* @@ -100,6 +101,16 @@ public enum QFieldType + /*************************************************************************** + ** + ***************************************************************************/ + public String getMixedCaseLabel() + { + return StringUtils.allCapsToMixedCase(name()); + } + + + /******************************************************************************* ** *******************************************************************************/ diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTransformStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTransformStep.java index 09d9ca8d..f2d87767 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTransformStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTransformStep.java @@ -29,6 +29,7 @@ import java.util.HashSet; import java.util.LinkedHashSet; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Optional; import java.util.Set; import com.kingsrook.qqq.backend.core.actions.customizers.AbstractPreInsertCustomizer; @@ -47,16 +48,25 @@ import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutp import com.kingsrook.qqq.backend.core.model.actions.processes.Status; import com.kingsrook.qqq.backend.core.model.actions.tables.QInputSource; import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; +import com.kingsrook.qqq.backend.core.model.dashboard.widgets.WidgetType; import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.dashboard.QWidgetMetaDataInterface; import com.kingsrook.qqq.backend.core.model.metadata.tables.Association; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QFieldSection; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.UniqueKey; +import com.kingsrook.qqq.backend.core.model.statusmessages.QErrorMessage; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping.BulkLoadRecordUtils; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping.BulkLoadValueTypeError; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkInsertMapping; import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.AbstractTransformStep; import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.LoadViaInsertStep; import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess; import com.kingsrook.qqq.backend.core.processes.implementations.general.ProcessSummaryWarningsAndErrorsRollup; import com.kingsrook.qqq.backend.core.utils.CollectionUtils; +import com.kingsrook.qqq.backend.core.utils.ListingHash; import com.kingsrook.qqq.backend.core.utils.StringUtils; +import com.kingsrook.qqq.backend.core.utils.ValueUtils; /******************************************************************************* @@ -66,7 +76,11 @@ public class BulkInsertTransformStep extends AbstractTransformStep { private ProcessSummaryLine okSummary = new ProcessSummaryLine(Status.OK); - private ProcessSummaryWarningsAndErrorsRollup processSummaryWarningsAndErrorsRollup = ProcessSummaryWarningsAndErrorsRollup.build("inserted"); + private ProcessSummaryWarningsAndErrorsRollup processSummaryWarningsAndErrorsRollup = ProcessSummaryWarningsAndErrorsRollup.build("inserted") + .withDoReplaceSingletonCountLinesWithSuffixOnly(false); + + private ListingHash errorToExampleRowValueMap = new ListingHash<>(); + private ListingHash errorToExampleRowsMap = new ListingHash<>(); private Map ukErrorSummaries = new HashMap<>(); private Map associationsToInsertSummaries = new HashMap<>(); @@ -77,6 +91,7 @@ public class BulkInsertTransformStep extends AbstractTransformStep private int rowsProcessed = 0; + private final int EXAMPLE_ROW_LIMIT = 10; /******************************************************************************* @@ -118,6 +133,44 @@ public class BulkInsertTransformStep extends AbstractTransformStep // make sure that if a saved profile was selected on a review screen, that the result screen knows about it. // /////////////////////////////////////////////////////////////////////////////////////////////////////////////// BulkInsertStepUtils.handleSavedBulkLoadProfileIdValue(runBackendStepInput, runBackendStepOutput); + + ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // set up the validationReview widget to render preview records using the table layout, and including the associations // + ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + runBackendStepOutput.addValue("formatPreviewRecordUsingTableLayout", table.getName()); + + BulkInsertMapping bulkInsertMapping = (BulkInsertMapping) runBackendStepOutput.getValue("bulkInsertMapping"); + if(bulkInsertMapping != null) + { + ArrayList previewRecordAssociatedTableNames = new ArrayList<>(); + ArrayList previewRecordAssociatedWidgetNames = new ArrayList<>(); + ArrayList previewRecordAssociationNames = new ArrayList<>(); + + for(String mappedAssociation : bulkInsertMapping.getMappedAssociations()) + { + Optional association = table.getAssociations().stream().filter(a -> a.getName().equals(mappedAssociation)).findFirst(); + if(association.isPresent()) + { + for(QFieldSection section : table.getSections()) + { + QWidgetMetaDataInterface widget = QContext.getQInstance().getWidget(section.getWidgetName()); + if(widget != null && WidgetType.CHILD_RECORD_LIST.getType().equals(widget.getType())) + { + Serializable widgetJoinName = widget.getDefaultValues().get("joinName"); + if(Objects.equals(widgetJoinName, association.get().getJoinName())) + { + previewRecordAssociatedTableNames.add(association.get().getAssociatedTableName()); + previewRecordAssociatedWidgetNames.add(widget.getName()); + previewRecordAssociationNames.add(association.get().getName()); + } + } + } + } + } + runBackendStepOutput.addValue("previewRecordAssociatedTableNames", previewRecordAssociatedTableNames); + runBackendStepOutput.addValue("previewRecordAssociatedWidgetNames", previewRecordAssociatedWidgetNames); + runBackendStepOutput.addValue("previewRecordAssociationNames", previewRecordAssociationNames); + } } @@ -131,7 +184,9 @@ public class BulkInsertTransformStep extends AbstractTransformStep int recordsInThisPage = runBackendStepInput.getRecords().size(); QTableMetaData table = QContext.getQInstance().getTable(runBackendStepInput.getTableName()); - // split the records w/o UK errors into those w/ e + ////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // split the records into 2 lists: those w/ errors (e.g., from the bulk-load mapping), and those that are okay // + ////////////////////////////////////////////////////////////////////////////////////////////////////////////////// List recordsWithoutAnyErrors = new ArrayList<>(); List recordsWithSomeErrors = new ArrayList<>(); for(QRecord record : runBackendStepInput.getRecords()) @@ -153,16 +208,26 @@ public class BulkInsertTransformStep extends AbstractTransformStep { for(QRecord record : recordsWithSomeErrors) { - String message = record.getErrors().get(0).getMessage(); - processSummaryWarningsAndErrorsRollup.addError(message, null); + for(QErrorMessage error : record.getErrors()) + { + if(error instanceof BulkLoadValueTypeError blvte) + { + processSummaryWarningsAndErrorsRollup.addError(blvte.getMessageToUseAsProcessSummaryRollupKey(), null); + addToErrorToExampleRowValueMap(blvte, record); + } + else + { + processSummaryWarningsAndErrorsRollup.addError(error.getMessage(), null); + } + } } } if(recordsWithoutAnyErrors.isEmpty()) { - //////////////////////////////////////////////////////////////////////////////// - // skip th rest of this method if there aren't any records w/o errors in them // - //////////////////////////////////////////////////////////////////////////////// + ///////////////////////////////////////////////////////////////////////////////// + // skip the rest of this method if there aren't any records w/o errors in them // + ///////////////////////////////////////////////////////////////////////////////// this.rowsProcessed += recordsInThisPage; } @@ -248,8 +313,11 @@ public class BulkInsertTransformStep extends AbstractTransformStep { if(CollectionUtils.nullSafeHasContents(record.getErrors())) { - String message = record.getErrors().get(0).getMessage(); - processSummaryWarningsAndErrorsRollup.addError(message, null); + for(QErrorMessage error : record.getErrors()) + { + processSummaryWarningsAndErrorsRollup.addError(error.getMessage(), null); + addToErrorToExampleRowMap(error.getMessage(), record); + } } else if(CollectionUtils.nullSafeHasContents(record.getWarnings())) { @@ -277,6 +345,37 @@ public class BulkInsertTransformStep extends AbstractTransformStep + /*************************************************************************** + ** + ***************************************************************************/ + private void addToErrorToExampleRowValueMap(BulkLoadValueTypeError bulkLoadValueTypeError, QRecord record) + { + String message = bulkLoadValueTypeError.getMessageToUseAsProcessSummaryRollupKey(); + List rowValues = errorToExampleRowValueMap.computeIfAbsent(message, k -> new ArrayList<>()); + + if(rowValues.size() < EXAMPLE_ROW_LIMIT) + { + rowValues.add(new RowValue(bulkLoadValueTypeError, record)); + } + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private void addToErrorToExampleRowMap(String message, QRecord record) + { + List rowNos = errorToExampleRowsMap.computeIfAbsent(message, k -> new ArrayList<>()); + + if(rowNos.size() < EXAMPLE_ROW_LIMIT) + { + rowNos.add(BulkLoadRecordUtils.getRowNosString(record)); + } + } + + + /******************************************************************************* ** *******************************************************************************/ @@ -411,9 +510,61 @@ public class BulkInsertTransformStep extends AbstractTransformStep ukErrorSummary.addSelfToListIfAnyCount(rs); } + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // for process summary lines that exist in the error-to-example-row-value map, add those example values to the lines. // + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + for(Map.Entry entry : processSummaryWarningsAndErrorsRollup.getErrorSummaries().entrySet()) + { + String message = entry.getKey(); + if(errorToExampleRowValueMap.containsKey(message)) + { + ProcessSummaryLine line = entry.getValue(); + List rowValues = errorToExampleRowValueMap.get(message); + String exampleOrFull = rowValues.size() < line.getCount() ? "Example " : ""; + line.setMessageSuffix(line.getMessageSuffix() + ". " + exampleOrFull + "Values:"); + line.setBulletsOfText(new ArrayList<>(rowValues.stream().map(String::valueOf).toList())); + } + else if(errorToExampleRowsMap.containsKey(message)) + { + ProcessSummaryLine line = entry.getValue(); + List rowDescriptions = errorToExampleRowsMap.get(message); + String exampleOrFull = rowDescriptions.size() < line.getCount() ? "Example " : ""; + line.setMessageSuffix(line.getMessageSuffix() + ". " + exampleOrFull + "Records:"); + line.setBulletsOfText(new ArrayList<>(rowDescriptions.stream().map(String::valueOf).toList())); + } + } + processSummaryWarningsAndErrorsRollup.addToList(rs); return (rs); } + + + /*************************************************************************** + ** + ***************************************************************************/ + private record RowValue(String row, String value) + { + + /*************************************************************************** + ** + ***************************************************************************/ + public RowValue(BulkLoadValueTypeError bulkLoadValueTypeError, QRecord record) + { + this(BulkLoadRecordUtils.getRowNosString(record), ValueUtils.getValueAsString(bulkLoadValueTypeError.getValue())); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public String toString() + { + return row + " [" + value + "]"; + } + } + } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadValueTypeError.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadValueTypeError.java new file mode 100644 index 00000000..a7e6f371 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadValueTypeError.java @@ -0,0 +1,76 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping; + + +import java.io.Serializable; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; +import com.kingsrook.qqq.backend.core.model.statusmessages.BadInputStatusMessage; + + +/******************************************************************************* + ** Specialized error for records, for bulk-load use-cases, where we want to + ** report back info to the user about the field & value. + *******************************************************************************/ +public class BulkLoadValueTypeError extends BadInputStatusMessage +{ + private final String fieldLabel; + private final String fieldName; + private final Serializable value; + private final QFieldType type; + + + + /******************************************************************************* + ** Constructor + ** + *******************************************************************************/ + public BulkLoadValueTypeError(String fieldName, Serializable value, QFieldType type, String fieldLabel) + { + super("Value [" + value + "] for field [" + fieldLabel + "] could not be converted to type [" + type + "]"); + this.fieldName = fieldName; + this.value = value; + this.type = type; + this.fieldLabel = fieldLabel; + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + public String getMessageToUseAsProcessSummaryRollupKey() + { + return ("Cannot convert value for field [" + fieldLabel + "] to type [" + type.getMixedCaseLabel() + "]"); + } + + + + /******************************************************************************* + ** Getter for value + ** + *******************************************************************************/ + public Serializable getValue() + { + return value; + } +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/ValueMapper.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/ValueMapper.java index fdf1b6bd..44a37f6a 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/ValueMapper.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/ValueMapper.java @@ -34,7 +34,6 @@ import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; import com.kingsrook.qqq.backend.core.model.metadata.tables.Association; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; -import com.kingsrook.qqq.backend.core.model.statusmessages.BadInputStatusMessage; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkInsertMapping; import com.kingsrook.qqq.backend.core.utils.CollectionUtils; import com.kingsrook.qqq.backend.core.utils.StringUtils; @@ -70,6 +69,9 @@ public class ValueMapper return; } + String associationNamePrefixForFields = StringUtils.hasContent(associationNameChain) ? associationNameChain + "." : ""; + String tableLabelPrefix = StringUtils.hasContent(associationNameChain) ? table.getLabel() + ": " : ""; + Map> mappingForTable = mapping.getFieldNameToValueMappingForTable(associationNameChain); for(QRecord record : records) { @@ -102,7 +104,7 @@ public class ValueMapper } catch(Exception e) { - record.addError(new BadInputStatusMessage("Value [" + value + "] for field [" + field.getLabel() + "] could not be converted to type [" + type + "]")); + record.addError(new BulkLoadValueTypeError(associationNamePrefixForFields + field.getName(), value, type, tableLabelPrefix + field.getLabel())); } } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/general/ProcessSummaryWarningsAndErrorsRollup.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/general/ProcessSummaryWarningsAndErrorsRollup.java index 606a026b..6296f31c 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/general/ProcessSummaryWarningsAndErrorsRollup.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/general/ProcessSummaryWarningsAndErrorsRollup.java @@ -477,4 +477,13 @@ public class ProcessSummaryWarningsAndErrorsRollup } + + /******************************************************************************* + ** Getter for errorSummaries + ** + *******************************************************************************/ + public Map getErrorSummaries() + { + return errorSummaries; + } } diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/actions/processes/ProcessSummaryAssert.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/actions/processes/ProcessSummaryAssert.java new file mode 100644 index 00000000..a0bf6a36 --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/actions/processes/ProcessSummaryAssert.java @@ -0,0 +1,208 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.actions.processes; + + +import java.util.ArrayList; +import java.util.List; +import com.kingsrook.qqq.backend.core.utils.StringUtils; +import org.assertj.core.api.AbstractAssert; +import org.assertj.core.api.Assertions; +import static org.junit.jupiter.api.Assertions.fail; + + +/******************************************************************************* + ** AssertJ assert class for ProcessSummary - that is - a list of ProcessSummaryLineInterface's + *******************************************************************************/ +public class ProcessSummaryAssert extends AbstractAssert> +{ + + /******************************************************************************* + ** + *******************************************************************************/ + protected ProcessSummaryAssert(List actual, Class selfType) + { + super(actual, selfType); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @SuppressWarnings("unchecked") + public static ProcessSummaryAssert assertThat(RunProcessOutput runProcessOutput) + { + List processResults = (List) runProcessOutput.getValue("processResults"); + if(processResults == null) + { + processResults = (List) runProcessOutput.getValue("validationSummary"); + } + + return (new ProcessSummaryAssert(processResults, ProcessSummaryAssert.class)); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @SuppressWarnings("unchecked") + public static ProcessSummaryAssert assertThat(RunBackendStepOutput runBackendStepOutput) + { + List processResults = (List) runBackendStepOutput.getValue("processResults"); + if(processResults == null) + { + processResults = (List) runBackendStepOutput.getValue("validationSummary"); + } + + if(processResults == null) + { + fail("Could not find process results in backend step output."); + } + + return (new ProcessSummaryAssert(processResults, ProcessSummaryAssert.class)); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static ProcessSummaryAssert assertThat(List actual) + { + return (new ProcessSummaryAssert(actual, ProcessSummaryAssert.class)); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public ProcessSummaryAssert hasSize(int expectedSize) + { + Assertions.assertThat(actual).hasSize(expectedSize); + return (this); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public ProcessSummaryLineInterfaceAssert hasLineWithMessageMatching(String regExp) + { + List foundMessages = new ArrayList<>(); + for(ProcessSummaryLineInterface processSummaryLineInterface : actual) + { + if(processSummaryLineInterface.getMessage() == null) + { + processSummaryLineInterface.prepareForFrontend(false); + } + + if(processSummaryLineInterface.getMessage() != null && processSummaryLineInterface.getMessage().matches(regExp)) + { + return (new ProcessSummaryLineInterfaceAssert(processSummaryLineInterface, ProcessSummaryLineInterfaceAssert.class)); + } + else + { + foundMessages.add(processSummaryLineInterface.getMessage()); + } + } + + failWithMessage("Failed to find a ProcessSummaryLine with message matching [" + regExp + "].\nFound messages were:\n" + StringUtils.join("\n", foundMessages)); + return (null); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public ProcessSummaryLineInterfaceAssert hasLineWithMessageContaining(String substr) + { + List foundMessages = new ArrayList<>(); + for(ProcessSummaryLineInterface processSummaryLineInterface : actual) + { + if(processSummaryLineInterface.getMessage() == null) + { + processSummaryLineInterface.prepareForFrontend(false); + } + + if(processSummaryLineInterface.getMessage() != null && processSummaryLineInterface.getMessage().contains(substr)) + { + return (new ProcessSummaryLineInterfaceAssert(processSummaryLineInterface, ProcessSummaryLineInterfaceAssert.class)); + } + else + { + foundMessages.add(processSummaryLineInterface.getMessage()); + } + } + + failWithMessage("Failed to find a ProcessSummaryLine with message containing [" + substr + "].\nFound messages were:\n" + StringUtils.join("\n", foundMessages)); + return (null); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public ProcessSummaryLineInterfaceAssert hasLineWithStatus(Status status) + { + List foundStatuses = new ArrayList<>(); + for(ProcessSummaryLineInterface processSummaryLineInterface : actual) + { + if(status.equals(processSummaryLineInterface.getStatus())) + { + return (new ProcessSummaryLineInterfaceAssert(processSummaryLineInterface, ProcessSummaryLineInterfaceAssert.class)); + } + else + { + foundStatuses.add(String.valueOf(processSummaryLineInterface.getStatus())); + } + } + + failWithMessage("Failed to find a ProcessSummaryLine with status [" + status + "].\nFound statuses were:\n" + StringUtils.join("\n", foundStatuses)); + return (null); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public ProcessSummaryAssert hasNoLineWithStatus(Status status) + { + for(ProcessSummaryLineInterface processSummaryLineInterface : actual) + { + if(status.equals(processSummaryLineInterface.getStatus())) + { + failWithMessage("Found a ProcessSummaryLine with status [" + status + "], which was not supposed to happen."); + return (null); + } + } + + return (this); + } + +} diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/actions/processes/ProcessSummaryLineInterfaceAssert.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/actions/processes/ProcessSummaryLineInterfaceAssert.java new file mode 100644 index 00000000..861e21a7 --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/actions/processes/ProcessSummaryLineInterfaceAssert.java @@ -0,0 +1,189 @@ + +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.actions.processes; + + +import org.assertj.core.api.AbstractAssert; +import org.assertj.core.api.Assertions; +import static org.junit.jupiter.api.Assertions.assertEquals; + + +/******************************************************************************* + ** AssertJ assert class for ProcessSummaryLine. + *******************************************************************************/ +public class ProcessSummaryLineInterfaceAssert extends AbstractAssert +{ + + /******************************************************************************* + ** + *******************************************************************************/ + protected ProcessSummaryLineInterfaceAssert(ProcessSummaryLineInterface actual, Class selfType) + { + super(actual, selfType); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static ProcessSummaryLineInterfaceAssert assertThat(ProcessSummaryLineInterface actual) + { + return (new ProcessSummaryLineInterfaceAssert(actual, ProcessSummaryLineInterfaceAssert.class)); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public ProcessSummaryLineInterfaceAssert hasCount(Integer count) + { + if(actual instanceof ProcessSummaryLine psl) + { + assertEquals(count, psl.getCount(), "Expected count in process summary line"); + } + else + { + failWithMessage("ProcessSummaryLineInterface is not of concrete type ProcessSummaryLine (is: " + actual.getClass().getSimpleName() + ")"); + } + + return (this); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public ProcessSummaryLineInterfaceAssert hasStatus(Status status) + { + assertEquals(status, actual.getStatus(), "Expected status in process summary line"); + return (this); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public ProcessSummaryLineInterfaceAssert hasMessageMatching(String regExp) + { + if(actual.getMessage() == null) + { + actual.prepareForFrontend(false); + } + + Assertions.assertThat(actual.getMessage()).matches(regExp); + return (this); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public ProcessSummaryLineInterfaceAssert hasMessageContaining(String substring) + { + if(actual.getMessage() == null) + { + actual.prepareForFrontend(false); + } + + Assertions.assertThat(actual.getMessage()).contains(substring); + return (this); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public ProcessSummaryLineInterfaceAssert doesNotHaveMessageMatching(String regExp) + { + if(actual.getMessage() == null) + { + actual.prepareForFrontend(false); + } + + Assertions.assertThat(actual.getMessage()).doesNotMatch(regExp); + return (this); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public ProcessSummaryLineInterfaceAssert doesNotHaveMessageContaining(String substring) + { + if(actual.getMessage() == null) + { + actual.prepareForFrontend(false); + } + + Assertions.assertThat(actual.getMessage()).doesNotContain(substring); + return (this); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public ProcessSummaryLineInterfaceAssert hasAnyBulletsOfTextContaining(String substring) + { + if(actual instanceof ProcessSummaryLine psl) + { + Assertions.assertThat(psl.getBulletsOfText()) + .isNotNull() + .anyMatch(s -> s.contains(substring)); + } + else + { + Assertions.fail("Process Summary Line was not the expected type."); + } + + return (this); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public ProcessSummaryLineInterfaceAssert doesNotHaveAnyBulletsOfTextContaining(String substring) + { + if(actual instanceof ProcessSummaryLine psl) + { + if(psl.getBulletsOfText() != null) + { + Assertions.assertThat(psl.getBulletsOfText()) + .noneMatch(s -> s.contains(substring)); + } + } + + return (this); + } + +} diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTransformStepTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTransformStepTest.java index b87b6799..5a8ae6d3 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTransformStepTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTransformStepTest.java @@ -22,10 +22,14 @@ package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert; +import java.io.Serializable; import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; import com.kingsrook.qqq.backend.core.BaseTest; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryAssert; import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLine; import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLineInterface; import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; @@ -38,8 +42,12 @@ import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.UniqueKey; import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryRecordStore; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping.BulkLoadRecordUtils; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping.BulkLoadValueTypeError; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow; import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess; import com.kingsrook.qqq.backend.core.utils.TestUtils; +import com.kingsrook.qqq.backend.core.utils.collections.ListBuilder; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -87,9 +95,9 @@ class BulkInsertTransformStepTest extends BaseTest // insert some records that will cause some UK violations // //////////////////////////////////////////////////////////// TestUtils.insertRecords(table, List.of( - newQRecord("uuid-A", "SKU-1", 1), - newQRecord("uuid-B", "SKU-2", 1), - newQRecord("uuid-C", "SKU-2", 2) + newUkTestQRecord("uuid-A", "SKU-1", 1), + newUkTestQRecord("uuid-B", "SKU-2", 1), + newUkTestQRecord("uuid-C", "SKU-2", 2) )); /////////////////////////////////////////// @@ -102,13 +110,13 @@ class BulkInsertTransformStepTest extends BaseTest input.setTableName(TABLE_NAME); input.setStepName(StreamedETLWithFrontendProcess.STEP_NAME_VALIDATE); input.setRecords(List.of( - newQRecord("uuid-1", "SKU-A", 1), // OK. - newQRecord("uuid-1", "SKU-B", 1), // violate uuid UK in this set - newQRecord("uuid-2", "SKU-C", 1), // OK. - newQRecord("uuid-3", "SKU-C", 2), // OK. - newQRecord("uuid-4", "SKU-C", 1), // violate sku/storeId UK in this set - newQRecord("uuid-A", "SKU-X", 1), // violate uuid UK from pre-existing records - newQRecord("uuid-D", "SKU-2", 1) // violate sku/storeId UK from pre-existing records + newUkTestQRecord("uuid-1", "SKU-A", 1), // OK. + newUkTestQRecord("uuid-1", "SKU-B", 1), // violate uuid UK in this set + newUkTestQRecord("uuid-2", "SKU-C", 1), // OK. + newUkTestQRecord("uuid-3", "SKU-C", 2), // OK. + newUkTestQRecord("uuid-4", "SKU-C", 1), // violate sku/storeId UK in this set + newUkTestQRecord("uuid-A", "SKU-X", 1), // violate uuid UK from pre-existing records + newUkTestQRecord("uuid-D", "SKU-2", 1) // violate sku/storeId UK from pre-existing records )); bulkInsertTransformStep.preRun(input, output); bulkInsertTransformStep.runOnePage(input, output); @@ -171,9 +179,9 @@ class BulkInsertTransformStepTest extends BaseTest // insert some records that will cause some UK violations // //////////////////////////////////////////////////////////// TestUtils.insertRecords(table, List.of( - newQRecord("uuid-A", "SKU-1", 1), - newQRecord("uuid-B", "SKU-2", 1), - newQRecord("uuid-C", "SKU-2", 2) + newUkTestQRecord("uuid-A", "SKU-1", 1), + newUkTestQRecord("uuid-B", "SKU-2", 1), + newUkTestQRecord("uuid-C", "SKU-2", 2) )); /////////////////////////////////////////// @@ -186,20 +194,20 @@ class BulkInsertTransformStepTest extends BaseTest input.setTableName(TABLE_NAME); input.setStepName(StreamedETLWithFrontendProcess.STEP_NAME_VALIDATE); input.setRecords(List.of( - newQRecord("uuid-1", "SKU-A", 1), // OK. - newQRecord("uuid-1", "SKU-B", 1), // violate uuid UK in this set - newQRecord("uuid-2", "SKU-C", 1), // OK. - newQRecord("uuid-3", "SKU-C", 2), // OK. - newQRecord("uuid-4", "SKU-C", 1), // violate sku/storeId UK in this set - newQRecord("uuid-A", "SKU-X", 1), // violate uuid UK from pre-existing records - newQRecord("uuid-D", "SKU-2", 1) // violate sku/storeId UK from pre-existing records + newUkTestQRecord("uuid-1", "SKU-A", 1), // OK. + newUkTestQRecord("uuid-1", "SKU-B", 1), // violate uuid UK in this set + newUkTestQRecord("uuid-2", "SKU-C", 1), // OK. + newUkTestQRecord("uuid-3", "SKU-C", 2), // OK. + newUkTestQRecord("uuid-4", "SKU-C", 1), // violate sku/storeId UK in this set + newUkTestQRecord("uuid-A", "SKU-X", 1), // violate uuid UK from pre-existing records + newUkTestQRecord("uuid-D", "SKU-2", 1) // violate sku/storeId UK from pre-existing records )); bulkInsertTransformStep.preRun(input, output); bulkInsertTransformStep.runOnePage(input, output); - /////////////////////////////////////////////////////// - // assert that all records pass. - /////////////////////////////////////////////////////// + /////////////////////////////////// + // assert that all records pass. // + /////////////////////////////////// assertEquals(7, output.getRecords().size()); } @@ -211,8 +219,8 @@ class BulkInsertTransformStepTest extends BaseTest private boolean recordEquals(QRecord record, String uuid, String sku, Integer storeId) { return (record.getValue("uuid").equals(uuid) - && record.getValue("sku").equals(sku) - && record.getValue("storeId").equals(storeId)); + && record.getValue("sku").equals(sku) + && record.getValue("storeId").equals(storeId)); } @@ -220,7 +228,7 @@ class BulkInsertTransformStepTest extends BaseTest /******************************************************************************* ** *******************************************************************************/ - private QRecord newQRecord(String uuid, String sku, int storeId) + private QRecord newUkTestQRecord(String uuid, String sku, int storeId) { return new QRecord() .withValue("uuid", uuid) @@ -229,4 +237,134 @@ class BulkInsertTransformStepTest extends BaseTest .withValue("name", "Some Item"); } + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testValueMappingTypeErrors() throws QException + { + /////////////////////////////////////////// + // setup & run the bulk insert transform // + /////////////////////////////////////////// + BulkInsertTransformStep bulkInsertTransformStep = new BulkInsertTransformStep(); + RunBackendStepInput input = new RunBackendStepInput(); + RunBackendStepOutput output = new RunBackendStepOutput(); + Serializable[] emptyValues = new Serializable[0]; + + input.setTableName(TestUtils.TABLE_NAME_ORDER); + input.setStepName(StreamedETLWithFrontendProcess.STEP_NAME_VALIDATE); + input.setRecords(ListBuilder.of( + BulkLoadRecordUtils.addBackendDetailsAboutFileRows(new QRecord(), new BulkLoadFileRow(emptyValues, 1)) + .withError(new BulkLoadValueTypeError("storeId", "A", QFieldType.INTEGER, "Store")) + .withError(new BulkLoadValueTypeError("orderDate", "47", QFieldType.DATE, "Order Date")), + BulkLoadRecordUtils.addBackendDetailsAboutFileRows(new QRecord(), new BulkLoadFileRow(emptyValues, 2)) + .withError(new BulkLoadValueTypeError("storeId", "BCD", QFieldType.INTEGER, "Store")) + )); + + /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // add 102 records with an error in the total field - which is more than the number of examples that should be given // + /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + for(int i = 0; i < 102; i++) + { + input.getRecords().add(BulkLoadRecordUtils.addBackendDetailsAboutFileRows(new QRecord(), new BulkLoadFileRow(emptyValues, 3 + i)) + .withError(new BulkLoadValueTypeError("total", "three-fifty-" + i, QFieldType.DECIMAL, "Total"))); + } + + bulkInsertTransformStep.preRun(input, output); + bulkInsertTransformStep.runOnePage(input, output); + ArrayList processSummary = bulkInsertTransformStep.getProcessSummary(output, false); + + ProcessSummaryAssert.assertThat(processSummary) + .hasLineWithMessageContaining("Cannot convert value for field [Store] to type [Integer]") + .hasMessageContaining("Values:") + .doesNotHaveMessageContaining("Example Values:") + .hasAnyBulletsOfTextContaining("Row 1 [A]") + .hasAnyBulletsOfTextContaining("Row 2 [BCD]") + .hasStatus(Status.ERROR) + .hasCount(2); + + ProcessSummaryAssert.assertThat(processSummary) + .hasLineWithMessageContaining("Cannot convert value for field [Order Date] to type [Date]") + .hasMessageContaining("Values:") + .doesNotHaveMessageContaining("Example Values:") + .hasAnyBulletsOfTextContaining("Row 1 [47]") + .hasStatus(Status.ERROR) + .hasCount(1); + + ProcessSummaryAssert.assertThat(processSummary) + .hasLineWithMessageContaining("Cannot convert value for field [Total] to type [Decimal]") + .hasMessageContaining("Example Values:") + .hasAnyBulletsOfTextContaining("Row 3 [three-fifty-0]") + .hasAnyBulletsOfTextContaining("Row 4 [three-fifty-1]") + .hasAnyBulletsOfTextContaining("Row 5 [three-fifty-2]") + .doesNotHaveAnyBulletsOfTextContaining("three-fifty-101") + .hasStatus(Status.ERROR) + .hasCount(102); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testRollupOfValidationErrors() throws QException + { + QContext.getQSession().withSecurityKeyValue(TestUtils.SECURITY_KEY_TYPE_STORE, 1); + + /////////////////////////////////////////// + // setup & run the bulk insert transform // + /////////////////////////////////////////// + BulkInsertTransformStep bulkInsertTransformStep = new BulkInsertTransformStep(); + RunBackendStepInput input = new RunBackendStepInput(); + RunBackendStepOutput output = new RunBackendStepOutput(); + Serializable[] emptyValues = new Serializable[0]; + + String tooLong = ".".repeat(201); + + input.setTableName(TestUtils.TABLE_NAME_ORDER); + input.setStepName(StreamedETLWithFrontendProcess.STEP_NAME_VALIDATE); + input.setRecords(ListBuilder.of( + BulkLoadRecordUtils.addBackendDetailsAboutFileRows(new QRecord().withValue("shipToName", tooLong), new BulkLoadFileRow(emptyValues, 1)), + BulkLoadRecordUtils.addBackendDetailsAboutFileRows(new QRecord().withValue("shipToName", "OK").withValue("storeId", 1), new BulkLoadFileRow(emptyValues, 2)) + )); + + ///////////////////////////////////////////////////////////////////// + // add 102 records with no security key - which should be an error // + ///////////////////////////////////////////////////////////////////// + for(int i = 0; i < 102; i++) + { + input.getRecords().add(BulkLoadRecordUtils.addBackendDetailsAboutFileRows(new QRecord(), new BulkLoadFileRow(emptyValues, 3 + i))); + } + + bulkInsertTransformStep.preRun(input, output); + bulkInsertTransformStep.runOnePage(input, output); + ArrayList processSummary = bulkInsertTransformStep.getProcessSummary(output, false); + + ProcessSummaryAssert.assertThat(processSummary) + .hasLineWithMessageContaining("value for Ship To Name is too long") + .hasMessageContaining("Records:") + .doesNotHaveMessageContaining("Example Records:") + .hasAnyBulletsOfTextContaining("Row 1") + .hasStatus(Status.ERROR) + .hasCount(1); + + ProcessSummaryAssert.assertThat(processSummary) + .hasLineWithMessageContaining("without a value in the field: Store Id") + .hasMessageContaining("Example Records:") + .hasAnyBulletsOfTextContaining("Row 1") + .hasAnyBulletsOfTextContaining("Row 3") + .hasAnyBulletsOfTextContaining("Row 4") + .doesNotHaveAnyBulletsOfTextContaining("Row 101") + .hasStatus(Status.ERROR) + .hasCount(103); // the 102, plus row 1. + + ProcessSummaryAssert.assertThat(processSummary) + .hasLineWithMessageContaining("Order record will be inserted") + .hasStatus(Status.OK) + .hasCount(1); + } + } \ No newline at end of file diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java index 89adac13..54502074 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java @@ -70,6 +70,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.dashboard.QWidgetMetaData; import com.kingsrook.qqq.backend.core.model.metadata.fields.DisplayFormat; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; +import com.kingsrook.qqq.backend.core.model.metadata.fields.ValueTooLongBehavior; import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinOn; import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinType; import com.kingsrook.qqq.backend.core.model.metadata.joins.QJoinMetaData; @@ -662,7 +663,7 @@ public class TestUtils .withField(new QFieldMetaData("createDate", QFieldType.DATE_TIME).withIsEditable(false)) .withField(new QFieldMetaData("modifyDate", QFieldType.DATE_TIME).withIsEditable(false)) .withField(new QFieldMetaData("orderNo", QFieldType.STRING)) - .withField(new QFieldMetaData("shipToName", QFieldType.STRING)) + .withField(new QFieldMetaData("shipToName", QFieldType.STRING).withMaxLength(200).withBehavior(ValueTooLongBehavior.ERROR)) .withField(new QFieldMetaData("orderDate", QFieldType.DATE)) .withField(new QFieldMetaData("storeId", QFieldType.INTEGER)) .withField(new QFieldMetaData("total", QFieldType.DECIMAL).withDisplayFormat(DisplayFormat.CURRENCY).withFieldSecurityLock(new FieldSecurityLock()