CE-1955 - Summarize with some examples (including rows nos) for value mapping and other validation errors

This commit is contained in:
2024-11-27 12:35:54 -06:00
parent 6ed9dfd498
commit c88fd5b7d4
9 changed files with 823 additions and 38 deletions

View File

@ -27,6 +27,7 @@ import java.time.Instant;
import java.time.LocalDate;
import java.time.LocalTime;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.utils.StringUtils;
/*******************************************************************************
@ -100,6 +101,16 @@ public enum QFieldType
/***************************************************************************
**
***************************************************************************/
public String getMixedCaseLabel()
{
return StringUtils.allCapsToMixedCase(name());
}
/*******************************************************************************
**
*******************************************************************************/

View File

@ -29,6 +29,7 @@ import java.util.HashSet;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import java.util.Set;
import com.kingsrook.qqq.backend.core.actions.customizers.AbstractPreInsertCustomizer;
@ -47,16 +48,25 @@ import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutp
import com.kingsrook.qqq.backend.core.model.actions.processes.Status;
import com.kingsrook.qqq.backend.core.model.actions.tables.QInputSource;
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
import com.kingsrook.qqq.backend.core.model.dashboard.widgets.WidgetType;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.dashboard.QWidgetMetaDataInterface;
import com.kingsrook.qqq.backend.core.model.metadata.tables.Association;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QFieldSection;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.tables.UniqueKey;
import com.kingsrook.qqq.backend.core.model.statusmessages.QErrorMessage;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping.BulkLoadRecordUtils;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping.BulkLoadValueTypeError;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkInsertMapping;
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.AbstractTransformStep;
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.LoadViaInsertStep;
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess;
import com.kingsrook.qqq.backend.core.processes.implementations.general.ProcessSummaryWarningsAndErrorsRollup;
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
import com.kingsrook.qqq.backend.core.utils.ListingHash;
import com.kingsrook.qqq.backend.core.utils.StringUtils;
import com.kingsrook.qqq.backend.core.utils.ValueUtils;
/*******************************************************************************
@ -66,7 +76,11 @@ public class BulkInsertTransformStep extends AbstractTransformStep
{
private ProcessSummaryLine okSummary = new ProcessSummaryLine(Status.OK);
private ProcessSummaryWarningsAndErrorsRollup processSummaryWarningsAndErrorsRollup = ProcessSummaryWarningsAndErrorsRollup.build("inserted");
private ProcessSummaryWarningsAndErrorsRollup processSummaryWarningsAndErrorsRollup = ProcessSummaryWarningsAndErrorsRollup.build("inserted")
.withDoReplaceSingletonCountLinesWithSuffixOnly(false);
private ListingHash<String, RowValue> errorToExampleRowValueMap = new ListingHash<>();
private ListingHash<String, String> errorToExampleRowsMap = new ListingHash<>();
private Map<UniqueKey, ProcessSummaryLineWithUKSampleValues> ukErrorSummaries = new HashMap<>();
private Map<String, ProcessSummaryLine> associationsToInsertSummaries = new HashMap<>();
@ -77,6 +91,7 @@ public class BulkInsertTransformStep extends AbstractTransformStep
private int rowsProcessed = 0;
private final int EXAMPLE_ROW_LIMIT = 10;
/*******************************************************************************
@ -118,6 +133,44 @@ public class BulkInsertTransformStep extends AbstractTransformStep
// make sure that if a saved profile was selected on a review screen, that the result screen knows about it. //
///////////////////////////////////////////////////////////////////////////////////////////////////////////////
BulkInsertStepUtils.handleSavedBulkLoadProfileIdValue(runBackendStepInput, runBackendStepOutput);
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// set up the validationReview widget to render preview records using the table layout, and including the associations //
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
runBackendStepOutput.addValue("formatPreviewRecordUsingTableLayout", table.getName());
BulkInsertMapping bulkInsertMapping = (BulkInsertMapping) runBackendStepOutput.getValue("bulkInsertMapping");
if(bulkInsertMapping != null)
{
ArrayList<String> previewRecordAssociatedTableNames = new ArrayList<>();
ArrayList<String> previewRecordAssociatedWidgetNames = new ArrayList<>();
ArrayList<String> previewRecordAssociationNames = new ArrayList<>();
for(String mappedAssociation : bulkInsertMapping.getMappedAssociations())
{
Optional<Association> association = table.getAssociations().stream().filter(a -> a.getName().equals(mappedAssociation)).findFirst();
if(association.isPresent())
{
for(QFieldSection section : table.getSections())
{
QWidgetMetaDataInterface widget = QContext.getQInstance().getWidget(section.getWidgetName());
if(widget != null && WidgetType.CHILD_RECORD_LIST.getType().equals(widget.getType()))
{
Serializable widgetJoinName = widget.getDefaultValues().get("joinName");
if(Objects.equals(widgetJoinName, association.get().getJoinName()))
{
previewRecordAssociatedTableNames.add(association.get().getAssociatedTableName());
previewRecordAssociatedWidgetNames.add(widget.getName());
previewRecordAssociationNames.add(association.get().getName());
}
}
}
}
}
runBackendStepOutput.addValue("previewRecordAssociatedTableNames", previewRecordAssociatedTableNames);
runBackendStepOutput.addValue("previewRecordAssociatedWidgetNames", previewRecordAssociatedWidgetNames);
runBackendStepOutput.addValue("previewRecordAssociationNames", previewRecordAssociationNames);
}
}
@ -131,7 +184,9 @@ public class BulkInsertTransformStep extends AbstractTransformStep
int recordsInThisPage = runBackendStepInput.getRecords().size();
QTableMetaData table = QContext.getQInstance().getTable(runBackendStepInput.getTableName());
// split the records w/o UK errors into those w/ e
//////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// split the records into 2 lists: those w/ errors (e.g., from the bulk-load mapping), and those that are okay //
//////////////////////////////////////////////////////////////////////////////////////////////////////////////////
List<QRecord> recordsWithoutAnyErrors = new ArrayList<>();
List<QRecord> recordsWithSomeErrors = new ArrayList<>();
for(QRecord record : runBackendStepInput.getRecords())
@ -153,16 +208,26 @@ public class BulkInsertTransformStep extends AbstractTransformStep
{
for(QRecord record : recordsWithSomeErrors)
{
String message = record.getErrors().get(0).getMessage();
processSummaryWarningsAndErrorsRollup.addError(message, null);
for(QErrorMessage error : record.getErrors())
{
if(error instanceof BulkLoadValueTypeError blvte)
{
processSummaryWarningsAndErrorsRollup.addError(blvte.getMessageToUseAsProcessSummaryRollupKey(), null);
addToErrorToExampleRowValueMap(blvte, record);
}
else
{
processSummaryWarningsAndErrorsRollup.addError(error.getMessage(), null);
}
}
}
}
if(recordsWithoutAnyErrors.isEmpty())
{
////////////////////////////////////////////////////////////////////////////////
// skip th rest of this method if there aren't any records w/o errors in them //
////////////////////////////////////////////////////////////////////////////////
/////////////////////////////////////////////////////////////////////////////////
// skip the rest of this method if there aren't any records w/o errors in them //
/////////////////////////////////////////////////////////////////////////////////
this.rowsProcessed += recordsInThisPage;
}
@ -248,8 +313,11 @@ public class BulkInsertTransformStep extends AbstractTransformStep
{
if(CollectionUtils.nullSafeHasContents(record.getErrors()))
{
String message = record.getErrors().get(0).getMessage();
processSummaryWarningsAndErrorsRollup.addError(message, null);
for(QErrorMessage error : record.getErrors())
{
processSummaryWarningsAndErrorsRollup.addError(error.getMessage(), null);
addToErrorToExampleRowMap(error.getMessage(), record);
}
}
else if(CollectionUtils.nullSafeHasContents(record.getWarnings()))
{
@ -277,6 +345,37 @@ public class BulkInsertTransformStep extends AbstractTransformStep
/***************************************************************************
**
***************************************************************************/
private void addToErrorToExampleRowValueMap(BulkLoadValueTypeError bulkLoadValueTypeError, QRecord record)
{
String message = bulkLoadValueTypeError.getMessageToUseAsProcessSummaryRollupKey();
List<RowValue> rowValues = errorToExampleRowValueMap.computeIfAbsent(message, k -> new ArrayList<>());
if(rowValues.size() < EXAMPLE_ROW_LIMIT)
{
rowValues.add(new RowValue(bulkLoadValueTypeError, record));
}
}
/***************************************************************************
**
***************************************************************************/
private void addToErrorToExampleRowMap(String message, QRecord record)
{
List<String> rowNos = errorToExampleRowsMap.computeIfAbsent(message, k -> new ArrayList<>());
if(rowNos.size() < EXAMPLE_ROW_LIMIT)
{
rowNos.add(BulkLoadRecordUtils.getRowNosString(record));
}
}
/*******************************************************************************
**
*******************************************************************************/
@ -411,9 +510,61 @@ public class BulkInsertTransformStep extends AbstractTransformStep
ukErrorSummary.addSelfToListIfAnyCount(rs);
}
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// for process summary lines that exist in the error-to-example-row-value map, add those example values to the lines. //
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
for(Map.Entry<String, ProcessSummaryLine> entry : processSummaryWarningsAndErrorsRollup.getErrorSummaries().entrySet())
{
String message = entry.getKey();
if(errorToExampleRowValueMap.containsKey(message))
{
ProcessSummaryLine line = entry.getValue();
List<RowValue> rowValues = errorToExampleRowValueMap.get(message);
String exampleOrFull = rowValues.size() < line.getCount() ? "Example " : "";
line.setMessageSuffix(line.getMessageSuffix() + ". " + exampleOrFull + "Values:");
line.setBulletsOfText(new ArrayList<>(rowValues.stream().map(String::valueOf).toList()));
}
else if(errorToExampleRowsMap.containsKey(message))
{
ProcessSummaryLine line = entry.getValue();
List<String> rowDescriptions = errorToExampleRowsMap.get(message);
String exampleOrFull = rowDescriptions.size() < line.getCount() ? "Example " : "";
line.setMessageSuffix(line.getMessageSuffix() + ". " + exampleOrFull + "Records:");
line.setBulletsOfText(new ArrayList<>(rowDescriptions.stream().map(String::valueOf).toList()));
}
}
processSummaryWarningsAndErrorsRollup.addToList(rs);
return (rs);
}
/***************************************************************************
**
***************************************************************************/
private record RowValue(String row, String value)
{
/***************************************************************************
**
***************************************************************************/
public RowValue(BulkLoadValueTypeError bulkLoadValueTypeError, QRecord record)
{
this(BulkLoadRecordUtils.getRowNosString(record), ValueUtils.getValueAsString(bulkLoadValueTypeError.getValue()));
}
/***************************************************************************
**
***************************************************************************/
@Override
public String toString()
{
return row + " [" + value + "]";
}
}
}

View File

@ -0,0 +1,76 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2024. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping;
import java.io.Serializable;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
import com.kingsrook.qqq.backend.core.model.statusmessages.BadInputStatusMessage;
/*******************************************************************************
** Specialized error for records, for bulk-load use-cases, where we want to
** report back info to the user about the field & value.
*******************************************************************************/
public class BulkLoadValueTypeError extends BadInputStatusMessage
{
private final String fieldLabel;
private final String fieldName;
private final Serializable value;
private final QFieldType type;
/*******************************************************************************
** Constructor
**
*******************************************************************************/
public BulkLoadValueTypeError(String fieldName, Serializable value, QFieldType type, String fieldLabel)
{
super("Value [" + value + "] for field [" + fieldLabel + "] could not be converted to type [" + type + "]");
this.fieldName = fieldName;
this.value = value;
this.type = type;
this.fieldLabel = fieldLabel;
}
/***************************************************************************
**
***************************************************************************/
public String getMessageToUseAsProcessSummaryRollupKey()
{
return ("Cannot convert value for field [" + fieldLabel + "] to type [" + type.getMixedCaseLabel() + "]");
}
/*******************************************************************************
** Getter for value
**
*******************************************************************************/
public Serializable getValue()
{
return value;
}
}

View File

@ -34,7 +34,6 @@ import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
import com.kingsrook.qqq.backend.core.model.metadata.tables.Association;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.model.statusmessages.BadInputStatusMessage;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkInsertMapping;
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
import com.kingsrook.qqq.backend.core.utils.StringUtils;
@ -70,6 +69,9 @@ public class ValueMapper
return;
}
String associationNamePrefixForFields = StringUtils.hasContent(associationNameChain) ? associationNameChain + "." : "";
String tableLabelPrefix = StringUtils.hasContent(associationNameChain) ? table.getLabel() + ": " : "";
Map<String, Map<String, Serializable>> mappingForTable = mapping.getFieldNameToValueMappingForTable(associationNameChain);
for(QRecord record : records)
{
@ -102,7 +104,7 @@ public class ValueMapper
}
catch(Exception e)
{
record.addError(new BadInputStatusMessage("Value [" + value + "] for field [" + field.getLabel() + "] could not be converted to type [" + type + "]"));
record.addError(new BulkLoadValueTypeError(associationNamePrefixForFields + field.getName(), value, type, tableLabelPrefix + field.getLabel()));
}
}

View File

@ -477,4 +477,13 @@ public class ProcessSummaryWarningsAndErrorsRollup
}
/*******************************************************************************
** Getter for errorSummaries
**
*******************************************************************************/
public Map<String, ProcessSummaryLine> getErrorSummaries()
{
return errorSummaries;
}
}

View File

@ -0,0 +1,208 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2024. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.model.actions.processes;
import java.util.ArrayList;
import java.util.List;
import com.kingsrook.qqq.backend.core.utils.StringUtils;
import org.assertj.core.api.AbstractAssert;
import org.assertj.core.api.Assertions;
import static org.junit.jupiter.api.Assertions.fail;
/*******************************************************************************
** AssertJ assert class for ProcessSummary - that is - a list of ProcessSummaryLineInterface's
*******************************************************************************/
public class ProcessSummaryAssert extends AbstractAssert<ProcessSummaryAssert, List<ProcessSummaryLineInterface>>
{
/*******************************************************************************
**
*******************************************************************************/
protected ProcessSummaryAssert(List<ProcessSummaryLineInterface> actual, Class<?> selfType)
{
super(actual, selfType);
}
/*******************************************************************************
**
*******************************************************************************/
@SuppressWarnings("unchecked")
public static ProcessSummaryAssert assertThat(RunProcessOutput runProcessOutput)
{
List<ProcessSummaryLineInterface> processResults = (List<ProcessSummaryLineInterface>) runProcessOutput.getValue("processResults");
if(processResults == null)
{
processResults = (List<ProcessSummaryLineInterface>) runProcessOutput.getValue("validationSummary");
}
return (new ProcessSummaryAssert(processResults, ProcessSummaryAssert.class));
}
/*******************************************************************************
**
*******************************************************************************/
@SuppressWarnings("unchecked")
public static ProcessSummaryAssert assertThat(RunBackendStepOutput runBackendStepOutput)
{
List<ProcessSummaryLineInterface> processResults = (List<ProcessSummaryLineInterface>) runBackendStepOutput.getValue("processResults");
if(processResults == null)
{
processResults = (List<ProcessSummaryLineInterface>) runBackendStepOutput.getValue("validationSummary");
}
if(processResults == null)
{
fail("Could not find process results in backend step output.");
}
return (new ProcessSummaryAssert(processResults, ProcessSummaryAssert.class));
}
/*******************************************************************************
**
*******************************************************************************/
public static ProcessSummaryAssert assertThat(List<ProcessSummaryLineInterface> actual)
{
return (new ProcessSummaryAssert(actual, ProcessSummaryAssert.class));
}
/*******************************************************************************
**
*******************************************************************************/
public ProcessSummaryAssert hasSize(int expectedSize)
{
Assertions.assertThat(actual).hasSize(expectedSize);
return (this);
}
/*******************************************************************************
**
*******************************************************************************/
public ProcessSummaryLineInterfaceAssert hasLineWithMessageMatching(String regExp)
{
List<String> foundMessages = new ArrayList<>();
for(ProcessSummaryLineInterface processSummaryLineInterface : actual)
{
if(processSummaryLineInterface.getMessage() == null)
{
processSummaryLineInterface.prepareForFrontend(false);
}
if(processSummaryLineInterface.getMessage() != null && processSummaryLineInterface.getMessage().matches(regExp))
{
return (new ProcessSummaryLineInterfaceAssert(processSummaryLineInterface, ProcessSummaryLineInterfaceAssert.class));
}
else
{
foundMessages.add(processSummaryLineInterface.getMessage());
}
}
failWithMessage("Failed to find a ProcessSummaryLine with message matching [" + regExp + "].\nFound messages were:\n" + StringUtils.join("\n", foundMessages));
return (null);
}
/*******************************************************************************
**
*******************************************************************************/
public ProcessSummaryLineInterfaceAssert hasLineWithMessageContaining(String substr)
{
List<String> foundMessages = new ArrayList<>();
for(ProcessSummaryLineInterface processSummaryLineInterface : actual)
{
if(processSummaryLineInterface.getMessage() == null)
{
processSummaryLineInterface.prepareForFrontend(false);
}
if(processSummaryLineInterface.getMessage() != null && processSummaryLineInterface.getMessage().contains(substr))
{
return (new ProcessSummaryLineInterfaceAssert(processSummaryLineInterface, ProcessSummaryLineInterfaceAssert.class));
}
else
{
foundMessages.add(processSummaryLineInterface.getMessage());
}
}
failWithMessage("Failed to find a ProcessSummaryLine with message containing [" + substr + "].\nFound messages were:\n" + StringUtils.join("\n", foundMessages));
return (null);
}
/*******************************************************************************
**
*******************************************************************************/
public ProcessSummaryLineInterfaceAssert hasLineWithStatus(Status status)
{
List<String> foundStatuses = new ArrayList<>();
for(ProcessSummaryLineInterface processSummaryLineInterface : actual)
{
if(status.equals(processSummaryLineInterface.getStatus()))
{
return (new ProcessSummaryLineInterfaceAssert(processSummaryLineInterface, ProcessSummaryLineInterfaceAssert.class));
}
else
{
foundStatuses.add(String.valueOf(processSummaryLineInterface.getStatus()));
}
}
failWithMessage("Failed to find a ProcessSummaryLine with status [" + status + "].\nFound statuses were:\n" + StringUtils.join("\n", foundStatuses));
return (null);
}
/*******************************************************************************
**
*******************************************************************************/
public ProcessSummaryAssert hasNoLineWithStatus(Status status)
{
for(ProcessSummaryLineInterface processSummaryLineInterface : actual)
{
if(status.equals(processSummaryLineInterface.getStatus()))
{
failWithMessage("Found a ProcessSummaryLine with status [" + status + "], which was not supposed to happen.");
return (null);
}
}
return (this);
}
}

View File

@ -0,0 +1,189 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2024. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.model.actions.processes;
import org.assertj.core.api.AbstractAssert;
import org.assertj.core.api.Assertions;
import static org.junit.jupiter.api.Assertions.assertEquals;
/*******************************************************************************
** AssertJ assert class for ProcessSummaryLine.
*******************************************************************************/
public class ProcessSummaryLineInterfaceAssert extends AbstractAssert<ProcessSummaryLineInterfaceAssert, ProcessSummaryLineInterface>
{
/*******************************************************************************
**
*******************************************************************************/
protected ProcessSummaryLineInterfaceAssert(ProcessSummaryLineInterface actual, Class<?> selfType)
{
super(actual, selfType);
}
/*******************************************************************************
**
*******************************************************************************/
public static ProcessSummaryLineInterfaceAssert assertThat(ProcessSummaryLineInterface actual)
{
return (new ProcessSummaryLineInterfaceAssert(actual, ProcessSummaryLineInterfaceAssert.class));
}
/*******************************************************************************
**
*******************************************************************************/
public ProcessSummaryLineInterfaceAssert hasCount(Integer count)
{
if(actual instanceof ProcessSummaryLine psl)
{
assertEquals(count, psl.getCount(), "Expected count in process summary line");
}
else
{
failWithMessage("ProcessSummaryLineInterface is not of concrete type ProcessSummaryLine (is: " + actual.getClass().getSimpleName() + ")");
}
return (this);
}
/*******************************************************************************
**
*******************************************************************************/
public ProcessSummaryLineInterfaceAssert hasStatus(Status status)
{
assertEquals(status, actual.getStatus(), "Expected status in process summary line");
return (this);
}
/*******************************************************************************
**
*******************************************************************************/
public ProcessSummaryLineInterfaceAssert hasMessageMatching(String regExp)
{
if(actual.getMessage() == null)
{
actual.prepareForFrontend(false);
}
Assertions.assertThat(actual.getMessage()).matches(regExp);
return (this);
}
/*******************************************************************************
**
*******************************************************************************/
public ProcessSummaryLineInterfaceAssert hasMessageContaining(String substring)
{
if(actual.getMessage() == null)
{
actual.prepareForFrontend(false);
}
Assertions.assertThat(actual.getMessage()).contains(substring);
return (this);
}
/*******************************************************************************
**
*******************************************************************************/
public ProcessSummaryLineInterfaceAssert doesNotHaveMessageMatching(String regExp)
{
if(actual.getMessage() == null)
{
actual.prepareForFrontend(false);
}
Assertions.assertThat(actual.getMessage()).doesNotMatch(regExp);
return (this);
}
/*******************************************************************************
**
*******************************************************************************/
public ProcessSummaryLineInterfaceAssert doesNotHaveMessageContaining(String substring)
{
if(actual.getMessage() == null)
{
actual.prepareForFrontend(false);
}
Assertions.assertThat(actual.getMessage()).doesNotContain(substring);
return (this);
}
/*******************************************************************************
**
*******************************************************************************/
public ProcessSummaryLineInterfaceAssert hasAnyBulletsOfTextContaining(String substring)
{
if(actual instanceof ProcessSummaryLine psl)
{
Assertions.assertThat(psl.getBulletsOfText())
.isNotNull()
.anyMatch(s -> s.contains(substring));
}
else
{
Assertions.fail("Process Summary Line was not the expected type.");
}
return (this);
}
/*******************************************************************************
**
*******************************************************************************/
public ProcessSummaryLineInterfaceAssert doesNotHaveAnyBulletsOfTextContaining(String substring)
{
if(actual instanceof ProcessSummaryLine psl)
{
if(psl.getBulletsOfText() != null)
{
Assertions.assertThat(psl.getBulletsOfText())
.noneMatch(s -> s.contains(substring));
}
}
return (this);
}
}

View File

@ -22,10 +22,14 @@
package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.List;
import java.util.stream.Collectors;
import com.kingsrook.qqq.backend.core.BaseTest;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryAssert;
import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLine;
import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLineInterface;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
@ -38,8 +42,12 @@ import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.tables.UniqueKey;
import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryRecordStore;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping.BulkLoadRecordUtils;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping.BulkLoadValueTypeError;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow;
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess;
import com.kingsrook.qqq.backend.core.utils.TestUtils;
import com.kingsrook.qqq.backend.core.utils.collections.ListBuilder;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
@ -87,9 +95,9 @@ class BulkInsertTransformStepTest extends BaseTest
// insert some records that will cause some UK violations //
////////////////////////////////////////////////////////////
TestUtils.insertRecords(table, List.of(
newQRecord("uuid-A", "SKU-1", 1),
newQRecord("uuid-B", "SKU-2", 1),
newQRecord("uuid-C", "SKU-2", 2)
newUkTestQRecord("uuid-A", "SKU-1", 1),
newUkTestQRecord("uuid-B", "SKU-2", 1),
newUkTestQRecord("uuid-C", "SKU-2", 2)
));
///////////////////////////////////////////
@ -102,13 +110,13 @@ class BulkInsertTransformStepTest extends BaseTest
input.setTableName(TABLE_NAME);
input.setStepName(StreamedETLWithFrontendProcess.STEP_NAME_VALIDATE);
input.setRecords(List.of(
newQRecord("uuid-1", "SKU-A", 1), // OK.
newQRecord("uuid-1", "SKU-B", 1), // violate uuid UK in this set
newQRecord("uuid-2", "SKU-C", 1), // OK.
newQRecord("uuid-3", "SKU-C", 2), // OK.
newQRecord("uuid-4", "SKU-C", 1), // violate sku/storeId UK in this set
newQRecord("uuid-A", "SKU-X", 1), // violate uuid UK from pre-existing records
newQRecord("uuid-D", "SKU-2", 1) // violate sku/storeId UK from pre-existing records
newUkTestQRecord("uuid-1", "SKU-A", 1), // OK.
newUkTestQRecord("uuid-1", "SKU-B", 1), // violate uuid UK in this set
newUkTestQRecord("uuid-2", "SKU-C", 1), // OK.
newUkTestQRecord("uuid-3", "SKU-C", 2), // OK.
newUkTestQRecord("uuid-4", "SKU-C", 1), // violate sku/storeId UK in this set
newUkTestQRecord("uuid-A", "SKU-X", 1), // violate uuid UK from pre-existing records
newUkTestQRecord("uuid-D", "SKU-2", 1) // violate sku/storeId UK from pre-existing records
));
bulkInsertTransformStep.preRun(input, output);
bulkInsertTransformStep.runOnePage(input, output);
@ -171,9 +179,9 @@ class BulkInsertTransformStepTest extends BaseTest
// insert some records that will cause some UK violations //
////////////////////////////////////////////////////////////
TestUtils.insertRecords(table, List.of(
newQRecord("uuid-A", "SKU-1", 1),
newQRecord("uuid-B", "SKU-2", 1),
newQRecord("uuid-C", "SKU-2", 2)
newUkTestQRecord("uuid-A", "SKU-1", 1),
newUkTestQRecord("uuid-B", "SKU-2", 1),
newUkTestQRecord("uuid-C", "SKU-2", 2)
));
///////////////////////////////////////////
@ -186,20 +194,20 @@ class BulkInsertTransformStepTest extends BaseTest
input.setTableName(TABLE_NAME);
input.setStepName(StreamedETLWithFrontendProcess.STEP_NAME_VALIDATE);
input.setRecords(List.of(
newQRecord("uuid-1", "SKU-A", 1), // OK.
newQRecord("uuid-1", "SKU-B", 1), // violate uuid UK in this set
newQRecord("uuid-2", "SKU-C", 1), // OK.
newQRecord("uuid-3", "SKU-C", 2), // OK.
newQRecord("uuid-4", "SKU-C", 1), // violate sku/storeId UK in this set
newQRecord("uuid-A", "SKU-X", 1), // violate uuid UK from pre-existing records
newQRecord("uuid-D", "SKU-2", 1) // violate sku/storeId UK from pre-existing records
newUkTestQRecord("uuid-1", "SKU-A", 1), // OK.
newUkTestQRecord("uuid-1", "SKU-B", 1), // violate uuid UK in this set
newUkTestQRecord("uuid-2", "SKU-C", 1), // OK.
newUkTestQRecord("uuid-3", "SKU-C", 2), // OK.
newUkTestQRecord("uuid-4", "SKU-C", 1), // violate sku/storeId UK in this set
newUkTestQRecord("uuid-A", "SKU-X", 1), // violate uuid UK from pre-existing records
newUkTestQRecord("uuid-D", "SKU-2", 1) // violate sku/storeId UK from pre-existing records
));
bulkInsertTransformStep.preRun(input, output);
bulkInsertTransformStep.runOnePage(input, output);
///////////////////////////////////////////////////////
// assert that all records pass.
///////////////////////////////////////////////////////
///////////////////////////////////
// assert that all records pass. //
///////////////////////////////////
assertEquals(7, output.getRecords().size());
}
@ -211,8 +219,8 @@ class BulkInsertTransformStepTest extends BaseTest
private boolean recordEquals(QRecord record, String uuid, String sku, Integer storeId)
{
return (record.getValue("uuid").equals(uuid)
&& record.getValue("sku").equals(sku)
&& record.getValue("storeId").equals(storeId));
&& record.getValue("sku").equals(sku)
&& record.getValue("storeId").equals(storeId));
}
@ -220,7 +228,7 @@ class BulkInsertTransformStepTest extends BaseTest
/*******************************************************************************
**
*******************************************************************************/
private QRecord newQRecord(String uuid, String sku, int storeId)
private QRecord newUkTestQRecord(String uuid, String sku, int storeId)
{
return new QRecord()
.withValue("uuid", uuid)
@ -229,4 +237,134 @@ class BulkInsertTransformStepTest extends BaseTest
.withValue("name", "Some Item");
}
/*******************************************************************************
**
*******************************************************************************/
@Test
void testValueMappingTypeErrors() throws QException
{
///////////////////////////////////////////
// setup & run the bulk insert transform //
///////////////////////////////////////////
BulkInsertTransformStep bulkInsertTransformStep = new BulkInsertTransformStep();
RunBackendStepInput input = new RunBackendStepInput();
RunBackendStepOutput output = new RunBackendStepOutput();
Serializable[] emptyValues = new Serializable[0];
input.setTableName(TestUtils.TABLE_NAME_ORDER);
input.setStepName(StreamedETLWithFrontendProcess.STEP_NAME_VALIDATE);
input.setRecords(ListBuilder.of(
BulkLoadRecordUtils.addBackendDetailsAboutFileRows(new QRecord(), new BulkLoadFileRow(emptyValues, 1))
.withError(new BulkLoadValueTypeError("storeId", "A", QFieldType.INTEGER, "Store"))
.withError(new BulkLoadValueTypeError("orderDate", "47", QFieldType.DATE, "Order Date")),
BulkLoadRecordUtils.addBackendDetailsAboutFileRows(new QRecord(), new BulkLoadFileRow(emptyValues, 2))
.withError(new BulkLoadValueTypeError("storeId", "BCD", QFieldType.INTEGER, "Store"))
));
///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// add 102 records with an error in the total field - which is more than the number of examples that should be given //
///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
for(int i = 0; i < 102; i++)
{
input.getRecords().add(BulkLoadRecordUtils.addBackendDetailsAboutFileRows(new QRecord(), new BulkLoadFileRow(emptyValues, 3 + i))
.withError(new BulkLoadValueTypeError("total", "three-fifty-" + i, QFieldType.DECIMAL, "Total")));
}
bulkInsertTransformStep.preRun(input, output);
bulkInsertTransformStep.runOnePage(input, output);
ArrayList<ProcessSummaryLineInterface> processSummary = bulkInsertTransformStep.getProcessSummary(output, false);
ProcessSummaryAssert.assertThat(processSummary)
.hasLineWithMessageContaining("Cannot convert value for field [Store] to type [Integer]")
.hasMessageContaining("Values:")
.doesNotHaveMessageContaining("Example Values:")
.hasAnyBulletsOfTextContaining("Row 1 [A]")
.hasAnyBulletsOfTextContaining("Row 2 [BCD]")
.hasStatus(Status.ERROR)
.hasCount(2);
ProcessSummaryAssert.assertThat(processSummary)
.hasLineWithMessageContaining("Cannot convert value for field [Order Date] to type [Date]")
.hasMessageContaining("Values:")
.doesNotHaveMessageContaining("Example Values:")
.hasAnyBulletsOfTextContaining("Row 1 [47]")
.hasStatus(Status.ERROR)
.hasCount(1);
ProcessSummaryAssert.assertThat(processSummary)
.hasLineWithMessageContaining("Cannot convert value for field [Total] to type [Decimal]")
.hasMessageContaining("Example Values:")
.hasAnyBulletsOfTextContaining("Row 3 [three-fifty-0]")
.hasAnyBulletsOfTextContaining("Row 4 [three-fifty-1]")
.hasAnyBulletsOfTextContaining("Row 5 [three-fifty-2]")
.doesNotHaveAnyBulletsOfTextContaining("three-fifty-101")
.hasStatus(Status.ERROR)
.hasCount(102);
}
/*******************************************************************************
**
*******************************************************************************/
@Test
void testRollupOfValidationErrors() throws QException
{
QContext.getQSession().withSecurityKeyValue(TestUtils.SECURITY_KEY_TYPE_STORE, 1);
///////////////////////////////////////////
// setup & run the bulk insert transform //
///////////////////////////////////////////
BulkInsertTransformStep bulkInsertTransformStep = new BulkInsertTransformStep();
RunBackendStepInput input = new RunBackendStepInput();
RunBackendStepOutput output = new RunBackendStepOutput();
Serializable[] emptyValues = new Serializable[0];
String tooLong = ".".repeat(201);
input.setTableName(TestUtils.TABLE_NAME_ORDER);
input.setStepName(StreamedETLWithFrontendProcess.STEP_NAME_VALIDATE);
input.setRecords(ListBuilder.of(
BulkLoadRecordUtils.addBackendDetailsAboutFileRows(new QRecord().withValue("shipToName", tooLong), new BulkLoadFileRow(emptyValues, 1)),
BulkLoadRecordUtils.addBackendDetailsAboutFileRows(new QRecord().withValue("shipToName", "OK").withValue("storeId", 1), new BulkLoadFileRow(emptyValues, 2))
));
/////////////////////////////////////////////////////////////////////
// add 102 records with no security key - which should be an error //
/////////////////////////////////////////////////////////////////////
for(int i = 0; i < 102; i++)
{
input.getRecords().add(BulkLoadRecordUtils.addBackendDetailsAboutFileRows(new QRecord(), new BulkLoadFileRow(emptyValues, 3 + i)));
}
bulkInsertTransformStep.preRun(input, output);
bulkInsertTransformStep.runOnePage(input, output);
ArrayList<ProcessSummaryLineInterface> processSummary = bulkInsertTransformStep.getProcessSummary(output, false);
ProcessSummaryAssert.assertThat(processSummary)
.hasLineWithMessageContaining("value for Ship To Name is too long")
.hasMessageContaining("Records:")
.doesNotHaveMessageContaining("Example Records:")
.hasAnyBulletsOfTextContaining("Row 1")
.hasStatus(Status.ERROR)
.hasCount(1);
ProcessSummaryAssert.assertThat(processSummary)
.hasLineWithMessageContaining("without a value in the field: Store Id")
.hasMessageContaining("Example Records:")
.hasAnyBulletsOfTextContaining("Row 1")
.hasAnyBulletsOfTextContaining("Row 3")
.hasAnyBulletsOfTextContaining("Row 4")
.doesNotHaveAnyBulletsOfTextContaining("Row 101")
.hasStatus(Status.ERROR)
.hasCount(103); // the 102, plus row 1.
ProcessSummaryAssert.assertThat(processSummary)
.hasLineWithMessageContaining("Order record will be inserted")
.hasStatus(Status.OK)
.hasCount(1);
}
}

View File

@ -70,6 +70,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.dashboard.QWidgetMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.fields.DisplayFormat;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
import com.kingsrook.qqq.backend.core.model.metadata.fields.ValueTooLongBehavior;
import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinOn;
import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinType;
import com.kingsrook.qqq.backend.core.model.metadata.joins.QJoinMetaData;
@ -662,7 +663,7 @@ public class TestUtils
.withField(new QFieldMetaData("createDate", QFieldType.DATE_TIME).withIsEditable(false))
.withField(new QFieldMetaData("modifyDate", QFieldType.DATE_TIME).withIsEditable(false))
.withField(new QFieldMetaData("orderNo", QFieldType.STRING))
.withField(new QFieldMetaData("shipToName", QFieldType.STRING))
.withField(new QFieldMetaData("shipToName", QFieldType.STRING).withMaxLength(200).withBehavior(ValueTooLongBehavior.ERROR))
.withField(new QFieldMetaData("orderDate", QFieldType.DATE))
.withField(new QFieldMetaData("storeId", QFieldType.INTEGER))
.withField(new QFieldMetaData("total", QFieldType.DECIMAL).withDisplayFormat(DisplayFormat.CURRENCY).withFieldSecurityLock(new FieldSecurityLock()