QRunIO/qqq
2
0
Fork 0
mirror of https://github.com/Kingsrook/qqq.git synced 2025-07-18 21:20:45 +00:00
Code Issues Packages Projects Releases Wiki Activity

Possibly run pre-insert-customizer in here, before checking UK values (in case customizer adjusts such fields). Add 'sample values' to UK error messaging.

Browse Source
This commit is contained in:
darin.kelkhoff
2023-10-16 08:22:41 -05:00
parent 3cfdf99b43
commit 14d0d18045
1 changed files with 155 additions and 70 deletions
Download Patch File Download Diff File Expand all files Collapse all files

225
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTransformStep.java
View File

@ -30,6 +30,10 @@ import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import com.kingsrook.qqq.backend.core.actions.customizers.AbstractPreInsertCustomizer;
import com.kingsrook.qqq.backend.core.actions.customizers.AbstractPreInsertCustomizer.WhenToRun;
import com.kingsrook.qqq.backend.core.actions.customizers.QCodeLoader;
import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizers;
import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
import com.kingsrook.qqq.backend.core.actions.tables.helpers.UniqueKeyHelper;
import com.kingsrook.qqq.backend.core.exceptions.QException;
@ -48,6 +52,7 @@ import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwith
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess;
import com.kingsrook.qqq.backend.core.processes.implementations.general.ProcessSummaryWarningsAndErrorsRollup;
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
import com.kingsrook.qqq.backend.core.utils.StringUtils;
/*******************************************************************************
@ -59,12 +64,34 @@ public class BulkInsertTransformStep extends AbstractTransformStep
private ProcessSummaryWarningsAndErrorsRollup processSummaryWarningsAndErrorsRollup = ProcessSummaryWarningsAndErrorsRollup.build("inserted");
private Map<UniqueKey, ProcessSummaryLine> ukErrorSummaries = new HashMap<>();
private Map<UniqueKey, ProcessSummaryLineWithUKSampleValues> ukErrorSummaries = new HashMap<>();
private QTableMetaData table;
private Map<UniqueKey, Set<List<Serializable>>> keysInThisFile = new HashMap<>();
private int rowsProcessed = 0;
/*******************************************************************************
**
*******************************************************************************/
private static class ProcessSummaryLineWithUKSampleValues extends ProcessSummaryLine
{
private List<String> sampleValues = new ArrayList<>();
/*******************************************************************************
**
*******************************************************************************/
public ProcessSummaryLineWithUKSampleValues(Status status)
{
super(status);
}
}
/*******************************************************************************
@ -89,14 +116,48 @@ public class BulkInsertTransformStep extends AbstractTransformStep
@Override
public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException
{
int rowsInThisPage = runBackendStepInput.getRecords().size();
QTableMetaData table = runBackendStepInput.getInstance().getTable(runBackendStepInput.getTableName());
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// set up an insert-input, which will be used as input to the pre-customizer as well as for additional validations //
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
InsertInput insertInput = new InsertInput();
insertInput.setInputSource(QInputSource.USER);
insertInput.setTableName(runBackendStepInput.getTableName());
insertInput.setRecords(runBackendStepInput.getRecords());
insertInput.setSkipUniqueKeyCheck(true);
//////////////////////////////////////////////////////////////////////
// load the pre-insert customizer and set it up, if there is one //
// then we'll run it based on its WhenToRun value //
// we do this, in case it needs to, for example, adjust values that //
// are part of a unique key //
//////////////////////////////////////////////////////////////////////
Optional<AbstractPreInsertCustomizer> preInsertCustomizer = QCodeLoader.getTableCustomizer(AbstractPreInsertCustomizer.class, table, TableCustomizers.PRE_INSERT_RECORD.getRole());
if(preInsertCustomizer.isPresent())
{
preInsertCustomizer.get().setInsertInput(insertInput);
preInsertCustomizer.get().setIsPreview(true);
AbstractPreInsertCustomizer.WhenToRun whenToRun = preInsertCustomizer.get().getWhenToRun();
if(WhenToRun.BEFORE_ALL_VALIDATIONS.equals(whenToRun) || WhenToRun.BEFORE_UNIQUE_KEY_CHECKS.equals(whenToRun))
{
List<QRecord> recordsAfterCustomizer = preInsertCustomizer.get().apply(runBackendStepInput.getRecords());
runBackendStepInput.setRecords(recordsAfterCustomizer);
///////////////////////////////////////////////////////////////////////////////////////
// todo - do we care if the customizer runs both now, and in the validation below? //
// right now we'll let it run both times, but maybe that should be protected against //
///////////////////////////////////////////////////////////////////////////////////////
}
}
Map<UniqueKey, Set<List<Serializable>>> existingKeys = new HashMap<>();
List<UniqueKey> uniqueKeys = CollectionUtils.nonNullList(table.getUniqueKeys());
for(UniqueKey uniqueKey : uniqueKeys)
{
existingKeys.put(uniqueKey, UniqueKeyHelper.getExistingKeys(null, table, runBackendStepInput.getRecords(), uniqueKey).keySet());
ukErrorSummaries.computeIfAbsent(uniqueKey, x -> new ProcessSummaryLine(Status.ERROR));
ukErrorSummaries.computeIfAbsent(uniqueKey, x -> new ProcessSummaryLineWithUKSampleValues(Status.ERROR));
}
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////
@ -105,7 +166,7 @@ public class BulkInsertTransformStep extends AbstractTransformStep
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////
if(runBackendStepInput.getStepName().equals(StreamedETLWithFrontendProcess.STEP_NAME_VALIDATE))
{
runBackendStepInput.getAsyncJobCallback().updateStatus("Processing row " + "%,d".formatted(okSummary.getCount()));
runBackendStepInput.getAsyncJobCallback().updateStatus("Processing row " + "%,d".formatted(rowsProcessed + 1));
}
else if(runBackendStepInput.getStepName().equals(StreamedETLWithFrontendProcess.STEP_NAME_EXECUTE))
{
@ -123,70 +184,13 @@ public class BulkInsertTransformStep extends AbstractTransformStep
// Note, we want to do our own UK checking here, even though InsertAction also tries to do it, because InsertAction //
// will only be getting the records in pages, but in here, we'll track UK's across pages!! //
//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
////////////////////////////////////////////////////
// if there are no UK's, proceed with all records //
////////////////////////////////////////////////////
List<QRecord> recordsWithoutUkErrors = new ArrayList<>();
if(existingKeys.isEmpty())
{
recordsWithoutUkErrors.addAll(runBackendStepInput.getRecords());
}
else
{
/////////////////////////////////////////////////////////////
// else, only proceed with records that don't violate a UK //
/////////////////////////////////////////////////////////////
for(UniqueKey uniqueKey : uniqueKeys)
{
keysInThisFile.computeIfAbsent(uniqueKey, x -> new HashSet<>());
}
///////////////////////////////////////////////////////////////////////////
// else, get each records keys and see if it already exists or not //
// also, build a set of keys we've seen (within this page (or overall?)) //
///////////////////////////////////////////////////////////////////////////
for(QRecord record : runBackendStepInput.getRecords())
{
//////////////////////////////////////////////////////////
// check if this record violates any of the unique keys //
//////////////////////////////////////////////////////////
boolean foundDupe = false;
for(UniqueKey uniqueKey : uniqueKeys)
{
Optional<List<Serializable>> keyValues = UniqueKeyHelper.getKeyValues(table, uniqueKey, record);
if(keyValues.isPresent() && (existingKeys.get(uniqueKey).contains(keyValues.get()) || keysInThisFile.get(uniqueKey).contains(keyValues.get())))
{
ukErrorSummaries.get(uniqueKey).incrementCount();
foundDupe = true;
break;
}
}
///////////////////////////////////////////////////////////////////////////////
// if this record doesn't violate any uk's, then we can add it to the output //
///////////////////////////////////////////////////////////////////////////////
if(!foundDupe)
{
for(UniqueKey uniqueKey : uniqueKeys)
{
Optional<List<Serializable>> keyValues = UniqueKeyHelper.getKeyValues(table, uniqueKey, record);
keyValues.ifPresent(kv -> keysInThisFile.get(uniqueKey).add(kv));
}
recordsWithoutUkErrors.add(record);
}
}
}
List<QRecord> recordsWithoutUkErrors = getRecordsWithoutUniqueKeyErrors(runBackendStepInput, existingKeys, uniqueKeys, table);
/////////////////////////////////////////////////////////////////////////////////
// run all validation from the insert action - in Preview mode (boolean param) //
/////////////////////////////////////////////////////////////////////////////////
InsertAction insertAction = new InsertAction();
InsertInput insertInput = new InsertInput();
insertInput.setInputSource(QInputSource.USER);
insertInput.setTableName(runBackendStepInput.getTableName());
insertInput.setRecords(recordsWithoutUkErrors);
insertInput.setSkipUniqueKeyCheck(true);
InsertAction insertAction = new InsertAction();
insertAction.performValidations(insertInput, true);
List<QRecord> validationResultRecords = insertInput.getRecords();
@ -215,6 +219,85 @@ public class BulkInsertTransformStep extends AbstractTransformStep
}
runBackendStepOutput.setRecords(outputRecords);
this.rowsProcessed += rowsInThisPage;
}
/*******************************************************************************
**
*******************************************************************************/
private List<QRecord> getRecordsWithoutUniqueKeyErrors(RunBackendStepInput runBackendStepInput, Map<UniqueKey, Set<List<Serializable>>> existingKeys, List<UniqueKey> uniqueKeys, QTableMetaData table)
{
////////////////////////////////////////////////////
// if there are no UK's, proceed with all records //
////////////////////////////////////////////////////
List<QRecord> recordsWithoutUkErrors = new ArrayList<>();
if(existingKeys.isEmpty())
{
recordsWithoutUkErrors.addAll(runBackendStepInput.getRecords());
}
else
{
/////////////////////////////////////////////////////////////
// else, only proceed with records that don't violate a UK //
/////////////////////////////////////////////////////////////
for(UniqueKey uniqueKey : uniqueKeys)
{
keysInThisFile.computeIfAbsent(uniqueKey, x -> new HashSet<>());
}
///////////////////////////////////////////////////////////////////////////
// else, get each records keys and see if it already exists or not //
// also, build a set of keys we've seen (within this page (or overall?)) //
///////////////////////////////////////////////////////////////////////////
for(QRecord record : runBackendStepInput.getRecords())
{
if(CollectionUtils.nullSafeHasContents(record.getErrors()))
{
///////////////////////////////////////////////////
// skip any records that may already be in error //
///////////////////////////////////////////////////
recordsWithoutUkErrors.add(record);
continue;
}
//////////////////////////////////////////////////////////
// check if this record violates any of the unique keys //
//////////////////////////////////////////////////////////
boolean foundDupe = false;
for(UniqueKey uniqueKey : uniqueKeys)
{
Optional<List<Serializable>> keyValues = UniqueKeyHelper.getKeyValues(table, uniqueKey, record);
if(keyValues.isPresent() && (existingKeys.get(uniqueKey).contains(keyValues.get()) || keysInThisFile.get(uniqueKey).contains(keyValues.get())))
{
ProcessSummaryLineWithUKSampleValues processSummaryLineWithUKSampleValues = ukErrorSummaries.get(uniqueKey);
processSummaryLineWithUKSampleValues.incrementCount();
if(processSummaryLineWithUKSampleValues.sampleValues.size() < 3)
{
processSummaryLineWithUKSampleValues.sampleValues.add(keyValues.get().toString());
}
foundDupe = true;
break;
}
}
///////////////////////////////////////////////////////////////////////////////
// if this record doesn't violate any uk's, then we can add it to the output //
///////////////////////////////////////////////////////////////////////////////
if(!foundDupe)
{
for(UniqueKey uniqueKey : uniqueKeys)
{
Optional<List<Serializable>> keyValues = UniqueKeyHelper.getKeyValues(table, uniqueKey, record);
keyValues.ifPresent(kv -> keysInThisFile.get(uniqueKey).add(kv));
}
recordsWithoutUkErrors.add(record);
}
}
}
return recordsWithoutUkErrors;
}
@ -236,17 +319,19 @@ public class BulkInsertTransformStep extends AbstractTransformStep
okSummary.pickMessage(isForResultScreen);
okSummary.addSelfToListIfAnyCount(rs);
for(Map.Entry<UniqueKey, ProcessSummaryLine> entry : ukErrorSummaries.entrySet())
for(Map.Entry<UniqueKey, ProcessSummaryLineWithUKSampleValues> entry : ukErrorSummaries.entrySet())
{
UniqueKey uniqueKey = entry.getKey();
ProcessSummaryLine ukErrorSummary = entry.getValue();
String ukErrorSuffix = " inserted, because of duplicate values in a unique key (" + uniqueKey.getDescription(table) + ")";
UniqueKey uniqueKey = entry.getKey();
ProcessSummaryLineWithUKSampleValues ukErrorSummary = entry.getValue();
ukErrorSummary
.withSingularFutureMessage(tableLabel + " record will not be" + ukErrorSuffix)
.withPluralFutureMessage(tableLabel + " records will not be" + ukErrorSuffix)
.withSingularPastMessage(tableLabel + " record was not" + ukErrorSuffix)
.withPluralPastMessage(tableLabel + " records were not" + ukErrorSuffix);
.withMessageSuffix(" inserted, because of duplicate values in a unique key on the fields (" + uniqueKey.getDescription(table) + "), with values such as: "
+ StringUtils.joinWithCommasAndAnd(ukErrorSummary.sampleValues))
.withSingularFutureMessage(tableLabel + " record will not be")
.withPluralFutureMessage(tableLabel + " records will not be")
.withSingularPastMessage(tableLabel + " record was not")
.withPluralPastMessage(tableLabel + " records were not");
ukErrorSummary.addSelfToListIfAnyCount(rs);
}