QRunIO/qqq
2
0
Fork 0
mirror of https://github.com/Kingsrook/qqq.git synced 2025-07-19 21:50:45 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add primary keys to process summary lines (and thus traces) for bulk load; better handling of errors and warnings also from bulk insert result step

Browse Source
This commit is contained in:
darin.kelkhoff
2025-04-25 16:05:54 -05:00
parent f81b257dd4
commit e5987238e6
5 changed files with 279 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

65
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertLoadStep.java
View File

@ -41,6 +41,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.LoadViaInsertStep;
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.ProcessSummaryProviderInterface;
import com.kingsrook.qqq.backend.core.processes.implementations.general.ProcessSummaryWarningsAndErrorsRollup;
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
@ -77,19 +78,77 @@ public class BulkInsertLoadStep extends LoadViaInsertStep implements ProcessSumm
QTableMetaData table = QContext.getQInstance().getTable(runBackendStepInput.getValueString("tableName"));
/////////////////////////////////////////////////////////////////////////////////////////////
// the transform step builds summary lines that it predicts will insert successfully. //
// but those lines don't have ids, which we'd like to have (e.g., for a process trace that //
// might link to the built record). also, it's possible that there was a fail that only //
// happened in the actual insert, so, basically, re-do the summary here //
/////////////////////////////////////////////////////////////////////////////////////////////
BulkInsertTransformStep transformStep = (BulkInsertTransformStep) getTransformStep();
ProcessSummaryLine okSummary = transformStep.okSummary;
okSummary.setCount(0);
okSummary.setPrimaryKeys(new ArrayList<>());
//////////////////////////////////////////////////////////////////////////////////////////////////////////
// but - since errors from the transform step don't even make it through to us here in the load step, //
// do re-use the ProcessSummaryWarningsAndErrorsRollup from transform step as follows: //
// clear out its warnings - we'll completely rebuild them here (with primary keys) //
// and add new error lines, e.g., in case of errors that only happened past the validation if possible. //
//////////////////////////////////////////////////////////////////////////////////////////////////////////
ProcessSummaryWarningsAndErrorsRollup processSummaryWarningsAndErrorsRollup = transformStep.processSummaryWarningsAndErrorsRollup;
processSummaryWarningsAndErrorsRollup.resetWarnings();
List<QRecord> insertedRecords = runBackendStepOutput.getRecords();
for(QRecord insertedRecord : insertedRecords)
{
if(CollectionUtils.nullSafeIsEmpty(insertedRecord.getErrors()))
Serializable primaryKey = insertedRecord.getValue(table.getPrimaryKeyField());
if(CollectionUtils.nullSafeIsEmpty(insertedRecord.getErrors()) && primaryKey != null)
{
/////////////////////////////////////////////////////////////////////////
// if the record had no errors, and we have a primary key for it, then //
// keep track of the range of primary keys (first and last) //
/////////////////////////////////////////////////////////////////////////
if(firstInsertedPrimaryKey == null)
{
firstInsertedPrimaryKey = insertedRecord.getValue(table.getPrimaryKeyField());
firstInsertedPrimaryKey = primaryKey;
}
lastInsertedPrimaryKey = insertedRecord.getValue(table.getPrimaryKeyField());
lastInsertedPrimaryKey = primaryKey;
if(!CollectionUtils.nullSafeIsEmpty(insertedRecord.getWarnings()))
{
/////////////////////////////////////////////////////////////////////////////
// if there were warnings on the inserted record, put it in a warning line //
/////////////////////////////////////////////////////////////////////////////
String message = insertedRecord.getWarnings().get(0).getMessage();
processSummaryWarningsAndErrorsRollup.addWarning(message, primaryKey);
}
else
{
////////////////////////////////////////////////////////////////////////
// if no warnings for the inserted record, then put it in the OK line //
////////////////////////////////////////////////////////////////////////
okSummary.incrementCountAndAddPrimaryKey(primaryKey);
}
}
else
{
//////////////////////////////////////////////////////////////////////
// else if there were errors or no primary key, build an error line //
//////////////////////////////////////////////////////////////////////
String message = "Failed to insert";
if(!CollectionUtils.nullSafeIsEmpty(insertedRecord.getErrors()))
{
//////////////////////////////////////////////////////////
// use the error message from the record if we have one //
//////////////////////////////////////////////////////////
message = insertedRecord.getErrors().get(0).getMessage();
}
processSummaryWarningsAndErrorsRollup.addError(message, primaryKey);
}
}
okSummary.pickMessage(true);
}

4
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTransformStep.java
View File

@ -75,9 +75,9 @@ import com.kingsrook.qqq.backend.core.utils.ValueUtils;
*******************************************************************************/
public class BulkInsertTransformStep extends AbstractTransformStep
{
private ProcessSummaryLine okSummary = new ProcessSummaryLine(Status.OK);
ProcessSummaryLine okSummary = new ProcessSummaryLine(Status.OK);
private ProcessSummaryWarningsAndErrorsRollup processSummaryWarningsAndErrorsRollup = ProcessSummaryWarningsAndErrorsRollup.build("inserted")
ProcessSummaryWarningsAndErrorsRollup processSummaryWarningsAndErrorsRollup = ProcessSummaryWarningsAndErrorsRollup.build("inserted")
.withDoReplaceSingletonCountLinesWithSuffixOnly(false);
private ListingHash<String, RowValue> errorToExampleRowValueMap = new ListingHash<>();

23
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/general/ProcessSummaryWarningsAndErrorsRollup.java
View File

@ -195,7 +195,7 @@ public class ProcessSummaryWarningsAndErrorsRollup
{
if(otherWarningsSummary == null)
{
otherWarningsSummary = new ProcessSummaryLine(Status.WARNING).withMessageSuffix("records had an other warning.");
otherWarningsSummary = buildOtherWarningsSummary();
}
processSummaryLine = otherWarningsSummary;
}
@ -214,6 +214,27 @@ public class ProcessSummaryWarningsAndErrorsRollup
/***************************************************************************
**
***************************************************************************/
private static ProcessSummaryLine buildOtherWarningsSummary()
{
return new ProcessSummaryLine(Status.WARNING).withMessageSuffix("records had an other warning.");
}
/***************************************************************************
**
***************************************************************************/
public void resetWarnings()
{
warningSummaries.clear();
otherWarningsSummary = buildOtherWarningsSummary();
}
/*******************************************************************************
** Wrapper around AlphaNumericComparator for ProcessSummaryLineInterface that
** extracts string messages out.