QRunIO/qqq
2
0
Fork 0
mirror of https://github.com/Kingsrook/qqq.git synced 2025-07-18 05:01:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add 'unspecifiedError' for higher-level exceptions; add primaryKeys in summary lines

Browse Source
This commit is contained in:
darin.kelkhoff
2023-09-29 17:07:24 -05:00
parent dd57a327dd
commit e89093f339
1 changed files with 71 additions and 42 deletions
Download Patch File Download Diff File Expand all files Collapse all files

113
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/tablesync/AbstractTableSyncTransformStep.java
View File

@ -35,6 +35,7 @@ import java.util.Set;
import java.util.stream.Collectors; import java.util.stream.Collectors;
import com.kingsrook.qqq.backend.core.actions.tables.QueryAction; import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
import com.kingsrook.qqq.backend.core.actions.values.QPossibleValueTranslator; import com.kingsrook.qqq.backend.core.actions.values.QPossibleValueTranslator;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.logging.QLogger; import com.kingsrook.qqq.backend.core.logging.QLogger;
import com.kingsrook.qqq.backend.core.model.actions.audits.AuditSingleInput; import com.kingsrook.qqq.backend.core.model.actions.audits.AuditSingleInput;
@ -72,20 +73,23 @@ public abstract class AbstractTableSyncTransformStep extends AbstractTransformSt
{ {
private static final QLogger LOG = QLogger.getLogger(AbstractTableSyncTransformStep.class); private static final QLogger LOG = QLogger.getLogger(AbstractTableSyncTransformStep.class);
private ProcessSummaryLine okToInsert = StandardProcessSummaryLineProducer.getOkToInsertLine(); private ProcessSummaryLine okToInsert = StandardProcessSummaryLineProducer.getOkToInsertLine();
private ProcessSummaryLine okToUpdate = StandardProcessSummaryLineProducer.getOkToUpdateLine(); private ProcessSummaryLine okToUpdate = StandardProcessSummaryLineProducer.getOkToUpdateLine();
private ProcessSummaryLine willNotInsert = new ProcessSummaryLine(Status.INFO)
private ProcessSummaryLine willNotInsert = new ProcessSummaryLine(Status.INFO)
.withMessageSuffix("because of this process' configuration.") .withMessageSuffix("because of this process' configuration.")
.withSingularFutureMessage("will not be inserted ") .withSingularFutureMessage("will not be inserted ")
.withPluralFutureMessage("will not be inserted ") .withPluralFutureMessage("will not be inserted ")
.withSingularPastMessage("was not inserted ") .withSingularPastMessage("was not inserted ")
.withPluralPastMessage("were not inserted "); .withPluralPastMessage("were not inserted ");
private ProcessSummaryLine willNotUpdate = new ProcessSummaryLine(Status.INFO)
private ProcessSummaryLine willNotUpdate = new ProcessSummaryLine(Status.INFO)
.withMessageSuffix("because of this process' configuration.") .withMessageSuffix("because of this process' configuration.")
.withSingularFutureMessage("will not be updated ") .withSingularFutureMessage("will not be updated ")
.withPluralFutureMessage("will not be updated ") .withPluralFutureMessage("will not be updated ")
.withSingularPastMessage("was not updated ") .withSingularPastMessage("was not updated ")
.withPluralPastMessage("were not updated "); .withPluralPastMessage("were not updated ");
private ProcessSummaryLine errorMissingKeyField = new ProcessSummaryLine(Status.ERROR) private ProcessSummaryLine errorMissingKeyField = new ProcessSummaryLine(Status.ERROR)
.withMessageSuffix("missing a value for the key field.") .withMessageSuffix("missing a value for the key field.")
.withSingularFutureMessage("will not be synced, because it is ") .withSingularFutureMessage("will not be synced, because it is ")
@ -93,6 +97,13 @@ public abstract class AbstractTableSyncTransformStep extends AbstractTransformSt
.withSingularPastMessage("was not synced, because it is ") .withSingularPastMessage("was not synced, because it is ")
.withPluralPastMessage("were not synced, because they are "); .withPluralPastMessage("were not synced, because they are ");
private ProcessSummaryLine unspecifiedError = new ProcessSummaryLine(Status.ERROR)
.withMessageSuffix("of an unexpected error: ")
.withSingularFutureMessage("will not be synced, ")
.withPluralFutureMessage("will not be synced, ")
.withSingularPastMessage("was not synced, ")
.withPluralPastMessage("were not synced, ");
protected RunBackendStepInput runBackendStepInput = null; protected RunBackendStepInput runBackendStepInput = null;
protected RunBackendStepOutput runBackendStepOutput = null; protected RunBackendStepOutput runBackendStepOutput = null;
protected RecordLookupHelper recordLookupHelper = null; protected RecordLookupHelper recordLookupHelper = null;
@ -107,8 +118,17 @@ public abstract class AbstractTableSyncTransformStep extends AbstractTransformSt
@Override @Override
public ArrayList<ProcessSummaryLineInterface> getProcessSummary(RunBackendStepOutput runBackendStepOutput, boolean isForResultScreen) public ArrayList<ProcessSummaryLineInterface> getProcessSummary(RunBackendStepOutput runBackendStepOutput, boolean isForResultScreen)
{ {
ArrayList<ProcessSummaryLineInterface> processSummaryLineList = StandardProcessSummaryLineProducer.toArrayList(okToInsert, okToUpdate, errorMissingKeyField, willNotInsert, willNotUpdate); return StandardProcessSummaryLineProducer.toArrayList(okToInsert, okToUpdate, errorMissingKeyField, unspecifiedError, willNotInsert, willNotUpdate);
return (processSummaryLineList); }
/*******************************************************************************
**
*******************************************************************************/
protected ArrayList<ProcessSummaryLineInterface> getErrorProcessSummaryLines(RunBackendStepOutput runBackendStepOutput, boolean isForResultScreen)
{
return StandardProcessSummaryLineProducer.toArrayList(errorMissingKeyField, unspecifiedError);
} }
@ -237,7 +257,8 @@ public abstract class AbstractTableSyncTransformStep extends AbstractTransformSt
Set<Serializable> processedSourceKeys = new HashSet<>(); Set<Serializable> processedSourceKeys = new HashSet<>();
for(QRecord sourceRecord : runBackendStepInput.getRecords()) for(QRecord sourceRecord : runBackendStepInput.getRecords())
{ {
Serializable sourceKeyValue = sourceRecord.getValue(sourceTableKeyField); Serializable sourcePrimaryKey = sourceRecord.getValue(QContext.getQInstance().getTable(config.sourceTable).getPrimaryKeyField());
Serializable sourceKeyValue = sourceRecord.getValue(sourceTableKeyField);
if(processedSourceKeys.contains(sourceKeyValue)) if(processedSourceKeys.contains(sourceKeyValue))
{ {
LOG.info("Skipping duplicated source-key within page", logPair("key", sourceKeyValue)); LOG.info("Skipping duplicated source-key within page", logPair("key", sourceKeyValue));
@ -247,7 +268,7 @@ public abstract class AbstractTableSyncTransformStep extends AbstractTransformSt
if(sourceKeyValue == null || "".equals(sourceKeyValue)) if(sourceKeyValue == null || "".equals(sourceKeyValue))
{ {
errorMissingKeyField.incrementCount(); errorMissingKeyField.incrementCountAndAddPrimaryKey(sourcePrimaryKey);
try try
{ {
@ -266,48 +287,56 @@ public abstract class AbstractTableSyncTransformStep extends AbstractTransformSt
////////////////////////////////////////////////////////////// //////////////////////////////////////////////////////////////
// look for the existing record, to determine insert/update // // look for the existing record, to determine insert/update //
////////////////////////////////////////////////////////////// //////////////////////////////////////////////////////////////
QRecord existingRecord = getExistingRecord(existingRecordsByForeignKey, destinationForeignKeyField, sourceKeyValue); try
{
QRecord existingRecord = getExistingRecord(existingRecordsByForeignKey, destinationForeignKeyField, sourceKeyValue);
QRecord recordToStore; QRecord recordToStore;
if(existingRecord != null && config.performUpdates) if(existingRecord != null && config.performUpdates)
{
recordToStore = existingRecord;
}
else if(existingRecord == null && config.performInserts)
{
recordToStore = new QRecord();
}
else
{
if(existingRecord != null)
{ {
LOG.info("Skipping storing existing record because this sync process is set to not perform updates"); recordToStore = existingRecord;
willNotInsert.incrementCount(); }
else if(existingRecord == null && config.performInserts)
{
recordToStore = new QRecord();
} }
else else
{ {
LOG.info("Skipping storing new record because this sync process is set to not perform inserts"); if(existingRecord != null)
willNotUpdate.incrementCount(); {
LOG.info("Skipping storing existing record because this sync process is set to not perform updates");
willNotInsert.incrementCountAndAddPrimaryKey(sourcePrimaryKey);
}
else
{
LOG.info("Skipping storing new record because this sync process is set to not perform inserts");
willNotUpdate.incrementCountAndAddPrimaryKey(sourcePrimaryKey);
}
continue;
}
//////////////////////////////////////////////////////////////////////////////////
// if we received a record to store add to the output records and summary lines //
//////////////////////////////////////////////////////////////////////////////////
recordToStore = populateRecordToStore(runBackendStepInput, recordToStore, sourceRecord);
if(recordToStore != null)
{
if(existingRecord != null)
{
okToUpdate.incrementCountAndAddPrimaryKey(sourcePrimaryKey);
}
else
{
okToInsert.incrementCountAndAddPrimaryKey(sourcePrimaryKey);
}
runBackendStepOutput.addRecord(recordToStore);
} }
continue;
} }
catch(Exception e)
//////////////////////////////////////////////////////////////////////////////////
// if we received a record to store add to the output records and summary lines //
//////////////////////////////////////////////////////////////////////////////////
recordToStore = populateRecordToStore(runBackendStepInput, recordToStore, sourceRecord);
if(recordToStore != null)
{ {
if(existingRecord != null) unspecifiedError.incrementCountAndAddPrimaryKey(sourcePrimaryKey);
{ unspecifiedError.setMessageSuffix(unspecifiedError.getMessageSuffix() + e.getMessage());
okToUpdate.incrementCount();
}
else
{
okToInsert.incrementCount();
}
runBackendStepOutput.addRecord(recordToStore);
} }
} }