QRunIO/qqq
2
0
Fork 0
mirror of https://github.com/Kingsrook/qqq.git synced 2025-07-18 13:10:44 +00:00
Code Issues Packages Projects Releases Wiki Activity

Let StreamedETLWithFrontendProcesses have different transaction levels

Browse Source
This commit is contained in:
darin.kelkhoff
2023-02-27 10:27:11 -06:00
parent ea731bac5c
commit 4f3c03de1a
2 changed files with 142 additions and 57 deletions
Download Patch File Download Diff File Expand all files Collapse all files

50
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLExecuteStep.java
View File

@ -101,9 +101,16 @@ public class StreamedETLExecuteStep extends BaseStreamedETLStep implements Backe
transformStep.preRun(runBackendStepInput, runBackendStepOutput);
loadStep.preRun(runBackendStepInput, runBackendStepOutput);
/////////////////////////////////////////////////////////////////////////////
// open a transaction for the whole process, if that's the requested level //
/////////////////////////////////////////////////////////////////////////////
boolean doProcessLevelTransaction = StreamedETLWithFrontendProcess.TRANSACTION_LEVEL_PROCESS.equals(runBackendStepInput.getValueString(StreamedETLWithFrontendProcess.FIELD_TRANSACTION_LEVEL));
if(doProcessLevelTransaction)
{
transaction = loadStep.openTransaction(runBackendStepInput);
loadStep.setTransaction(transaction);
transformStep.setTransaction(transaction);
}
List<QRecord> loadedRecordList = new ArrayList<>();
int recordCount = new AsyncRecordPipeLoop().run("StreamedETL>Execute>ExtractStep", null, recordPipe, (status) ->
@ -145,10 +152,10 @@ public class StreamedETLExecuteStep extends BaseStreamedETLStep implements Backe
//////////////////////////////////////////////////////////////////////////////
runBackendStepOutput.addValue(RunProcessAction.BASEPULL_READY_TO_UPDATE_TIMESTAMP_FIELD, true);
/////////////////////
// commit the work //
/////////////////////
if(transaction.isPresent())
////////////////////////////////////////////////////////
// commit the work at the process level if applicable //
////////////////////////////////////////////////////////
if(doProcessLevelTransaction && transaction.isPresent())
{
transaction.get().commit();
}
@ -182,6 +189,20 @@ public class StreamedETLExecuteStep extends BaseStreamedETLStep implements Backe
**
*******************************************************************************/
private int consumeRecordsFromPipe(RecordPipe recordPipe, AbstractTransformStep transformStep, AbstractLoadStep loadStep, RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput, List<QRecord> loadedRecordList) throws QException
{
/////////////////////////////////////////////////////////////////////////////
// open a transaction for the whole process, if that's the requested level //
/////////////////////////////////////////////////////////////////////////////
Optional<QBackendTransaction> transaction = Optional.empty();
boolean doPageLevelTransaction = StreamedETLWithFrontendProcess.TRANSACTION_LEVEL_PAGE.equals(runBackendStepInput.getValueString(StreamedETLWithFrontendProcess.FIELD_TRANSACTION_LEVEL));
if(doPageLevelTransaction)
{
transaction = loadStep.openTransaction(runBackendStepInput);
loadStep.setTransaction(transaction);
transformStep.setTransaction(transaction);
}
try
{
Integer totalRows = runBackendStepInput.getValueInteger(StreamedETLWithFrontendProcess.FIELD_RECORD_COUNT);
if(totalRows != null)
@ -238,8 +259,29 @@ public class StreamedETLExecuteStep extends BaseStreamedETLStep implements Backe
}
runBackendStepOutput.setAuditInputList(null);
if(doPageLevelTransaction && transaction.isPresent())
{
transaction.get().commit();
}
currentRowCount += qRecords.size();
return (qRecords.size());
}
catch(Exception e)
{
if(doPageLevelTransaction && transaction.isPresent())
{
transaction.get().rollback();
}
throw (e);
}
finally
{
if(doPageLevelTransaction && transaction.isPresent())
{
transaction.get().close();
}
}
}
}

43
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLWithFrontendProcess.java
View File

@ -93,6 +93,11 @@ public class StreamedETLWithFrontendProcess
public static final String DEFAULT_PREVIEW_MESSAGE_FOR_DELETE = "This is a preview of the records that will be deleted.";
public static final String FIELD_PREVIEW_MESSAGE = "previewMessage";
public static final String FIELD_TRANSACTION_LEVEL = "transactionLevel";
public static final String TRANSACTION_LEVEL_AUTO_COMMIT = "autoCommit";
public static final String TRANSACTION_LEVEL_PAGE = "page";
public static final String TRANSACTION_LEVEL_PROCESS = "process";
/*******************************************************************************
@ -109,6 +114,7 @@ public class StreamedETLWithFrontendProcess
Map<String, Serializable> defaultFieldValues = new HashMap<>();
defaultFieldValues.put(FIELD_SOURCE_TABLE, sourceTableName);
defaultFieldValues.put(FIELD_DESTINATION_TABLE, destinationTableName);
defaultFieldValues.put(FIELD_TRANSACTION_LEVEL, TRANSACTION_LEVEL_PROCESS);
return defineProcessMetaData(extractStepClass, transformStepClass, loadStepClass, defaultFieldValues);
}
@ -141,6 +147,7 @@ public class StreamedETLWithFrontendProcess
.withField(new QFieldMetaData(FIELD_EXTRACT_CODE, QFieldType.STRING).withDefaultValue(extractStepClass == null ? null : new QCodeReference(extractStepClass)))
.withField(new QFieldMetaData(FIELD_TRANSFORM_CODE, QFieldType.STRING).withDefaultValue(transformStepClass == null ? null : new QCodeReference(transformStepClass)))
.withField(new QFieldMetaData(FIELD_PREVIEW_MESSAGE, QFieldType.STRING).withDefaultValue(defaultFieldValues.getOrDefault(FIELD_PREVIEW_MESSAGE, DEFAULT_PREVIEW_MESSAGE_FOR_INSERT)))
.withField(new QFieldMetaData(FIELD_TRANSACTION_LEVEL, QFieldType.STRING).withDefaultValue(defaultFieldValues.getOrDefault(FIELD_TRANSACTION_LEVEL, TRANSACTION_LEVEL_PROCESS)))
);
QFrontendStepMetaData reviewStep = new QFrontendStepMetaData()
@ -278,6 +285,42 @@ public class StreamedETLWithFrontendProcess
/*******************************************************************************
** Fluent setter to set transaction level to auto-commit
**
*******************************************************************************/
public Builder withTransactionLevelAutoCommit()
{
setInputFieldDefaultValue(FIELD_TRANSACTION_LEVEL, TRANSACTION_LEVEL_AUTO_COMMIT);
return (this);
}
/*******************************************************************************
** Fluent setter to set transaction level to page
**
*******************************************************************************/
public Builder withTransactionLevelPage()
{
setInputFieldDefaultValue(FIELD_TRANSACTION_LEVEL, TRANSACTION_LEVEL_PAGE);
return (this);
}
/*******************************************************************************
** Fluent setter to set transaction level to process
**
*******************************************************************************/
public Builder withTransactionLevelProcess()
{
setInputFieldDefaultValue(FIELD_TRANSACTION_LEVEL, TRANSACTION_LEVEL_PROCESS);
return (this);
}
/*******************************************************************************
** Fluent setter for doFullValidation
**