QRunIO/qqq
2
0
Fork 0
mirror of https://github.com/Kingsrook/qqq.git synced 2025-07-18 05:01:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

CE-1727 - Introduce concept of stepFlow to processes - LINEAR (the previous), and STATE_MACHINE (designed to be more flexible)

Browse Source
This commit is contained in:
darin.kelkhoff
2024-09-20 09:57:02 -05:00
parent c18aa44010
commit e4bef88406
7 changed files with 1015 additions and 97 deletions
Download Patch File Download Diff File Expand all files Collapse all files

381
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/processes/RunProcessAction.java
View File

@ -28,6 +28,7 @@ import java.time.temporal.ChronoUnit;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import java.util.UUID;
import com.kingsrook.qqq.backend.core.actions.ActionHelper;
@ -58,6 +59,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaD
import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendComponentMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendStepMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QStateMachineStep;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QStepMetaData;
import com.kingsrook.qqq.backend.core.model.session.QSession;
import com.kingsrook.qqq.backend.core.processes.implementations.basepull.BasepullConfiguration;
@ -133,90 +135,11 @@ public class RunProcessAction
try
{
String lastStepName = runProcessInput.getStartAfterStep();
STEP_LOOP:
while(true)
switch(Objects.requireNonNull(process.getStepFlow(), "Process [" + process.getName() + "] has a null stepFlow."))
{
///////////////////////////////////////////////////////////////////////////////////////////////////////
// always refresh the step list - as any step that runs can modify it (in the process state). //
// this is why we don't do a loop over the step list - as we'd get ConcurrentModificationExceptions. //
///////////////////////////////////////////////////////////////////////////////////////////////////////
List<QStepMetaData> stepList = getAvailableStepList(processState, process, lastStepName);
if(stepList.isEmpty())
{
break;
}
QStepMetaData step = stepList.get(0);
lastStepName = step.getName();
if(step instanceof QFrontendStepMetaData frontendStep)
{
////////////////////////////////////////////////////////////////
// Handle what to do with frontend steps, per request setting //
////////////////////////////////////////////////////////////////
switch(runProcessInput.getFrontendStepBehavior())
{
case BREAK ->
{
LOG.trace("Breaking process [" + process.getName() + "] at frontend step (as requested by caller): " + step.getName());
processFrontendStepFieldDefaultValues(processState, frontendStep);
processFrontendComponents(processState, frontendStep);
processState.setNextStepName(step.getName());
break STEP_LOOP;
}
case SKIP ->
{
LOG.trace("Skipping frontend step [" + step.getName() + "] in process [" + process.getName() + "] (as requested by caller)");
//////////////////////////////////////////////////////////////////////
// much less error prone in case this code changes in the future... //
//////////////////////////////////////////////////////////////////////
// noinspection UnnecessaryContinue
continue;
}
case FAIL ->
{
LOG.trace("Throwing error for frontend step [" + step.getName() + "] in process [" + process.getName() + "] (as requested by caller)");
throw (new QException("Failing process at step " + step.getName() + " (as requested, to fail on frontend steps)"));
}
default -> throw new IllegalStateException("Unexpected value: " + runProcessInput.getFrontendStepBehavior());
}
}
else if(step instanceof QBackendStepMetaData backendStepMetaData)
{
///////////////////////
// Run backend steps //
///////////////////////
LOG.debug("Running backend step [" + step.getName() + "] in process [" + process.getName() + "]");
RunBackendStepOutput runBackendStepOutput = runBackendStep(runProcessInput, process, runProcessOutput, stateKey, backendStepMetaData, process, processState);
/////////////////////////////////////////////////////////////////////////////////////////
// if the step returned an override lastStepName, use that to determine how we proceed //
/////////////////////////////////////////////////////////////////////////////////////////
if(runBackendStepOutput.getOverrideLastStepName() != null)
{
LOG.debug("Process step [" + lastStepName + "] returned an overrideLastStepName [" + runBackendStepOutput.getOverrideLastStepName() + "]!");
lastStepName = runBackendStepOutput.getOverrideLastStepName();
}
/////////////////////////////////////////////////////////////////////////////////////////////
// similarly, if the step produced an updatedFrontendStepList, propagate that data outward //
/////////////////////////////////////////////////////////////////////////////////////////////
if(runBackendStepOutput.getUpdatedFrontendStepList() != null)
{
LOG.debug("Process step [" + lastStepName + "] generated an updatedFrontendStepList [" + runBackendStepOutput.getUpdatedFrontendStepList().stream().map(s -> s.getName()).toList() + "]!");
runProcessOutput.setUpdatedFrontendStepList(runBackendStepOutput.getUpdatedFrontendStepList());
}
}
else
{
//////////////////////////////////////////////////
// in case we have a different step type, throw //
//////////////////////////////////////////////////
throw (new QException("Unsure how to run a step of type: " + step.getClass().getName()));
}
case LINEAR -> runLinearStepLoop(process, processState, stateKey, runProcessInput, runProcessOutput);
case STATE_MACHINE -> runStateMachineStep(runProcessInput.getStartAfterStep(), process, processState, stateKey, runProcessInput, runProcessOutput, 0);
default -> throw (new QException("Unhandled process step flow: " + process.getStepFlow()));
}
///////////////////////////////////////////////////////////////////////////
@ -258,6 +181,270 @@ public class RunProcessAction
/***************************************************************************
**
***************************************************************************/
private void runLinearStepLoop(QProcessMetaData process, ProcessState processState, UUIDAndTypeStateKey stateKey, RunProcessInput runProcessInput, RunProcessOutput runProcessOutput) throws Exception
{
String lastStepName = runProcessInput.getStartAfterStep();
while(true)
{
///////////////////////////////////////////////////////////////////////////////////////////////////////
// always refresh the step list - as any step that runs can modify it (in the process state). //
// this is why we don't do a loop over the step list - as we'd get ConcurrentModificationExceptions. //
///////////////////////////////////////////////////////////////////////////////////////////////////////
List<QStepMetaData> stepList = getAvailableStepList(processState, process, lastStepName);
if(stepList.isEmpty())
{
break;
}
QStepMetaData step = stepList.get(0);
lastStepName = step.getName();
if(step instanceof QFrontendStepMetaData frontendStep)
{
LoopTodo loopTodo = prepareForFrontendStep(runProcessInput, process, frontendStep, processState);
if(loopTodo == LoopTodo.BREAK)
{
break;
}
}
else if(step instanceof QBackendStepMetaData backendStepMetaData)
{
RunBackendStepOutput runBackendStepOutput = runBackendStep(process, processState, stateKey, runProcessInput, runProcessOutput, backendStepMetaData, step);
/////////////////////////////////////////////////////////////////////////////////////////
// if the step returned an override lastStepName, use that to determine how we proceed //
/////////////////////////////////////////////////////////////////////////////////////////
if(runBackendStepOutput.getOverrideLastStepName() != null)
{
LOG.debug("Process step [" + lastStepName + "] returned an overrideLastStepName [" + runBackendStepOutput.getOverrideLastStepName() + "]!");
lastStepName = runBackendStepOutput.getOverrideLastStepName();
}
}
else
{
//////////////////////////////////////////////////
// in case we have a different step type, throw //
//////////////////////////////////////////////////
throw (new QException("Unsure how to run a step of type: " + step.getClass().getName()));
}
}
}
/***************************************************************************
**
***************************************************************************/
private enum LoopTodo
{
BREAK,
CONTINUE
}
/***************************************************************************
**
***************************************************************************/
private LoopTodo prepareForFrontendStep(RunProcessInput runProcessInput, QProcessMetaData process, QFrontendStepMetaData step, ProcessState processState) throws QException
{
////////////////////////////////////////////////////////////////
// Handle what to do with frontend steps, per request setting //
////////////////////////////////////////////////////////////////
switch(runProcessInput.getFrontendStepBehavior())
{
case BREAK ->
{
LOG.trace("Breaking process [" + process.getName() + "] at frontend step (as requested by caller): " + step.getName());
processFrontendStepFieldDefaultValues(processState, step);
processFrontendComponents(processState, step);
processState.setNextStepName(step.getName());
return LoopTodo.BREAK;
}
case SKIP ->
{
LOG.trace("Skipping frontend step [" + step.getName() + "] in process [" + process.getName() + "] (as requested by caller)");
return LoopTodo.CONTINUE;
}
case FAIL ->
{
LOG.trace("Throwing error for frontend step [" + step.getName() + "] in process [" + process.getName() + "] (as requested by caller)");
throw (new QException("Failing process at step " + step.getName() + " (as requested, to fail on frontend steps)"));
}
default -> throw new IllegalStateException("Unexpected value: " + runProcessInput.getFrontendStepBehavior());
}
}
/***************************************************************************
**
***************************************************************************/
private void runStateMachineStep(String lastStepName, QProcessMetaData process, ProcessState processState, UUIDAndTypeStateKey stateKey, RunProcessInput runProcessInput, RunProcessOutput runProcessOutput, int stackDepth) throws Exception
{
//////////////////////////////
// check for stack-overflow //
//////////////////////////////
Integer maxStateMachineProcessStepFlowStackDepth = Objects.requireNonNullElse(runProcessInput.getValueInteger("maxStateMachineProcessStepFlowStackDepth"), 20);
if(stackDepth > maxStateMachineProcessStepFlowStackDepth)
{
throw (new QException("StateMachine process recurred too many times (exceeded maxStateMachineProcessStepFlowStackDepth of " + maxStateMachineProcessStepFlowStackDepth + ")"));
}
//////////////////////////////////
// figure out what step to run: //
//////////////////////////////////
QStepMetaData step = null;
if(!StringUtils.hasContent(lastStepName))
{
////////////////////////////////////////////////////////////////////
// if no lastStepName is given, start at the process's first step //
////////////////////////////////////////////////////////////////////
if(CollectionUtils.nullSafeIsEmpty(process.getStepList()))
{
throw (new QException("Process [" + process.getName() + "] does not have a step list defined."));
}
step = process.getStepList().get(0);
}
else
{
/////////////////////////////////////
// else run the given lastStepName //
/////////////////////////////////////
processState.clearNextStepName();
step = process.getStep(lastStepName);
if(step == null)
{
throw (new QException("Could not find step by name [" + lastStepName + "]"));
}
}
/////////////////////////////////////////////////////////////////////////
// for the flow of: //
// we were on a frontend step (as a sub-step of a state machine step), //
// and now we're here to run that state-step's backend step - //
// find the state-machine step containing this frontend step. //
/////////////////////////////////////////////////////////////////////////
String skipSubStepsUntil = null;
if(step instanceof QFrontendStepMetaData frontendStepMetaData)
{
QStateMachineStep stateMachineStep = getStateMachineStepContainingSubStep(process, frontendStepMetaData.getName());
if(stateMachineStep == null)
{
throw (new QException("Could not find stateMachineStep that contains last-frontend step: " + frontendStepMetaData.getName()));
}
step = stateMachineStep;
//////////////////////////////////////////////////////////////////////////////////
// set this flag, to know to skip this frontend step in the sub-step loop below //
//////////////////////////////////////////////////////////////////////////////////
skipSubStepsUntil = frontendStepMetaData.getName();
}
if(!(step instanceof QStateMachineStep stateMachineStep))
{
throw (new QException("Have a non-stateMachineStep in a process using stateMachine flow... " + step.getClass().getName()));
}
///////////////////////
// run the sub-steps //
///////////////////////
boolean ranAnySubSteps = false;
for(QStepMetaData subStep : stateMachineStep.getSubSteps())
{
///////////////////////////////////////////////////////////////////////////////////////////////
// ok, well, skip them if this flag is set (and clear the flag once we've hit this sub-step) //
///////////////////////////////////////////////////////////////////////////////////////////////
if(skipSubStepsUntil != null)
{
if(skipSubStepsUntil.equals(subStep.getName()))
{
skipSubStepsUntil = null;
}
continue;
}
ranAnySubSteps = true;
if(subStep instanceof QFrontendStepMetaData frontendStep)
{
LoopTodo loopTodo = prepareForFrontendStep(runProcessInput, process, frontendStep, processState);
if(loopTodo == LoopTodo.BREAK)
{
return;
}
}
else if(subStep instanceof QBackendStepMetaData backendStepMetaData)
{
RunBackendStepOutput runBackendStepOutput = runBackendStep(process, processState, stateKey, runProcessInput, runProcessOutput, backendStepMetaData, step);
Optional<String> nextStepName = runBackendStepOutput.getProcessState().getNextStepName();
if(nextStepName.isEmpty() && StringUtils.hasContent(stateMachineStep.getDefaultNextStepName()))
{
nextStepName = Optional.of(stateMachineStep.getDefaultNextStepName());
}
if(nextStepName.isPresent())
{
//////////////////////////////////////////////////////////////////////////////////////////////////////
// if we've been given a next-step-name, go to that step now. //
// it might be a backend-only stateMachineStep, in which case, we should run that backend step now. //
// or it might be a frontend-then-backend step, in which case, we want to go to that frontend step. //
// if we weren't given a next-step-name, then we should stay in the same state - either to finish //
// its sub-steps, or, to fall out of the loop and end the process. //
//////////////////////////////////////////////////////////////////////////////////////////////////////
processState.clearNextStepName();
runStateMachineStep(nextStepName.get(), process, processState, stateKey, runProcessInput, runProcessOutput, stackDepth + 1);
return;
}
}
else
{
//////////////////////////////////////////////////
// in case we have a different step type, throw //
//////////////////////////////////////////////////
throw (new QException("Unsure how to run a step of type: " + step.getClass().getName()));
}
}
if(!ranAnySubSteps)
{
if(StringUtils.hasContent(stateMachineStep.getDefaultNextStepName()))
{
runStateMachineStep(stateMachineStep.getDefaultNextStepName(), process, processState, stateKey, runProcessInput, runProcessOutput, stackDepth + 1);
}
}
}
/*******************************************************************************
**
*******************************************************************************/
public QStateMachineStep getStateMachineStepContainingSubStep(QProcessMetaData process, String stepName)
{
for(QStepMetaData step : process.getAllSteps().values())
{
if(step instanceof QStateMachineStep stateMachineStep)
{
for(QStepMetaData subStep : stateMachineStep.getSubSteps())
{
if(subStep.getName().equals(stepName))
{
return (stateMachineStep);
}
}
}
}
return (null);
}
/*******************************************************************************
**
*******************************************************************************/
@ -355,16 +542,40 @@ public class RunProcessAction
}
ProcessState processState = optionalProcessState.get();
processState.clearNextStepName();
return processState;
}
/***************************************************************************
**
***************************************************************************/
private RunBackendStepOutput runBackendStep(QProcessMetaData process, ProcessState processState, UUIDAndTypeStateKey stateKey, RunProcessInput runProcessInput, RunProcessOutput runProcessOutput, QBackendStepMetaData backendStepMetaData, QStepMetaData step) throws Exception
{
///////////////////////
// Run backend steps //
///////////////////////
LOG.debug("Running backend step [" + step.getName() + "] in process [" + process.getName() + "]");
RunBackendStepOutput runBackendStepOutput = runBackendStep(runProcessInput, process, runProcessOutput, stateKey, backendStepMetaData, process, processState);
//////////////////////////////////////////////////////////////////////////////////////////////
// similarly, if the step produced a processMetaDataAdjustment, propagate that data outward //
//////////////////////////////////////////////////////////////////////////////////////////////
if(runBackendStepOutput.getProcessMetaDataAdjustment() != null)
{
LOG.debug("Process step [" + step.getName() + "] generated a ProcessMetaDataAdjustment [" + runBackendStepOutput.getProcessMetaDataAdjustment() + "]!");
runProcessOutput.setProcessMetaDataAdjustment(runBackendStepOutput.getProcessMetaDataAdjustment());
}
return runBackendStepOutput;
}
/*******************************************************************************
** Run a single backend step.
*******************************************************************************/
protected RunBackendStepOutput runBackendStep(RunProcessInput runProcessInput, QProcessMetaData process, RunProcessOutput runProcessOutput, UUIDAndTypeStateKey stateKey, QBackendStepMetaData backendStep, QProcessMetaData qProcessMetaData, ProcessState processState) throws Exception
RunBackendStepOutput runBackendStep(RunProcessInput runProcessInput, QProcessMetaData process, RunProcessOutput runProcessOutput, UUIDAndTypeStateKey stateKey, QBackendStepMetaData backendStep, QProcessMetaData qProcessMetaData, ProcessState processState) throws Exception
{
RunBackendStepInput runBackendStepInput = new RunBackendStepInput(processState);
runBackendStepInput.setProcessName(process.getName());

46
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java
View File

@ -54,10 +54,12 @@ import com.kingsrook.qqq.backend.core.model.metadata.permissions.MetaDataWithPer
import com.kingsrook.qqq.backend.core.model.metadata.permissions.QPermissionRules;
import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSource;
import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSourceType;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QComponentType;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendComponentMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendStepMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QStateMachineStep;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QStepMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QSupplementalProcessMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportDataSource;
@ -75,6 +77,7 @@ import com.kingsrook.qqq.backend.core.processes.implementations.bulk.edit.BulkEd
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkInsertExtractStep;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkInsertLoadStep;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkInsertTransformStep;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkLoadReceiveFileStep;
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.ExtractViaQueryStep;
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess;
import com.kingsrook.qqq.backend.core.scheduler.QScheduleManager;
@ -410,10 +413,27 @@ public class QInstanceEnricher
**
*******************************************************************************/
private void enrichStep(QStepMetaData step)
{
enrichStep(step, false);
}
/***************************************************************************
**
***************************************************************************/
private void enrichStep(QStepMetaData step, boolean isSubStep)
{
if(!StringUtils.hasContent(step.getLabel()))
{
step.setLabel(nameToLabel(step.getName()));
if(isSubStep && (step.getName().endsWith(".backend") || step.getName().endsWith(".frontend")))
{
step.setLabel(nameToLabel(step.getName().replaceFirst("\\.(backend|frontend)", "")));
}
else
{
step.setLabel(nameToLabel(step.getName()));
}
}
step.getInputFields().forEach(this::enrichField);
@ -434,6 +454,13 @@ public class QInstanceEnricher
frontendStepMetaData.getRecordListFields().forEach(this::enrichField);
}
}
else if(step instanceof QStateMachineStep stateMachineStep)
{
for(QStepMetaData subStep : CollectionUtils.nonNullList(stateMachineStep.getSubSteps()))
{
enrichStep(subStep, true);
}
}
}
@ -846,6 +873,22 @@ public class QInstanceEnricher
}
}
QFrontendStepMetaData fileMappingScreen = new QFrontendStepMetaData()
.withName("fileMapping")
.withComponent(new QFrontendComponentMetaData().withType(QComponentType.BULK_LOAD_MAPPING));
process.addStep(0, fileMappingScreen);
//////////////////////////////////////////////////////////////////
// add a backend step to receive the file before the ETL starts //
//////////////////////////////////////////////////////////////////
QBackendStepMetaData receiveFileStep = new QBackendStepMetaData()
.withName("receiveFile")
.withCode(new QCodeReference(BulkLoadReceiveFileStep.class));
process.addStep(0, receiveFileStep);
//////////////////////////////////////
// add an upload screen before that //
//////////////////////////////////////
String fieldsForHelpText = editableFields.stream()
.map(QFieldMetaData::getLabel)
.collect(Collectors.joining(", "));
@ -862,6 +905,7 @@ public class QInstanceEnricher
process.addStep(0, uploadScreen);
process.getFrontendStep("review").setRecordListFields(editableFields);
qInstance.addProcess(process);
}

32
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/frontend/QFrontendProcessMetaData.java
View File

@ -31,6 +31,7 @@ import com.kingsrook.qqq.backend.core.actions.permissions.PermissionsHelper;
import com.kingsrook.qqq.backend.core.model.actions.AbstractActionInput;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendStepMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QStateMachineStep;
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
@ -50,6 +51,7 @@ public class QFrontendProcessMetaData
private String iconName;
private List<QFrontendStepMetaData> frontendSteps;
private String stepFlow;
private boolean hasPermission;
@ -68,15 +70,27 @@ public class QFrontendProcessMetaData
this.label = processMetaData.getLabel();
this.tableName = processMetaData.getTableName();
this.isHidden = processMetaData.getIsHidden();
this.stepFlow = processMetaData.getStepFlow().toString();
if(includeSteps)
{
if(CollectionUtils.nullSafeHasContents(processMetaData.getStepList()))
{
this.frontendSteps = processMetaData.getStepList().stream()
.filter(QFrontendStepMetaData.class::isInstance)
.map(QFrontendStepMetaData.class::cast)
.collect(Collectors.toList());
this.frontendSteps = switch(processMetaData.getStepFlow())
{
case LINEAR -> processMetaData.getStepList().stream()
.filter(QFrontendStepMetaData.class::isInstance)
.map(QFrontendStepMetaData.class::cast)
.collect(Collectors.toList());
case STATE_MACHINE -> processMetaData.getAllSteps().values().stream()
.filter(QStateMachineStep.class::isInstance)
.map(QStateMachineStep.class::cast)
.flatMap(step -> step.getSubSteps().stream())
.filter(QFrontendStepMetaData.class::isInstance)
.map(QFrontendStepMetaData.class::cast)
.collect(Collectors.toList());
};
}
else
{
@ -180,4 +194,14 @@ public class QFrontendProcessMetaData
return hasPermission;
}
/*******************************************************************************
** Getter for stepFlow
**
*******************************************************************************/
public String getStepFlow()
{
return stepFlow;
}
}

38
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/processes/ProcessStepFlow.java Normal file
View File

@ -0,0 +1,38 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2024. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.model.metadata.processes;
/*******************************************************************************
** Possible ways the steps of a process can flow.
**
** LINEAR - (the default) - the list of steps in the process are executed in-order
**
** STATE_MACHINE - concept of "states", each which has a backend & frontend step;
** a backend step can (must?) set the field "stepState" (or "nextStepName") to
** say what the next (frontend) step is.
*******************************************************************************/
public enum ProcessStepFlow
{
LINEAR,
STATE_MACHINE
}

100
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/processes/QProcessMetaData.java
View File

@ -57,6 +57,7 @@ public class QProcessMetaData implements QAppChildMetaData, MetaDataWithPermissi
private Integer minInputRecords = null;
private Integer maxInputRecords = null;
private ProcessStepFlow stepFlow = ProcessStepFlow.LINEAR;
private List<QStepMetaData> stepList; // these are the steps that are ran, by-default, in the order they are ran in
private Map<String, QStepMetaData> steps; // this is the full map of possible steps
@ -213,11 +214,10 @@ public class QProcessMetaData implements QAppChildMetaData, MetaDataWithPermissi
/*******************************************************************************
** add a step to the stepList and map
/***************************************************************************
**
*******************************************************************************/
public QProcessMetaData addStep(QStepMetaData step)
***************************************************************************/
public QProcessMetaData withStep(QStepMetaData step)
{
int index = 0;
if(this.stepList != null)
@ -231,11 +231,23 @@ public class QProcessMetaData implements QAppChildMetaData, MetaDataWithPermissi
/*******************************************************************************
** add a step to the stepList and map
**
*******************************************************************************/
@Deprecated(since = "withStep was added")
public QProcessMetaData addStep(QStepMetaData step)
{
return (withStep(step));
}
/*******************************************************************************
** add a step to the stepList (at the specified index) and the step map
**
*******************************************************************************/
public QProcessMetaData addStep(int index, QStepMetaData step)
public QProcessMetaData withStep(int index, QStepMetaData step)
{
if(this.stepList == null)
{
@ -260,11 +272,23 @@ public class QProcessMetaData implements QAppChildMetaData, MetaDataWithPermissi
/*******************************************************************************
** add a step to the stepList (at the specified index) and the step map
**
*******************************************************************************/
@Deprecated(since = "withStep was added")
public QProcessMetaData addStep(int index, QStepMetaData step)
{
return (withStep(index, step));
}
/*******************************************************************************
** add a step ONLY to the step map - NOT the list w/ default execution order.
**
*******************************************************************************/
public QProcessMetaData addOptionalStep(QStepMetaData step)
public QProcessMetaData withOptionalStep(QStepMetaData step)
{
if(this.steps == null)
{
@ -283,6 +307,18 @@ public class QProcessMetaData implements QAppChildMetaData, MetaDataWithPermissi
/*******************************************************************************
** add a step ONLY to the step map - NOT the list w/ default execution order.
**
*******************************************************************************/
@Deprecated(since = "withOptionalStep was added")
public QProcessMetaData addOptionalStep(QStepMetaData step)
{
return (withOptionalStep(step));
}
/*******************************************************************************
** Setter for stepList
**
@ -299,7 +335,26 @@ public class QProcessMetaData implements QAppChildMetaData, MetaDataWithPermissi
*******************************************************************************/
public QStepMetaData getStep(String stepName)
{
return (steps.get(stepName));
if(steps.containsKey(stepName))
{
return steps.get(stepName);
}
for(QStepMetaData step : steps.values())
{
if(step instanceof QStateMachineStep stateMachineStep)
{
for(QStepMetaData subStep : stateMachineStep.getSubSteps())
{
if(subStep.getName().equals(stepName))
{
return (subStep);
}
}
}
}
return (null);
}
@ -780,4 +835,35 @@ public class QProcessMetaData implements QAppChildMetaData, MetaDataWithPermissi
return (this);
}
/*******************************************************************************
** Getter for stepFlow
*******************************************************************************/
public ProcessStepFlow getStepFlow()
{
return (this.stepFlow);
}
/*******************************************************************************
** Setter for stepFlow
*******************************************************************************/
public void setStepFlow(ProcessStepFlow stepFlow)
{
this.stepFlow = stepFlow;
}
/*******************************************************************************
** Fluent setter for stepFlow
*******************************************************************************/
public QProcessMetaData withStepFlow(ProcessStepFlow stepFlow)
{
this.stepFlow = stepFlow;
return (this);
}
}

188
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/processes/QStateMachineStep.java Normal file
View File

@ -0,0 +1,188 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2024. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.model.metadata.processes;
import java.util.ArrayList;
import java.util.List;
import com.fasterxml.jackson.annotation.JsonIgnore;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
import com.kingsrook.qqq.backend.core.utils.StringUtils;
/*******************************************************************************
** A step for a state-machine flow based Process.
**
** Consists of 1 or 2 sub-steps, which are frontend and/or backend.
*******************************************************************************/
public class QStateMachineStep extends QStepMetaData
{
private List<QStepMetaData> subSteps = new ArrayList<>();
private String defaultNextStepName;
/*******************************************************************************
** Constructor
**
*******************************************************************************/
private QStateMachineStep(List<QStepMetaData> subSteps)
{
setStepType("stateMachine");
this.subSteps.addAll(subSteps);
}
/***************************************************************************
**
***************************************************************************/
public static QStateMachineStep frontendOnly(String name, QFrontendStepMetaData frontendStepMetaData)
{
if(!StringUtils.hasContent(frontendStepMetaData.getName()))
{
frontendStepMetaData.setName(name + ".frontend");
}
return (new QStateMachineStep(List.of(frontendStepMetaData)).withName(name));
}
/***************************************************************************
**
***************************************************************************/
public static QStateMachineStep backendOnly(String name, QBackendStepMetaData backendStepMetaData)
{
if(!StringUtils.hasContent(backendStepMetaData.getName()))
{
backendStepMetaData.setName(name + ".backend");
}
return (new QStateMachineStep(List.of(backendStepMetaData)).withName(name));
}
/***************************************************************************
**
***************************************************************************/
public static QStateMachineStep frontendThenBackend(String name, QFrontendStepMetaData frontendStepMetaData, QBackendStepMetaData backendStepMetaData)
{
if(!StringUtils.hasContent(frontendStepMetaData.getName()))
{
frontendStepMetaData.setName(name + ".frontend");
}
if(!StringUtils.hasContent(backendStepMetaData.getName()))
{
backendStepMetaData.setName(name + ".backend");
}
return (new QStateMachineStep(List.of(frontendStepMetaData, backendStepMetaData)).withName(name));
}
/***************************************************************************
**
***************************************************************************/
@Override
public QStateMachineStep withName(String name)
{
super.withName(name);
return (this);
}
/***************************************************************************
**
***************************************************************************/
@Override
public QStateMachineStep withLabel(String label)
{
super.withLabel(label);
return (this);
}
/*******************************************************************************
** Getter for subSteps
**
*******************************************************************************/
public List<QStepMetaData> getSubSteps()
{
return subSteps;
}
/*******************************************************************************
** Getter for defaultNextStepName
*******************************************************************************/
public String getDefaultNextStepName()
{
return (this.defaultNextStepName);
}
/*******************************************************************************
** Setter for defaultNextStepName
*******************************************************************************/
public void setDefaultNextStepName(String defaultNextStepName)
{
this.defaultNextStepName = defaultNextStepName;
}
/*******************************************************************************
** Fluent setter for defaultNextStepName
*******************************************************************************/
public QStateMachineStep withDefaultNextStepName(String defaultNextStepName)
{
this.defaultNextStepName = defaultNextStepName;
return (this);
}
/*******************************************************************************
** Get a list of all of the input fields used by this step (all of its sub-steps)
*******************************************************************************/
@JsonIgnore
@Override
public List<QFieldMetaData> getInputFields()
{
List<QFieldMetaData> rs = new ArrayList<>();
for(QStepMetaData subStep : subSteps)
{
rs.addAll(subStep.getInputFields());
}
return (rs);
}
}

327
qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/processes/RunProcessActionTest.java Normal file
View File

@ -0,0 +1,327 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2024. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.actions.processes;
import java.util.ArrayList;
import java.util.List;
import com.kingsrook.qqq.backend.core.BaseTest;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessOutput;
import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReferenceLambda;
import com.kingsrook.qqq.backend.core.model.metadata.processes.ProcessStepFlow;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendStepMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QStateMachineStep;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
import static org.junit.jupiter.api.Assertions.assertEquals;
/*******************************************************************************
** Unit test for RunProcessAction
*******************************************************************************/
class RunProcessActionTest extends BaseTest
{
private static List<String> log = new ArrayList<>();
/*******************************************************************************
**
*******************************************************************************/
@BeforeEach
void beforeEach()
{
log.clear();
}
/*******************************************************************************
**
*******************************************************************************/
@Test
void testStateMachineTwoBackendSteps() throws QException
{
QProcessMetaData process = new QProcessMetaData().withName("test")
/////////////////////////////////////////////////////////////////
// two-steps - a, points at b; b has no next-step, so it exits //
/////////////////////////////////////////////////////////////////
.addStep(QStateMachineStep.backendOnly("a", new QBackendStepMetaData().withName("aBackend")
.withCode(new QCodeReferenceLambda<BackendStep>((runBackendStepInput, runBackendStepOutput) ->
{
log.add("in StepA");
runBackendStepOutput.getProcessState().setNextStepName("b");
}))))
.addStep(QStateMachineStep.backendOnly("b", new QBackendStepMetaData().withName("bBackend")
.withCode(new QCodeReferenceLambda<BackendStep>((runBackendStepInput, runBackendStepOutput) ->
{
log.add("in StepB");
}))))
.withStepFlow(ProcessStepFlow.STATE_MACHINE);
QContext.getQInstance().addProcess(process);
RunProcessInput input = new RunProcessInput();
input.setProcessName("test");
input.setFrontendStepBehavior(RunProcessInput.FrontendStepBehavior.BREAK);
RunProcessOutput runProcessOutput = new RunProcessAction().execute(input);
assertEquals(List.of("in StepA", "in StepB"), log);
assertThat(runProcessOutput.getProcessState().getNextStepName()).isEmpty();
}
/*******************************************************************************
**
*******************************************************************************/
@Test
void testStateMachineTwoFrontendOnlySteps() throws QException
{
QProcessMetaData process = new QProcessMetaData().withName("test")
.addStep(QStateMachineStep.frontendOnly("a", new QFrontendStepMetaData().withName("aFrontend")).withDefaultNextStepName("b"))
.addStep(QStateMachineStep.frontendOnly("b", new QFrontendStepMetaData().withName("bFrontend")))
.withStepFlow(ProcessStepFlow.STATE_MACHINE);
QContext.getQInstance().addProcess(process);
RunProcessInput input = new RunProcessInput();
input.setProcessName("test");
input.setFrontendStepBehavior(RunProcessInput.FrontendStepBehavior.BREAK);
RunProcessOutput runProcessOutput = new RunProcessAction().execute(input);
assertThat(runProcessOutput.getProcessState().getNextStepName())
.isPresent().get()
.isEqualTo("aFrontend");
/////////////////////////////
// resume after a, go to b //
/////////////////////////////
input.setStartAfterStep("aFrontend");
runProcessOutput = new RunProcessAction().execute(input);
assertThat(runProcessOutput.getProcessState().getNextStepName())
.isPresent().get()
.isEqualTo("bFrontend");
}
/*******************************************************************************
**
*******************************************************************************/
@Test
void testStateMachineOneBackendStepReferencingItselfDoesNotInfiniteLoop() throws QException
{
QProcessMetaData process = new QProcessMetaData().withName("test")
///////////////////////////////////////////////////////////////
// set up step that always points back at itself. //
// since it never goes to the frontend, it'll stack overflow //
// (though we'll catch it ourselves before JVM does) //
///////////////////////////////////////////////////////////////
.addStep(QStateMachineStep.backendOnly("a", new QBackendStepMetaData().withName("aBackend")
.withCode(new QCodeReferenceLambda<BackendStep>((runBackendStepInput, runBackendStepOutput) ->
{
log.add("in StepA");
runBackendStepOutput.getProcessState().setNextStepName("a");
}))))
.withStepFlow(ProcessStepFlow.STATE_MACHINE);
QContext.getQInstance().addProcess(process);
RunProcessInput input = new RunProcessInput();
input.setProcessName("test");
input.setFrontendStepBehavior(RunProcessInput.FrontendStepBehavior.BREAK);
assertThatThrownBy(() -> new RunProcessAction().execute(input))
.isInstanceOf(QException.class)
.hasMessageContaining("maxStateMachineProcessStepFlowStackDepth of 20");
///////////////////////////////////////////////////
// make sure we can set a custom max-stack-depth //
///////////////////////////////////////////////////
input.addValue("maxStateMachineProcessStepFlowStackDepth", 5);
assertThatThrownBy(() -> new RunProcessAction().execute(input))
.isInstanceOf(QException.class)
.hasMessageContaining("maxStateMachineProcessStepFlowStackDepth of 5");
}
/*******************************************************************************
**
*******************************************************************************/
@Test
void testStateMachineTwoBackendStepsReferencingEachOtherDoesNotInfiniteLoop() throws QException
{
QProcessMetaData process = new QProcessMetaData().withName("test")
///////////////////////////////////////////////////////////////
// set up two steps that always points back at each other. //
// since it never goes to the frontend, it'll stack overflow //
// (though we'll catch it ourselves before JVM does) //
///////////////////////////////////////////////////////////////
.addStep(QStateMachineStep.backendOnly("a", new QBackendStepMetaData().withName("aBackend")
.withCode(new QCodeReferenceLambda<BackendStep>((runBackendStepInput, runBackendStepOutput) ->
{
log.add("in StepA");
runBackendStepOutput.getProcessState().setNextStepName("b");
}))))
.addStep(QStateMachineStep.backendOnly("b", new QBackendStepMetaData().withName("bBackend")
.withCode(new QCodeReferenceLambda<BackendStep>((runBackendStepInput, runBackendStepOutput) ->
{
log.add("in StepB");
runBackendStepOutput.getProcessState().setNextStepName("a");
}))))
.withStepFlow(ProcessStepFlow.STATE_MACHINE);
QContext.getQInstance().addProcess(process);
RunProcessInput input = new RunProcessInput();
input.setProcessName("test");
input.setFrontendStepBehavior(RunProcessInput.FrontendStepBehavior.BREAK);
assertThatThrownBy(() -> new RunProcessAction().execute(input))
.isInstanceOf(QException.class)
.hasMessageContaining("maxStateMachineProcessStepFlowStackDepth of 20");
///////////////////////////////////////////////////
// make sure we can set a custom max-stack-depth //
///////////////////////////////////////////////////
input.addValue("maxStateMachineProcessStepFlowStackDepth", 5);
assertThatThrownBy(() -> new RunProcessAction().execute(input))
.isInstanceOf(QException.class)
.hasMessageContaining("maxStateMachineProcessStepFlowStackDepth of 5");
}
/*******************************************************************************
**
*******************************************************************************/
@Test
void testStateSequenceOfFrontendAndBackendSteps() throws QException
{
QProcessMetaData process = new QProcessMetaData().withName("test")
.addStep(QStateMachineStep.frontendThenBackend("a",
new QFrontendStepMetaData().withName("aFrontend"),
new QBackendStepMetaData().withName("aBackend")
.withCode(new QCodeReferenceLambda<BackendStep>((runBackendStepInput, runBackendStepOutput) ->
{
log.add("in StepA");
runBackendStepOutput.getProcessState().setNextStepName("b");
}))))
.addStep(QStateMachineStep.frontendThenBackend("b",
new QFrontendStepMetaData().withName("bFrontend"),
new QBackendStepMetaData().withName("bBackend")
.withCode(new QCodeReferenceLambda<BackendStep>((runBackendStepInput, runBackendStepOutput) ->
{
log.add("in StepB");
runBackendStepOutput.getProcessState().setNextStepName("c");
}))))
.addStep(QStateMachineStep.frontendThenBackend("c",
new QFrontendStepMetaData().withName("cFrontend"),
new QBackendStepMetaData().withName("cBackend")
.withCode(new QCodeReferenceLambda<BackendStep>((runBackendStepInput, runBackendStepOutput) ->
{
log.add("in StepC");
runBackendStepOutput.getProcessState().setNextStepName("d");
}))))
.addStep(QStateMachineStep.frontendOnly("d",
new QFrontendStepMetaData().withName("dFrontend")))
.withStepFlow(ProcessStepFlow.STATE_MACHINE);
QContext.getQInstance().addProcess(process);
RunProcessInput input = new RunProcessInput();
input.setProcessName("test");
input.setFrontendStepBehavior(RunProcessInput.FrontendStepBehavior.BREAK);
////////////////////////////////////////////////////////
// start the process - we should be sent to aFrontend //
////////////////////////////////////////////////////////
RunProcessOutput runProcessOutput = new RunProcessAction().execute(input);
assertThat(runProcessOutput.getProcessState().getNextStepName())
.isPresent().get()
.isEqualTo("aFrontend");
///////////////////////////////////////////////////////////////////////////////////////////
// resume after aFrontend - we should run StepA (backend), and then be sent to bFrontend //
///////////////////////////////////////////////////////////////////////////////////////////
input.setStartAfterStep("aFrontend");
runProcessOutput = new RunProcessAction().execute(input);
assertEquals(List.of("in StepA"), log);
assertThat(runProcessOutput.getProcessState().getNextStepName())
.isPresent().get()
.isEqualTo("bFrontend");
///////////////////////////////////////////////////////////////////////////////////////////
// resume after bFrontend - we should run StepB (backend), and then be sent to cFrontend //
///////////////////////////////////////////////////////////////////////////////////////////
input.setStartAfterStep("bFrontend");
runProcessOutput = new RunProcessAction().execute(input);
assertEquals(List.of("in StepA", "in StepB"), log);
assertThat(runProcessOutput.getProcessState().getNextStepName())
.isPresent().get()
.isEqualTo("cFrontend");
///////////////////////////////////////////////////////////////////////////////////////////
// resume after cFrontend - we should run StepC (backend), and then be sent to dFrontend //
///////////////////////////////////////////////////////////////////////////////////////////
input.setStartAfterStep("cFrontend");
runProcessOutput = new RunProcessAction().execute(input);
assertEquals(List.of("in StepA", "in StepB", "in StepC"), log);
assertThat(runProcessOutput.getProcessState().getNextStepName())
.isPresent().get()
.isEqualTo("dFrontend");
////////////////////////////////////////////////////////////////////////////////////
// if we resume again here, we'll be past the end of the process, so no next-step //
////////////////////////////////////////////////////////////////////////////////////
input.setStartAfterStep("dFrontend");
runProcessOutput = new RunProcessAction().execute(input);
assertEquals(List.of("in StepA", "in StepB", "in StepC"), log);
assertThat(runProcessOutput.getProcessState().getNextStepName()).isEmpty();
}
}