diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/processes/RunProcessAction.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/processes/RunProcessAction.java index 704e2313..6825d523 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/processes/RunProcessAction.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/processes/RunProcessAction.java @@ -28,6 +28,7 @@ import java.time.temporal.ChronoUnit; import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Optional; import java.util.UUID; import com.kingsrook.qqq.backend.core.actions.ActionHelper; @@ -58,6 +59,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaD import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendComponentMetaData; import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendStepMetaData; import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QStateMachineStep; import com.kingsrook.qqq.backend.core.model.metadata.processes.QStepMetaData; import com.kingsrook.qqq.backend.core.model.session.QSession; import com.kingsrook.qqq.backend.core.processes.implementations.basepull.BasepullConfiguration; @@ -133,90 +135,11 @@ public class RunProcessAction try { - String lastStepName = runProcessInput.getStartAfterStep(); - - STEP_LOOP: - while(true) + switch(Objects.requireNonNull(process.getStepFlow(), "Process [" + process.getName() + "] has a null stepFlow.")) { - /////////////////////////////////////////////////////////////////////////////////////////////////////// - // always refresh the step list - as any step that runs can modify it (in the process state). // - // this is why we don't do a loop over the step list - as we'd get ConcurrentModificationExceptions. // - /////////////////////////////////////////////////////////////////////////////////////////////////////// - List stepList = getAvailableStepList(processState, process, lastStepName); - if(stepList.isEmpty()) - { - break; - } - - QStepMetaData step = stepList.get(0); - lastStepName = step.getName(); - - if(step instanceof QFrontendStepMetaData frontendStep) - { - //////////////////////////////////////////////////////////////// - // Handle what to do with frontend steps, per request setting // - //////////////////////////////////////////////////////////////// - switch(runProcessInput.getFrontendStepBehavior()) - { - case BREAK -> - { - LOG.trace("Breaking process [" + process.getName() + "] at frontend step (as requested by caller): " + step.getName()); - processFrontendStepFieldDefaultValues(processState, frontendStep); - processFrontendComponents(processState, frontendStep); - processState.setNextStepName(step.getName()); - break STEP_LOOP; - } - case SKIP -> - { - LOG.trace("Skipping frontend step [" + step.getName() + "] in process [" + process.getName() + "] (as requested by caller)"); - - ////////////////////////////////////////////////////////////////////// - // much less error prone in case this code changes in the future... // - ////////////////////////////////////////////////////////////////////// - // noinspection UnnecessaryContinue - continue; - } - case FAIL -> - { - LOG.trace("Throwing error for frontend step [" + step.getName() + "] in process [" + process.getName() + "] (as requested by caller)"); - throw (new QException("Failing process at step " + step.getName() + " (as requested, to fail on frontend steps)")); - } - default -> throw new IllegalStateException("Unexpected value: " + runProcessInput.getFrontendStepBehavior()); - } - } - else if(step instanceof QBackendStepMetaData backendStepMetaData) - { - /////////////////////// - // Run backend steps // - /////////////////////// - LOG.debug("Running backend step [" + step.getName() + "] in process [" + process.getName() + "]"); - RunBackendStepOutput runBackendStepOutput = runBackendStep(runProcessInput, process, runProcessOutput, stateKey, backendStepMetaData, process, processState); - - ///////////////////////////////////////////////////////////////////////////////////////// - // if the step returned an override lastStepName, use that to determine how we proceed // - ///////////////////////////////////////////////////////////////////////////////////////// - if(runBackendStepOutput.getOverrideLastStepName() != null) - { - LOG.debug("Process step [" + lastStepName + "] returned an overrideLastStepName [" + runBackendStepOutput.getOverrideLastStepName() + "]!"); - lastStepName = runBackendStepOutput.getOverrideLastStepName(); - } - - ///////////////////////////////////////////////////////////////////////////////////////////// - // similarly, if the step produced an updatedFrontendStepList, propagate that data outward // - ///////////////////////////////////////////////////////////////////////////////////////////// - if(runBackendStepOutput.getUpdatedFrontendStepList() != null) - { - LOG.debug("Process step [" + lastStepName + "] generated an updatedFrontendStepList [" + runBackendStepOutput.getUpdatedFrontendStepList().stream().map(s -> s.getName()).toList() + "]!"); - runProcessOutput.setUpdatedFrontendStepList(runBackendStepOutput.getUpdatedFrontendStepList()); - } - } - else - { - ////////////////////////////////////////////////// - // in case we have a different step type, throw // - ////////////////////////////////////////////////// - throw (new QException("Unsure how to run a step of type: " + step.getClass().getName())); - } + case LINEAR -> runLinearStepLoop(process, processState, stateKey, runProcessInput, runProcessOutput); + case STATE_MACHINE -> runStateMachineStep(runProcessInput.getStartAfterStep(), process, processState, stateKey, runProcessInput, runProcessOutput, 0); + default -> throw (new QException("Unhandled process step flow: " + process.getStepFlow())); } /////////////////////////////////////////////////////////////////////////// @@ -258,6 +181,270 @@ public class RunProcessAction + /*************************************************************************** + ** + ***************************************************************************/ + private void runLinearStepLoop(QProcessMetaData process, ProcessState processState, UUIDAndTypeStateKey stateKey, RunProcessInput runProcessInput, RunProcessOutput runProcessOutput) throws Exception + { + String lastStepName = runProcessInput.getStartAfterStep(); + + while(true) + { + /////////////////////////////////////////////////////////////////////////////////////////////////////// + // always refresh the step list - as any step that runs can modify it (in the process state). // + // this is why we don't do a loop over the step list - as we'd get ConcurrentModificationExceptions. // + /////////////////////////////////////////////////////////////////////////////////////////////////////// + List stepList = getAvailableStepList(processState, process, lastStepName); + if(stepList.isEmpty()) + { + break; + } + + QStepMetaData step = stepList.get(0); + lastStepName = step.getName(); + + if(step instanceof QFrontendStepMetaData frontendStep) + { + LoopTodo loopTodo = prepareForFrontendStep(runProcessInput, process, frontendStep, processState); + if(loopTodo == LoopTodo.BREAK) + { + break; + } + } + else if(step instanceof QBackendStepMetaData backendStepMetaData) + { + RunBackendStepOutput runBackendStepOutput = runBackendStep(process, processState, stateKey, runProcessInput, runProcessOutput, backendStepMetaData, step); + + ///////////////////////////////////////////////////////////////////////////////////////// + // if the step returned an override lastStepName, use that to determine how we proceed // + ///////////////////////////////////////////////////////////////////////////////////////// + if(runBackendStepOutput.getOverrideLastStepName() != null) + { + LOG.debug("Process step [" + lastStepName + "] returned an overrideLastStepName [" + runBackendStepOutput.getOverrideLastStepName() + "]!"); + lastStepName = runBackendStepOutput.getOverrideLastStepName(); + } + } + else + { + ////////////////////////////////////////////////// + // in case we have a different step type, throw // + ////////////////////////////////////////////////// + throw (new QException("Unsure how to run a step of type: " + step.getClass().getName())); + } + } + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private enum LoopTodo + { + BREAK, + CONTINUE + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private LoopTodo prepareForFrontendStep(RunProcessInput runProcessInput, QProcessMetaData process, QFrontendStepMetaData step, ProcessState processState) throws QException + { + //////////////////////////////////////////////////////////////// + // Handle what to do with frontend steps, per request setting // + //////////////////////////////////////////////////////////////// + switch(runProcessInput.getFrontendStepBehavior()) + { + case BREAK -> + { + LOG.trace("Breaking process [" + process.getName() + "] at frontend step (as requested by caller): " + step.getName()); + processFrontendStepFieldDefaultValues(processState, step); + processFrontendComponents(processState, step); + processState.setNextStepName(step.getName()); + return LoopTodo.BREAK; + } + case SKIP -> + { + LOG.trace("Skipping frontend step [" + step.getName() + "] in process [" + process.getName() + "] (as requested by caller)"); + return LoopTodo.CONTINUE; + } + case FAIL -> + { + LOG.trace("Throwing error for frontend step [" + step.getName() + "] in process [" + process.getName() + "] (as requested by caller)"); + throw (new QException("Failing process at step " + step.getName() + " (as requested, to fail on frontend steps)")); + } + default -> throw new IllegalStateException("Unexpected value: " + runProcessInput.getFrontendStepBehavior()); + } + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private void runStateMachineStep(String lastStepName, QProcessMetaData process, ProcessState processState, UUIDAndTypeStateKey stateKey, RunProcessInput runProcessInput, RunProcessOutput runProcessOutput, int stackDepth) throws Exception + { + ////////////////////////////// + // check for stack-overflow // + ////////////////////////////// + Integer maxStateMachineProcessStepFlowStackDepth = Objects.requireNonNullElse(runProcessInput.getValueInteger("maxStateMachineProcessStepFlowStackDepth"), 20); + if(stackDepth > maxStateMachineProcessStepFlowStackDepth) + { + throw (new QException("StateMachine process recurred too many times (exceeded maxStateMachineProcessStepFlowStackDepth of " + maxStateMachineProcessStepFlowStackDepth + ")")); + } + + ////////////////////////////////// + // figure out what step to run: // + ////////////////////////////////// + QStepMetaData step = null; + if(!StringUtils.hasContent(lastStepName)) + { + //////////////////////////////////////////////////////////////////// + // if no lastStepName is given, start at the process's first step // + //////////////////////////////////////////////////////////////////// + if(CollectionUtils.nullSafeIsEmpty(process.getStepList())) + { + throw (new QException("Process [" + process.getName() + "] does not have a step list defined.")); + } + step = process.getStepList().get(0); + } + else + { + ///////////////////////////////////// + // else run the given lastStepName // + ///////////////////////////////////// + processState.clearNextStepName(); + step = process.getStep(lastStepName); + if(step == null) + { + throw (new QException("Could not find step by name [" + lastStepName + "]")); + } + } + + ///////////////////////////////////////////////////////////////////////// + // for the flow of: // + // we were on a frontend step (as a sub-step of a state machine step), // + // and now we're here to run that state-step's backend step - // + // find the state-machine step containing this frontend step. // + ///////////////////////////////////////////////////////////////////////// + String skipSubStepsUntil = null; + if(step instanceof QFrontendStepMetaData frontendStepMetaData) + { + QStateMachineStep stateMachineStep = getStateMachineStepContainingSubStep(process, frontendStepMetaData.getName()); + if(stateMachineStep == null) + { + throw (new QException("Could not find stateMachineStep that contains last-frontend step: " + frontendStepMetaData.getName())); + } + step = stateMachineStep; + + ////////////////////////////////////////////////////////////////////////////////// + // set this flag, to know to skip this frontend step in the sub-step loop below // + ////////////////////////////////////////////////////////////////////////////////// + skipSubStepsUntil = frontendStepMetaData.getName(); + } + + if(!(step instanceof QStateMachineStep stateMachineStep)) + { + throw (new QException("Have a non-stateMachineStep in a process using stateMachine flow... " + step.getClass().getName())); + } + + /////////////////////// + // run the sub-steps // + /////////////////////// + boolean ranAnySubSteps = false; + for(QStepMetaData subStep : stateMachineStep.getSubSteps()) + { + /////////////////////////////////////////////////////////////////////////////////////////////// + // ok, well, skip them if this flag is set (and clear the flag once we've hit this sub-step) // + /////////////////////////////////////////////////////////////////////////////////////////////// + if(skipSubStepsUntil != null) + { + if(skipSubStepsUntil.equals(subStep.getName())) + { + skipSubStepsUntil = null; + } + continue; + } + + ranAnySubSteps = true; + if(subStep instanceof QFrontendStepMetaData frontendStep) + { + LoopTodo loopTodo = prepareForFrontendStep(runProcessInput, process, frontendStep, processState); + if(loopTodo == LoopTodo.BREAK) + { + return; + } + } + else if(subStep instanceof QBackendStepMetaData backendStepMetaData) + { + RunBackendStepOutput runBackendStepOutput = runBackendStep(process, processState, stateKey, runProcessInput, runProcessOutput, backendStepMetaData, step); + Optional nextStepName = runBackendStepOutput.getProcessState().getNextStepName(); + + if(nextStepName.isEmpty() && StringUtils.hasContent(stateMachineStep.getDefaultNextStepName())) + { + nextStepName = Optional.of(stateMachineStep.getDefaultNextStepName()); + } + + if(nextStepName.isPresent()) + { + ////////////////////////////////////////////////////////////////////////////////////////////////////// + // if we've been given a next-step-name, go to that step now. // + // it might be a backend-only stateMachineStep, in which case, we should run that backend step now. // + // or it might be a frontend-then-backend step, in which case, we want to go to that frontend step. // + // if we weren't given a next-step-name, then we should stay in the same state - either to finish // + // its sub-steps, or, to fall out of the loop and end the process. // + ////////////////////////////////////////////////////////////////////////////////////////////////////// + processState.clearNextStepName(); + runStateMachineStep(nextStepName.get(), process, processState, stateKey, runProcessInput, runProcessOutput, stackDepth + 1); + return; + } + } + else + { + ////////////////////////////////////////////////// + // in case we have a different step type, throw // + ////////////////////////////////////////////////// + throw (new QException("Unsure how to run a step of type: " + step.getClass().getName())); + } + } + + if(!ranAnySubSteps) + { + if(StringUtils.hasContent(stateMachineStep.getDefaultNextStepName())) + { + runStateMachineStep(stateMachineStep.getDefaultNextStepName(), process, processState, stateKey, runProcessInput, runProcessOutput, stackDepth + 1); + } + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public QStateMachineStep getStateMachineStepContainingSubStep(QProcessMetaData process, String stepName) + { + for(QStepMetaData step : process.getAllSteps().values()) + { + if(step instanceof QStateMachineStep stateMachineStep) + { + for(QStepMetaData subStep : stateMachineStep.getSubSteps()) + { + if(subStep.getName().equals(stepName)) + { + return (stateMachineStep); + } + } + } + } + + return (null); + } + + + /******************************************************************************* ** *******************************************************************************/ @@ -355,16 +542,40 @@ public class RunProcessAction } ProcessState processState = optionalProcessState.get(); - processState.clearNextStepName(); return processState; } + /*************************************************************************** + ** + ***************************************************************************/ + private RunBackendStepOutput runBackendStep(QProcessMetaData process, ProcessState processState, UUIDAndTypeStateKey stateKey, RunProcessInput runProcessInput, RunProcessOutput runProcessOutput, QBackendStepMetaData backendStepMetaData, QStepMetaData step) throws Exception + { + /////////////////////// + // Run backend steps // + /////////////////////// + LOG.debug("Running backend step [" + step.getName() + "] in process [" + process.getName() + "]"); + RunBackendStepOutput runBackendStepOutput = runBackendStep(runProcessInput, process, runProcessOutput, stateKey, backendStepMetaData, process, processState); + + ////////////////////////////////////////////////////////////////////////////////////////////// + // similarly, if the step produced a processMetaDataAdjustment, propagate that data outward // + ////////////////////////////////////////////////////////////////////////////////////////////// + if(runBackendStepOutput.getProcessMetaDataAdjustment() != null) + { + LOG.debug("Process step [" + step.getName() + "] generated a ProcessMetaDataAdjustment [" + runBackendStepOutput.getProcessMetaDataAdjustment() + "]!"); + runProcessOutput.setProcessMetaDataAdjustment(runBackendStepOutput.getProcessMetaDataAdjustment()); + } + + return runBackendStepOutput; + } + + + /******************************************************************************* ** Run a single backend step. *******************************************************************************/ - protected RunBackendStepOutput runBackendStep(RunProcessInput runProcessInput, QProcessMetaData process, RunProcessOutput runProcessOutput, UUIDAndTypeStateKey stateKey, QBackendStepMetaData backendStep, QProcessMetaData qProcessMetaData, ProcessState processState) throws Exception + RunBackendStepOutput runBackendStep(RunProcessInput runProcessInput, QProcessMetaData process, RunProcessOutput runProcessOutput, UUIDAndTypeStateKey stateKey, QBackendStepMetaData backendStep, QProcessMetaData qProcessMetaData, ProcessState processState) throws Exception { RunBackendStepInput runBackendStepInput = new RunBackendStepInput(processState); runBackendStepInput.setProcessName(process.getName()); diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java index 0e101379..d0e9f4fb 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java @@ -54,10 +54,12 @@ import com.kingsrook.qqq.backend.core.model.metadata.permissions.MetaDataWithPer import com.kingsrook.qqq.backend.core.model.metadata.permissions.QPermissionRules; import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSource; import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSourceType; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData; import com.kingsrook.qqq.backend.core.model.metadata.processes.QComponentType; import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendComponentMetaData; import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendStepMetaData; import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QStateMachineStep; import com.kingsrook.qqq.backend.core.model.metadata.processes.QStepMetaData; import com.kingsrook.qqq.backend.core.model.metadata.processes.QSupplementalProcessMetaData; import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportDataSource; @@ -75,6 +77,7 @@ import com.kingsrook.qqq.backend.core.processes.implementations.bulk.edit.BulkEd import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkInsertExtractStep; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkInsertLoadStep; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkInsertTransformStep; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkLoadReceiveFileStep; import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.ExtractViaQueryStep; import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess; import com.kingsrook.qqq.backend.core.scheduler.QScheduleManager; @@ -410,10 +413,27 @@ public class QInstanceEnricher ** *******************************************************************************/ private void enrichStep(QStepMetaData step) + { + enrichStep(step, false); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private void enrichStep(QStepMetaData step, boolean isSubStep) { if(!StringUtils.hasContent(step.getLabel())) { - step.setLabel(nameToLabel(step.getName())); + if(isSubStep && (step.getName().endsWith(".backend") || step.getName().endsWith(".frontend"))) + { + step.setLabel(nameToLabel(step.getName().replaceFirst("\\.(backend|frontend)", ""))); + } + else + { + step.setLabel(nameToLabel(step.getName())); + } } step.getInputFields().forEach(this::enrichField); @@ -434,6 +454,13 @@ public class QInstanceEnricher frontendStepMetaData.getRecordListFields().forEach(this::enrichField); } } + else if(step instanceof QStateMachineStep stateMachineStep) + { + for(QStepMetaData subStep : CollectionUtils.nonNullList(stateMachineStep.getSubSteps())) + { + enrichStep(subStep, true); + } + } } @@ -846,6 +873,22 @@ public class QInstanceEnricher } } + QFrontendStepMetaData fileMappingScreen = new QFrontendStepMetaData() + .withName("fileMapping") + .withComponent(new QFrontendComponentMetaData().withType(QComponentType.BULK_LOAD_MAPPING)); + process.addStep(0, fileMappingScreen); + + ////////////////////////////////////////////////////////////////// + // add a backend step to receive the file before the ETL starts // + ////////////////////////////////////////////////////////////////// + QBackendStepMetaData receiveFileStep = new QBackendStepMetaData() + .withName("receiveFile") + .withCode(new QCodeReference(BulkLoadReceiveFileStep.class)); + process.addStep(0, receiveFileStep); + + ////////////////////////////////////// + // add an upload screen before that // + ////////////////////////////////////// String fieldsForHelpText = editableFields.stream() .map(QFieldMetaData::getLabel) .collect(Collectors.joining(", ")); @@ -862,6 +905,7 @@ public class QInstanceEnricher process.addStep(0, uploadScreen); process.getFrontendStep("review").setRecordListFields(editableFields); + qInstance.addProcess(process); } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/frontend/QFrontendProcessMetaData.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/frontend/QFrontendProcessMetaData.java index ba2bfa60..ee37a388 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/frontend/QFrontendProcessMetaData.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/frontend/QFrontendProcessMetaData.java @@ -31,6 +31,7 @@ import com.kingsrook.qqq.backend.core.actions.permissions.PermissionsHelper; import com.kingsrook.qqq.backend.core.model.actions.AbstractActionInput; import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendStepMetaData; import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QStateMachineStep; import com.kingsrook.qqq.backend.core.utils.CollectionUtils; @@ -50,6 +51,7 @@ public class QFrontendProcessMetaData private String iconName; private List frontendSteps; + private String stepFlow; private boolean hasPermission; @@ -68,15 +70,27 @@ public class QFrontendProcessMetaData this.label = processMetaData.getLabel(); this.tableName = processMetaData.getTableName(); this.isHidden = processMetaData.getIsHidden(); + this.stepFlow = processMetaData.getStepFlow().toString(); if(includeSteps) { if(CollectionUtils.nullSafeHasContents(processMetaData.getStepList())) { - this.frontendSteps = processMetaData.getStepList().stream() - .filter(QFrontendStepMetaData.class::isInstance) - .map(QFrontendStepMetaData.class::cast) - .collect(Collectors.toList()); + this.frontendSteps = switch(processMetaData.getStepFlow()) + { + case LINEAR -> processMetaData.getStepList().stream() + .filter(QFrontendStepMetaData.class::isInstance) + .map(QFrontendStepMetaData.class::cast) + .collect(Collectors.toList()); + + case STATE_MACHINE -> processMetaData.getAllSteps().values().stream() + .filter(QStateMachineStep.class::isInstance) + .map(QStateMachineStep.class::cast) + .flatMap(step -> step.getSubSteps().stream()) + .filter(QFrontendStepMetaData.class::isInstance) + .map(QFrontendStepMetaData.class::cast) + .collect(Collectors.toList()); + }; } else { @@ -180,4 +194,14 @@ public class QFrontendProcessMetaData return hasPermission; } + + + /******************************************************************************* + ** Getter for stepFlow + ** + *******************************************************************************/ + public String getStepFlow() + { + return stepFlow; + } } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/processes/ProcessStepFlow.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/processes/ProcessStepFlow.java new file mode 100644 index 00000000..492496dd --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/processes/ProcessStepFlow.java @@ -0,0 +1,38 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.metadata.processes; + + +/******************************************************************************* + ** Possible ways the steps of a process can flow. + ** + ** LINEAR - (the default) - the list of steps in the process are executed in-order + ** + ** STATE_MACHINE - concept of "states", each which has a backend & frontend step; + ** a backend step can (must?) set the field "stepState" (or "nextStepName") to + ** say what the next (frontend) step is. + *******************************************************************************/ +public enum ProcessStepFlow +{ + LINEAR, + STATE_MACHINE +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/processes/QProcessMetaData.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/processes/QProcessMetaData.java index fc7c7687..8bb459ef 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/processes/QProcessMetaData.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/processes/QProcessMetaData.java @@ -57,6 +57,7 @@ public class QProcessMetaData implements QAppChildMetaData, MetaDataWithPermissi private Integer minInputRecords = null; private Integer maxInputRecords = null; + private ProcessStepFlow stepFlow = ProcessStepFlow.LINEAR; private List stepList; // these are the steps that are ran, by-default, in the order they are ran in private Map steps; // this is the full map of possible steps @@ -213,11 +214,10 @@ public class QProcessMetaData implements QAppChildMetaData, MetaDataWithPermissi - /******************************************************************************* - ** add a step to the stepList and map + /*************************************************************************** ** - *******************************************************************************/ - public QProcessMetaData addStep(QStepMetaData step) + ***************************************************************************/ + public QProcessMetaData withStep(QStepMetaData step) { int index = 0; if(this.stepList != null) @@ -231,11 +231,23 @@ public class QProcessMetaData implements QAppChildMetaData, MetaDataWithPermissi + /******************************************************************************* + ** add a step to the stepList and map + ** + *******************************************************************************/ + @Deprecated(since = "withStep was added") + public QProcessMetaData addStep(QStepMetaData step) + { + return (withStep(step)); + } + + + /******************************************************************************* ** add a step to the stepList (at the specified index) and the step map ** *******************************************************************************/ - public QProcessMetaData addStep(int index, QStepMetaData step) + public QProcessMetaData withStep(int index, QStepMetaData step) { if(this.stepList == null) { @@ -260,11 +272,23 @@ public class QProcessMetaData implements QAppChildMetaData, MetaDataWithPermissi + /******************************************************************************* + ** add a step to the stepList (at the specified index) and the step map + ** + *******************************************************************************/ + @Deprecated(since = "withStep was added") + public QProcessMetaData addStep(int index, QStepMetaData step) + { + return (withStep(index, step)); + } + + + /******************************************************************************* ** add a step ONLY to the step map - NOT the list w/ default execution order. ** *******************************************************************************/ - public QProcessMetaData addOptionalStep(QStepMetaData step) + public QProcessMetaData withOptionalStep(QStepMetaData step) { if(this.steps == null) { @@ -283,6 +307,18 @@ public class QProcessMetaData implements QAppChildMetaData, MetaDataWithPermissi + /******************************************************************************* + ** add a step ONLY to the step map - NOT the list w/ default execution order. + ** + *******************************************************************************/ + @Deprecated(since = "withOptionalStep was added") + public QProcessMetaData addOptionalStep(QStepMetaData step) + { + return (withOptionalStep(step)); + } + + + /******************************************************************************* ** Setter for stepList ** @@ -299,7 +335,26 @@ public class QProcessMetaData implements QAppChildMetaData, MetaDataWithPermissi *******************************************************************************/ public QStepMetaData getStep(String stepName) { - return (steps.get(stepName)); + if(steps.containsKey(stepName)) + { + return steps.get(stepName); + } + + for(QStepMetaData step : steps.values()) + { + if(step instanceof QStateMachineStep stateMachineStep) + { + for(QStepMetaData subStep : stateMachineStep.getSubSteps()) + { + if(subStep.getName().equals(stepName)) + { + return (subStep); + } + } + } + } + + return (null); } @@ -780,4 +835,35 @@ public class QProcessMetaData implements QAppChildMetaData, MetaDataWithPermissi return (this); } + + + /******************************************************************************* + ** Getter for stepFlow + *******************************************************************************/ + public ProcessStepFlow getStepFlow() + { + return (this.stepFlow); + } + + + + /******************************************************************************* + ** Setter for stepFlow + *******************************************************************************/ + public void setStepFlow(ProcessStepFlow stepFlow) + { + this.stepFlow = stepFlow; + } + + + + /******************************************************************************* + ** Fluent setter for stepFlow + *******************************************************************************/ + public QProcessMetaData withStepFlow(ProcessStepFlow stepFlow) + { + this.stepFlow = stepFlow; + return (this); + } + } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/processes/QStateMachineStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/processes/QStateMachineStep.java new file mode 100644 index 00000000..dd370142 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/processes/QStateMachineStep.java @@ -0,0 +1,188 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.metadata.processes; + + +import java.util.ArrayList; +import java.util.List; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.utils.StringUtils; + + +/******************************************************************************* + ** A step for a state-machine flow based Process. + ** + ** Consists of 1 or 2 sub-steps, which are frontend and/or backend. + *******************************************************************************/ +public class QStateMachineStep extends QStepMetaData +{ + private List subSteps = new ArrayList<>(); + + private String defaultNextStepName; + + + + /******************************************************************************* + ** Constructor + ** + *******************************************************************************/ + private QStateMachineStep(List subSteps) + { + setStepType("stateMachine"); + this.subSteps.addAll(subSteps); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + public static QStateMachineStep frontendOnly(String name, QFrontendStepMetaData frontendStepMetaData) + { + if(!StringUtils.hasContent(frontendStepMetaData.getName())) + { + frontendStepMetaData.setName(name + ".frontend"); + } + + return (new QStateMachineStep(List.of(frontendStepMetaData)).withName(name)); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + public static QStateMachineStep backendOnly(String name, QBackendStepMetaData backendStepMetaData) + { + if(!StringUtils.hasContent(backendStepMetaData.getName())) + { + backendStepMetaData.setName(name + ".backend"); + } + + return (new QStateMachineStep(List.of(backendStepMetaData)).withName(name)); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + public static QStateMachineStep frontendThenBackend(String name, QFrontendStepMetaData frontendStepMetaData, QBackendStepMetaData backendStepMetaData) + { + if(!StringUtils.hasContent(frontendStepMetaData.getName())) + { + frontendStepMetaData.setName(name + ".frontend"); + } + + if(!StringUtils.hasContent(backendStepMetaData.getName())) + { + backendStepMetaData.setName(name + ".backend"); + } + + return (new QStateMachineStep(List.of(frontendStepMetaData, backendStepMetaData)).withName(name)); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public QStateMachineStep withName(String name) + { + super.withName(name); + return (this); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public QStateMachineStep withLabel(String label) + { + super.withLabel(label); + return (this); + } + + + + /******************************************************************************* + ** Getter for subSteps + ** + *******************************************************************************/ + public List getSubSteps() + { + return subSteps; + } + + + + /******************************************************************************* + ** Getter for defaultNextStepName + *******************************************************************************/ + public String getDefaultNextStepName() + { + return (this.defaultNextStepName); + } + + + + /******************************************************************************* + ** Setter for defaultNextStepName + *******************************************************************************/ + public void setDefaultNextStepName(String defaultNextStepName) + { + this.defaultNextStepName = defaultNextStepName; + } + + + + /******************************************************************************* + ** Fluent setter for defaultNextStepName + *******************************************************************************/ + public QStateMachineStep withDefaultNextStepName(String defaultNextStepName) + { + this.defaultNextStepName = defaultNextStepName; + return (this); + } + + + + /******************************************************************************* + ** Get a list of all of the input fields used by this step (all of its sub-steps) + *******************************************************************************/ + @JsonIgnore + @Override + public List getInputFields() + { + List rs = new ArrayList<>(); + for(QStepMetaData subStep : subSteps) + { + rs.addAll(subStep.getInputFields()); + } + return (rs); + } + +} diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/processes/RunProcessActionTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/processes/RunProcessActionTest.java new file mode 100644 index 00000000..1d989f96 --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/processes/RunProcessActionTest.java @@ -0,0 +1,327 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.actions.processes; + + +import java.util.ArrayList; +import java.util.List; +import com.kingsrook.qqq.backend.core.BaseTest; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessOutput; +import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReferenceLambda; +import com.kingsrook.qqq.backend.core.model.metadata.processes.ProcessStepFlow; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendStepMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QStateMachineStep; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.junit.jupiter.api.Assertions.assertEquals; + + +/******************************************************************************* + ** Unit test for RunProcessAction + *******************************************************************************/ +class RunProcessActionTest extends BaseTest +{ + private static List log = new ArrayList<>(); + + + + /******************************************************************************* + ** + *******************************************************************************/ + @BeforeEach + void beforeEach() + { + log.clear(); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testStateMachineTwoBackendSteps() throws QException + { + QProcessMetaData process = new QProcessMetaData().withName("test") + + ///////////////////////////////////////////////////////////////// + // two-steps - a, points at b; b has no next-step, so it exits // + ///////////////////////////////////////////////////////////////// + .addStep(QStateMachineStep.backendOnly("a", new QBackendStepMetaData().withName("aBackend") + .withCode(new QCodeReferenceLambda((runBackendStepInput, runBackendStepOutput) -> + { + log.add("in StepA"); + runBackendStepOutput.getProcessState().setNextStepName("b"); + })))) + + .addStep(QStateMachineStep.backendOnly("b", new QBackendStepMetaData().withName("bBackend") + .withCode(new QCodeReferenceLambda((runBackendStepInput, runBackendStepOutput) -> + { + log.add("in StepB"); + })))) + + .withStepFlow(ProcessStepFlow.STATE_MACHINE); + + QContext.getQInstance().addProcess(process); + + RunProcessInput input = new RunProcessInput(); + input.setProcessName("test"); + input.setFrontendStepBehavior(RunProcessInput.FrontendStepBehavior.BREAK); + + RunProcessOutput runProcessOutput = new RunProcessAction().execute(input); + assertEquals(List.of("in StepA", "in StepB"), log); + assertThat(runProcessOutput.getProcessState().getNextStepName()).isEmpty(); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testStateMachineTwoFrontendOnlySteps() throws QException + { + QProcessMetaData process = new QProcessMetaData().withName("test") + + .addStep(QStateMachineStep.frontendOnly("a", new QFrontendStepMetaData().withName("aFrontend")).withDefaultNextStepName("b")) + .addStep(QStateMachineStep.frontendOnly("b", new QFrontendStepMetaData().withName("bFrontend"))) + + .withStepFlow(ProcessStepFlow.STATE_MACHINE); + + QContext.getQInstance().addProcess(process); + + RunProcessInput input = new RunProcessInput(); + input.setProcessName("test"); + input.setFrontendStepBehavior(RunProcessInput.FrontendStepBehavior.BREAK); + + RunProcessOutput runProcessOutput = new RunProcessAction().execute(input); + assertThat(runProcessOutput.getProcessState().getNextStepName()) + .isPresent().get() + .isEqualTo("aFrontend"); + + ///////////////////////////// + // resume after a, go to b // + ///////////////////////////// + input.setStartAfterStep("aFrontend"); + runProcessOutput = new RunProcessAction().execute(input); + assertThat(runProcessOutput.getProcessState().getNextStepName()) + .isPresent().get() + .isEqualTo("bFrontend"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testStateMachineOneBackendStepReferencingItselfDoesNotInfiniteLoop() throws QException + { + QProcessMetaData process = new QProcessMetaData().withName("test") + + /////////////////////////////////////////////////////////////// + // set up step that always points back at itself. // + // since it never goes to the frontend, it'll stack overflow // + // (though we'll catch it ourselves before JVM does) // + /////////////////////////////////////////////////////////////// + .addStep(QStateMachineStep.backendOnly("a", new QBackendStepMetaData().withName("aBackend") + .withCode(new QCodeReferenceLambda((runBackendStepInput, runBackendStepOutput) -> + { + log.add("in StepA"); + runBackendStepOutput.getProcessState().setNextStepName("a"); + })))) + + .withStepFlow(ProcessStepFlow.STATE_MACHINE); + + QContext.getQInstance().addProcess(process); + + RunProcessInput input = new RunProcessInput(); + input.setProcessName("test"); + input.setFrontendStepBehavior(RunProcessInput.FrontendStepBehavior.BREAK); + + assertThatThrownBy(() -> new RunProcessAction().execute(input)) + .isInstanceOf(QException.class) + .hasMessageContaining("maxStateMachineProcessStepFlowStackDepth of 20"); + + /////////////////////////////////////////////////// + // make sure we can set a custom max-stack-depth // + /////////////////////////////////////////////////// + input.addValue("maxStateMachineProcessStepFlowStackDepth", 5); + assertThatThrownBy(() -> new RunProcessAction().execute(input)) + .isInstanceOf(QException.class) + .hasMessageContaining("maxStateMachineProcessStepFlowStackDepth of 5"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testStateMachineTwoBackendStepsReferencingEachOtherDoesNotInfiniteLoop() throws QException + { + QProcessMetaData process = new QProcessMetaData().withName("test") + + /////////////////////////////////////////////////////////////// + // set up two steps that always points back at each other. // + // since it never goes to the frontend, it'll stack overflow // + // (though we'll catch it ourselves before JVM does) // + /////////////////////////////////////////////////////////////// + .addStep(QStateMachineStep.backendOnly("a", new QBackendStepMetaData().withName("aBackend") + .withCode(new QCodeReferenceLambda((runBackendStepInput, runBackendStepOutput) -> + { + log.add("in StepA"); + runBackendStepOutput.getProcessState().setNextStepName("b"); + })))) + + .addStep(QStateMachineStep.backendOnly("b", new QBackendStepMetaData().withName("bBackend") + .withCode(new QCodeReferenceLambda((runBackendStepInput, runBackendStepOutput) -> + { + log.add("in StepB"); + runBackendStepOutput.getProcessState().setNextStepName("a"); + })))) + + .withStepFlow(ProcessStepFlow.STATE_MACHINE); + + QContext.getQInstance().addProcess(process); + + RunProcessInput input = new RunProcessInput(); + input.setProcessName("test"); + input.setFrontendStepBehavior(RunProcessInput.FrontendStepBehavior.BREAK); + + assertThatThrownBy(() -> new RunProcessAction().execute(input)) + .isInstanceOf(QException.class) + .hasMessageContaining("maxStateMachineProcessStepFlowStackDepth of 20"); + + /////////////////////////////////////////////////// + // make sure we can set a custom max-stack-depth // + /////////////////////////////////////////////////// + input.addValue("maxStateMachineProcessStepFlowStackDepth", 5); + assertThatThrownBy(() -> new RunProcessAction().execute(input)) + .isInstanceOf(QException.class) + .hasMessageContaining("maxStateMachineProcessStepFlowStackDepth of 5"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testStateSequenceOfFrontendAndBackendSteps() throws QException + { + QProcessMetaData process = new QProcessMetaData().withName("test") + + .addStep(QStateMachineStep.frontendThenBackend("a", + new QFrontendStepMetaData().withName("aFrontend"), + new QBackendStepMetaData().withName("aBackend") + .withCode(new QCodeReferenceLambda((runBackendStepInput, runBackendStepOutput) -> + { + log.add("in StepA"); + runBackendStepOutput.getProcessState().setNextStepName("b"); + })))) + + .addStep(QStateMachineStep.frontendThenBackend("b", + new QFrontendStepMetaData().withName("bFrontend"), + new QBackendStepMetaData().withName("bBackend") + .withCode(new QCodeReferenceLambda((runBackendStepInput, runBackendStepOutput) -> + { + log.add("in StepB"); + runBackendStepOutput.getProcessState().setNextStepName("c"); + })))) + + .addStep(QStateMachineStep.frontendThenBackend("c", + new QFrontendStepMetaData().withName("cFrontend"), + new QBackendStepMetaData().withName("cBackend") + .withCode(new QCodeReferenceLambda((runBackendStepInput, runBackendStepOutput) -> + { + log.add("in StepC"); + runBackendStepOutput.getProcessState().setNextStepName("d"); + })))) + + .addStep(QStateMachineStep.frontendOnly("d", + new QFrontendStepMetaData().withName("dFrontend"))) + + .withStepFlow(ProcessStepFlow.STATE_MACHINE); + + QContext.getQInstance().addProcess(process); + + RunProcessInput input = new RunProcessInput(); + input.setProcessName("test"); + input.setFrontendStepBehavior(RunProcessInput.FrontendStepBehavior.BREAK); + + //////////////////////////////////////////////////////// + // start the process - we should be sent to aFrontend // + //////////////////////////////////////////////////////// + RunProcessOutput runProcessOutput = new RunProcessAction().execute(input); + assertThat(runProcessOutput.getProcessState().getNextStepName()) + .isPresent().get() + .isEqualTo("aFrontend"); + + /////////////////////////////////////////////////////////////////////////////////////////// + // resume after aFrontend - we should run StepA (backend), and then be sent to bFrontend // + /////////////////////////////////////////////////////////////////////////////////////////// + input.setStartAfterStep("aFrontend"); + runProcessOutput = new RunProcessAction().execute(input); + assertEquals(List.of("in StepA"), log); + assertThat(runProcessOutput.getProcessState().getNextStepName()) + .isPresent().get() + .isEqualTo("bFrontend"); + + /////////////////////////////////////////////////////////////////////////////////////////// + // resume after bFrontend - we should run StepB (backend), and then be sent to cFrontend // + /////////////////////////////////////////////////////////////////////////////////////////// + input.setStartAfterStep("bFrontend"); + runProcessOutput = new RunProcessAction().execute(input); + assertEquals(List.of("in StepA", "in StepB"), log); + assertThat(runProcessOutput.getProcessState().getNextStepName()) + .isPresent().get() + .isEqualTo("cFrontend"); + + /////////////////////////////////////////////////////////////////////////////////////////// + // resume after cFrontend - we should run StepC (backend), and then be sent to dFrontend // + /////////////////////////////////////////////////////////////////////////////////////////// + input.setStartAfterStep("cFrontend"); + runProcessOutput = new RunProcessAction().execute(input); + assertEquals(List.of("in StepA", "in StepB", "in StepC"), log); + assertThat(runProcessOutput.getProcessState().getNextStepName()) + .isPresent().get() + .isEqualTo("dFrontend"); + + //////////////////////////////////////////////////////////////////////////////////// + // if we resume again here, we'll be past the end of the process, so no next-step // + //////////////////////////////////////////////////////////////////////////////////// + input.setStartAfterStep("dFrontend"); + runProcessOutput = new RunProcessAction().execute(input); + assertEquals(List.of("in StepA", "in StepB", "in StepC"), log); + assertThat(runProcessOutput.getProcessState().getNextStepName()).isEmpty(); + + } + +} \ No newline at end of file