diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/QCodeLoader.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/QCodeLoader.java index 4b5fbd18..fbbc55c8 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/QCodeLoader.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/QCodeLoader.java @@ -119,7 +119,9 @@ public class QCodeLoader try { Class customizerClass = Class.forName(codeReference.getName()); - return ((T) customizerClass.getConstructor().newInstance()); + @SuppressWarnings("unchecked") + T t = (T) customizerClass.getConstructor().newInstance(); + return t; } catch(Exception e) { diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/TableCustomizers.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/TableCustomizers.java index e5899fad..ef5f77b1 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/TableCustomizers.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/TableCustomizers.java @@ -24,6 +24,7 @@ public enum TableCustomizers { POST_QUERY_RECORD(new TableCustomizer("postQueryRecord", Function.class, ((Object x) -> { + @SuppressWarnings("unchecked") Function function = (Function) x; QRecord output = function.apply(new QRecord()); }))); diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/processes/RunProcessAction.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/processes/RunProcessAction.java index 67d32747..39a9fc51 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/processes/RunProcessAction.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/processes/RunProcessAction.java @@ -82,15 +82,28 @@ public class RunProcessAction runProcessOutput.setProcessUUID(runProcessInput.getProcessUUID()); UUIDAndTypeStateKey stateKey = new UUIDAndTypeStateKey(UUID.fromString(runProcessInput.getProcessUUID()), StateType.PROCESS_STATUS); - ProcessState processState = primeProcessState(runProcessInput, stateKey); + ProcessState processState = primeProcessState(runProcessInput, stateKey, process); - // todo - custom routing - List stepList = getAvailableStepList(process, runProcessInput); try { + String lastStepName = runProcessInput.getStartAfterStep(); + STEP_LOOP: - for(QStepMetaData step : stepList) + while(true) { + /////////////////////////////////////////////////////////////////////////////////////////////////////// + // always refresh the step list - as any step that runs can modify it (in the process state). // + // this is why we don't do a loop over the step list - as we'd get ConcurrentModificationExceptions. // + /////////////////////////////////////////////////////////////////////////////////////////////////////// + List stepList = getAvailableStepList(processState, process, lastStepName); + if(stepList.isEmpty()) + { + break; + } + + QStepMetaData step = stepList.get(0); + lastStepName = step.getName(); + if(step instanceof QFrontendStepMetaData) { //////////////////////////////////////////////////////////////// @@ -127,6 +140,7 @@ public class RunProcessAction /////////////////////// // Run backend steps // /////////////////////// + LOG.info("Running backend step [" + step.getName() + "] in process [" + process.getName() + "]"); runBackendStep(runProcessInput, process, runProcessOutput, stateKey, backendStepMetaData, process, processState); } else @@ -169,7 +183,7 @@ public class RunProcessAction ** When we start running a process (or resuming it), get data in the RunProcessRequest ** either from the state provider (if they're found, for a resume). *******************************************************************************/ - ProcessState primeProcessState(RunProcessInput runProcessInput, UUIDAndTypeStateKey stateKey) throws QException + ProcessState primeProcessState(RunProcessInput runProcessInput, UUIDAndTypeStateKey stateKey, QProcessMetaData process) throws QException { Optional optionalProcessState = loadState(stateKey); if(optionalProcessState.isEmpty()) @@ -177,11 +191,13 @@ public class RunProcessAction if(runProcessInput.getStartAfterStep() == null) { /////////////////////////////////////////////////////////////////////////////////// - // this is fine - it means its our first time running in the backend. // + // this is fine - it means it's our first time running in the backend. // // Go ahead and store the state that we have (e.g., w/ initial records & values) // /////////////////////////////////////////////////////////////////////////////////// - storeState(stateKey, runProcessInput.getProcessState()); - optionalProcessState = Optional.of(runProcessInput.getProcessState()); + ProcessState processState = runProcessInput.getProcessState(); + processState.setStepList(process.getStepList().stream().map(QStepMetaData::getName).toList()); + storeState(stateKey, processState); + optionalProcessState = Optional.of(processState); } else { @@ -249,41 +265,63 @@ public class RunProcessAction /******************************************************************************* ** Get the list of steps which are eligible to run. *******************************************************************************/ - private List getAvailableStepList(QProcessMetaData process, RunProcessInput runProcessInput) + private List getAvailableStepList(ProcessState processState, QProcessMetaData process, String lastStep) throws QException { - if(runProcessInput.getStartAfterStep() == null) + if(lastStep == null) { - ///////////////////////////////////////////////////////////////////////////// - // if the caller did not supply a 'startAfterStep', then use the full list // - ///////////////////////////////////////////////////////////////////////////// - return (process.getStepList()); + /////////////////////////////////////////////////////////////////////// + // if the caller did not supply a 'lastStep', then use the full list // + /////////////////////////////////////////////////////////////////////// + return (stepNamesToSteps(process, processState.getStepList())); } else { - //////////////////////////////////////////////////////////////////////////////// - // else, loop until the startAfterStep is found, and return the ones after it // - //////////////////////////////////////////////////////////////////////////////// - boolean foundStartAfterStep = false; - List rs = new ArrayList<>(); + //////////////////////////////////////////////////////////////////////////// + // else, loop until the 'lastStep' is found, and return the ones after it // + //////////////////////////////////////////////////////////////////////////// + boolean foundLastStep = false; + List validStepNames = new ArrayList<>(); - for(QStepMetaData step : process.getStepList()) + for(String stepName : processState.getStepList()) { - if(foundStartAfterStep) + if(foundLastStep) { - rs.add(step); + validStepNames.add(stepName); } - if(step.getName().equals(runProcessInput.getStartAfterStep())) + if(stepName.equals(lastStep)) { - foundStartAfterStep = true; + foundLastStep = true; } } - return (rs); + return (stepNamesToSteps(process, validStepNames)); } } + /******************************************************************************* + ** + *******************************************************************************/ + private List stepNamesToSteps(QProcessMetaData process, List stepNames) throws QException + { + List result = new ArrayList<>(); + + for(String stepName : stepNames) + { + QStepMetaData step = process.getStep(stepName); + if(step == null) + { + throw(new QException("Could not find a step named [" + stepName + "] in this process.")); + } + result.add(step); + } + + return (result); + } + + + /******************************************************************************* ** Load an instance of the appropriate state provider ** diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/ProcessState.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/ProcessState.java index bdd87270..ccc95108 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/ProcessState.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/ProcessState.java @@ -38,6 +38,7 @@ public class ProcessState implements Serializable { private List records = new ArrayList<>(); private Map values = new HashMap<>(); + private List stepList = new ArrayList<>(); private Optional nextStepName = Optional.empty(); @@ -117,4 +118,25 @@ public class ProcessState implements Serializable this.nextStepName = Optional.empty(); } + + + /******************************************************************************* + ** Getter for stepList + ** + *******************************************************************************/ + public List getStepList() + { + return stepList; + } + + + + /******************************************************************************* + ** Setter for stepList + ** + *******************************************************************************/ + public void setStepList(List stepList) + { + this.stepList = stepList; + } } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/ProcessSummaryLine.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/ProcessSummaryLine.java new file mode 100644 index 00000000..db627b43 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/ProcessSummaryLine.java @@ -0,0 +1,180 @@ +package com.kingsrook.qqq.backend.core.model.actions.processes; + + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.List; + + +/******************************************************************************* + ** For processes that may show a review & result screen, this class provides a + ** standard way to summarize information about the records in the process. + ** + *******************************************************************************/ +public class ProcessSummaryLine implements Serializable +{ + private Status status; + private Integer count; + private String message; + + ////////////////////////////////////////////////////////////////////////// + // using ArrayList, because we need to be Serializable, and List is not // + ////////////////////////////////////////////////////////////////////////// + private ArrayList primaryKeys; + + + + /******************************************************************************* + ** + *******************************************************************************/ + public ProcessSummaryLine(Status status, Integer count, String message, ArrayList primaryKeys) + { + this.status = status; + this.count = count; + this.message = message; + this.primaryKeys = primaryKeys; + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public ProcessSummaryLine(Status status, Integer count, String message) + { + this.status = status; + this.count = count; + this.message = message; + } + + + + /******************************************************************************* + ** Getter for status + ** + *******************************************************************************/ + public Status getStatus() + { + return status; + } + + + + /******************************************************************************* + ** Setter for status + ** + *******************************************************************************/ + public void setStatus(Status status) + { + this.status = status; + } + + + + /******************************************************************************* + ** Getter for primaryKeys + ** + *******************************************************************************/ + public List getPrimaryKeys() + { + return primaryKeys; + } + + + + /******************************************************************************* + ** Setter for primaryKeys + ** + *******************************************************************************/ + public void setPrimaryKeys(ArrayList primaryKeys) + { + this.primaryKeys = primaryKeys; + } + + + + /******************************************************************************* + ** Getter for count + ** + *******************************************************************************/ + public Integer getCount() + { + return count; + } + + + + /******************************************************************************* + ** Setter for count + ** + *******************************************************************************/ + public void setCount(Integer count) + { + this.count = count; + } + + + + /******************************************************************************* + ** Getter for message + ** + *******************************************************************************/ + public String getMessage() + { + return message; + } + + + + /******************************************************************************* + ** Setter for message + ** + *******************************************************************************/ + public void setMessage(String message) + { + this.message = message; + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public void incrementCount() + { + if(count == null) + { + count = 0; + } + count++; + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public void incrementCountAndAddPrimaryKey(Serializable primaryKey) + { + incrementCount(); + + if(primaryKeys == null) + { + primaryKeys = new ArrayList<>(); + } + primaryKeys.add(primaryKey); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public void addSelfToListIfAnyCount(ArrayList rs) + { + if(count != null && count > 0) + { + rs.add(this); + } + } +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/RunBackendStepInput.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/RunBackendStepInput.java index 712e1335..dd85679b 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/RunBackendStepInput.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/RunBackendStepInput.java @@ -354,7 +354,30 @@ public class RunBackendStepInput extends AbstractActionInput *******************************************************************************/ public String getValueString(String fieldName) { - return ((String) getValue(fieldName)); + return (ValueUtils.getValueAsString(getValue(fieldName))); + } + + + + /******************************************************************************* + ** Getter for a single field's value + ** + *******************************************************************************/ + public Boolean getValueBoolean(String fieldName) + { + return (ValueUtils.getValueAsBoolean(getValue(fieldName))); + } + + + + /******************************************************************************* + ** Getter for a single field's value as a primitive boolean + ** + *******************************************************************************/ + public boolean getValue_boolean(String fieldName) + { + Boolean valueAsBoolean = ValueUtils.getValueAsBoolean(getValue(fieldName)); + return (valueAsBoolean != null && valueAsBoolean); } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/Status.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/Status.java new file mode 100644 index 00000000..60eb77ad --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/Status.java @@ -0,0 +1,13 @@ +package com.kingsrook.qqq.backend.core.model.actions.processes; + + +/******************************************************************************* + ** Simple status enum - initially for statusesqqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/Status.java in process status lines. + *******************************************************************************/ +public enum Status +{ + OK, + WARNING, + ERROR, + INFO +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/data/QRecord.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/data/QRecord.java index 77cdd6e8..29903fc3 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/data/QRecord.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/data/QRecord.java @@ -34,6 +34,7 @@ import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.utils.ValueUtils; +import org.apache.commons.lang.SerializationUtils; /******************************************************************************* @@ -87,16 +88,60 @@ public class QRecord implements Serializable /******************************************************************************* ** Copy constructor. - ** TODO ... should this do deep copies? + ** *******************************************************************************/ + @SuppressWarnings("unchecked") public QRecord(QRecord record) { this.tableName = record.tableName; this.recordLabel = record.recordLabel; - this.values = record.values; - this.displayValues = record.displayValues; - this.backendDetails = record.backendDetails; - this.errors = record.errors; + + this.values = doDeepCopy(record.values); + this.displayValues = doDeepCopy(record.displayValues); + this.backendDetails = doDeepCopy(record.backendDetails); + this.errors = doDeepCopy(record.errors); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @SuppressWarnings({ "rawtypes", "unchecked" }) + private Map doDeepCopy(Map map) + { + if(map == null) + { + return (null); + } + + if(map instanceof Serializable serializableMap) + { + return (Map) SerializationUtils.clone(serializableMap); + } + + return (new LinkedHashMap(map)); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @SuppressWarnings({ "rawtypes", "unchecked" }) + private List doDeepCopy(List list) + { + if(list == null) + { + return (null); + } + + if(list instanceof Serializable serializableList) + { + return (List) SerializationUtils.clone(serializableList); + } + + return (new ArrayList(list)); } @@ -142,7 +187,6 @@ public class QRecord implements Serializable - /******************************************************************************* ** *******************************************************************************/ @@ -209,6 +253,7 @@ public class QRecord implements Serializable } + /******************************************************************************* ** Fluent setter for recordLabel ** diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/QBackendMetaData.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/QBackendMetaData.java index ee9c53f7..ea4cd840 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/QBackendMetaData.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/QBackendMetaData.java @@ -75,11 +75,10 @@ public class QBackendMetaData /******************************************************************************* ** Fluent setter, returning generically, to help sub-class fluent flows *******************************************************************************/ - @SuppressWarnings("unchecked") - public T withName(String name) + public QBackendMetaData withName(String name) { this.name = name; - return (T) this; + return this; } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/QInstance.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/QInstance.java index 3712d077..e2d4988f 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/QInstance.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/QInstance.java @@ -35,6 +35,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; import com.kingsrook.qqq.backend.core.model.metadata.processes.QStepMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.modules.authentication.metadata.QAuthenticationMetaData; +import com.kingsrook.qqq.backend.core.utils.StringUtils; /******************************************************************************* @@ -129,6 +130,10 @@ public class QInstance *******************************************************************************/ public void addBackend(String name, QBackendMetaData backend) { + if(!StringUtils.hasContent(name)) + { + throw (new IllegalArgumentException("Attempted to add a backend without a name.")); + } if(this.backends.containsKey(name)) { throw (new IllegalArgumentException("Attempted to add a second backend with name: " + name)); @@ -163,6 +168,10 @@ public class QInstance *******************************************************************************/ public void addTable(String name, QTableMetaData table) { + if(!StringUtils.hasContent(name)) + { + throw (new IllegalArgumentException("Attempted to add a table without a name.")); + } if(this.tables.containsKey(name)) { throw (new IllegalArgumentException("Attempted to add a second table with name: " + name)); @@ -202,6 +211,10 @@ public class QInstance *******************************************************************************/ public void addPossibleValueSource(String name, QPossibleValueSource possibleValueSource) { + if(!StringUtils.hasContent(name)) + { + throw (new IllegalArgumentException("Attempted to add a possibleValueSource without a name.")); + } if(this.possibleValueSources.containsKey(name)) { throw (new IllegalArgumentException("Attempted to add a second possibleValueSource with name: " + name)); @@ -252,6 +265,10 @@ public class QInstance *******************************************************************************/ public void addProcess(String name, QProcessMetaData process) { + if(!StringUtils.hasContent(name)) + { + throw (new IllegalArgumentException("Attempted to add a process without a name.")); + } if(this.processes.containsKey(name)) { throw (new IllegalArgumentException("Attempted to add a second process with name: " + name)); @@ -286,6 +303,10 @@ public class QInstance *******************************************************************************/ public void addApp(String name, QAppMetaData app) { + if(!StringUtils.hasContent(name)) + { + throw (new IllegalArgumentException("Attempted to add an app without a name.")); + } if(this.apps.containsKey(name)) { throw (new IllegalArgumentException("Attempted to add a second app with name: " + name)); diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/processes/QComponentType.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/processes/QComponentType.java index 5c0ba61e..70e6a191 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/processes/QComponentType.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/processes/QComponentType.java @@ -28,7 +28,12 @@ package com.kingsrook.qqq.backend.core.model.metadata.processes; public enum QComponentType { HELP_TEXT, - BULK_EDIT_FORM; + BULK_EDIT_FORM, + VALIDATION_REVIEW_SCREEN, + EDIT_FORM, + VIEW_FORM, + RECORD_LIST, + PROCESS_SUMMARY_RESULTS; /////////////////////////////////////////////////////////////////////////// // keep these values in sync with QComponentType.ts in qqq-frontend-core // /////////////////////////////////////////////////////////////////////////// diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/processes/QFunctionOutputMetaData.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/processes/QFunctionOutputMetaData.java index 024b0f7e..7df56fa5 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/processes/QFunctionOutputMetaData.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/processes/QFunctionOutputMetaData.java @@ -33,7 +33,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; *******************************************************************************/ public class QFunctionOutputMetaData { - private QRecordListMetaData recordListMetaData; + private QRecordListMetaData recordListMetaData; private List fieldList; @@ -106,11 +106,12 @@ public class QFunctionOutputMetaData + /******************************************************************************* - ** Setter for fieldList + ** Fluently add a field to the list ** *******************************************************************************/ - public QFunctionOutputMetaData addField(QFieldMetaData field) + public QFunctionOutputMetaData withField(QFieldMetaData field) { if(this.fieldList == null) { diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/processes/QProcessMetaData.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/processes/QProcessMetaData.java index a9fa5611..b2bd3f8c 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/processes/QProcessMetaData.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/processes/QProcessMetaData.java @@ -23,7 +23,9 @@ package com.kingsrook.qqq.backend.core.model.metadata.processes; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; +import java.util.Map; import com.fasterxml.jackson.annotation.JsonIgnore; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppChildMetaData; @@ -41,7 +43,8 @@ public class QProcessMetaData implements QAppChildMetaData private String tableName; private boolean isHidden = false; - private List stepList; + private List stepList; // these are the steps that are ran, by-default, in the order they are ran in + private Map steps; // this is the full map of possible steps private String parentAppName; private QIcon icon; @@ -167,14 +170,18 @@ public class QProcessMetaData implements QAppChildMetaData *******************************************************************************/ public QProcessMetaData withStepList(List stepList) { - this.stepList = stepList; + if(stepList != null) + { + stepList.forEach(this::addStep); + } + return (this); } /******************************************************************************* - ** Setter for stepList + ** add a step to the stepList and map ** *******************************************************************************/ public QProcessMetaData addStep(QStepMetaData step) @@ -184,6 +191,30 @@ public class QProcessMetaData implements QAppChildMetaData this.stepList = new ArrayList<>(); } this.stepList.add(step); + + if(this.steps == null) + { + this.steps = new HashMap<>(); + } + this.steps.put(step.getName(), step); + + return (this); + } + + + + /******************************************************************************* + ** add a step ONLY to the step map - NOT the list w/ default execution order. + ** + *******************************************************************************/ + public QProcessMetaData addOptionalStep(QStepMetaData step) + { + if(this.steps == null) + { + this.steps = new HashMap<>(); + } + this.steps.put(step.getName(), step); + return (this); } @@ -205,15 +236,7 @@ public class QProcessMetaData implements QAppChildMetaData *******************************************************************************/ public QStepMetaData getStep(String stepName) { - for(QStepMetaData step : stepList) - { - if(step.getName().equals(stepName)) - { - return (step); - } - } - - return (null); + return (steps.get(stepName)); } @@ -245,9 +268,9 @@ public class QProcessMetaData implements QAppChildMetaData public List getInputFields() { List rs = new ArrayList<>(); - if(stepList != null) + if(steps != null) { - for(QStepMetaData step : stepList) + for(QStepMetaData step : steps.values()) { rs.addAll(step.getInputFields()); } @@ -264,9 +287,9 @@ public class QProcessMetaData implements QAppChildMetaData public List getOutputFields() { List rs = new ArrayList<>(); - if(stepList != null) + if(steps != null) { - for(QStepMetaData step : stepList) + for(QStepMetaData step : steps.values()) { rs.addAll(step.getOutputFields()); } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/basic/BasicETLProcess.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/basic/BasicETLProcess.java index 1cdcb132..8dbce297 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/basic/BasicETLProcess.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/basic/BasicETLProcess.java @@ -83,7 +83,7 @@ public class BasicETLProcess .withInputData(new QFunctionInputMetaData() .withField(new QFieldMetaData(FIELD_DESTINATION_TABLE, QFieldType.STRING))) .withOutputMetaData(new QFunctionOutputMetaData() - .addField(new QFieldMetaData(FIELD_RECORD_COUNT, QFieldType.INTEGER))); + .withField(new QFieldMetaData(FIELD_RECORD_COUNT, QFieldType.INTEGER))); return new QProcessMetaData() .withName(PROCESS_NAME) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamed/StreamedETLProcess.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamed/StreamedETLProcess.java index 2fa636e7..bb56cf25 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamed/StreamedETLProcess.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamed/StreamedETLProcess.java @@ -66,7 +66,7 @@ public class StreamedETLProcess .withField(new QFieldMetaData(FIELD_MAPPING_JSON, QFieldType.STRING)) .withField(new QFieldMetaData(FIELD_DESTINATION_TABLE, QFieldType.STRING))) .withOutputMetaData(new QFunctionOutputMetaData() - .addField(new QFieldMetaData(FIELD_RECORD_COUNT, QFieldType.INTEGER))); + .withField(new QFieldMetaData(FIELD_RECORD_COUNT, QFieldType.INTEGER))); return new QProcessMetaData() .withName(PROCESS_NAME) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/ProcessSummaryProviderInterface.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/ProcessSummaryProviderInterface.java new file mode 100644 index 00000000..a4b74fe4 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/ProcessSummaryProviderInterface.java @@ -0,0 +1,19 @@ +package com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend; + + +import java.util.ArrayList; +import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLine; + + +/******************************************************************************* + ** + *******************************************************************************/ +public interface ProcessSummaryProviderInterface +{ + + /******************************************************************************* + ** + *******************************************************************************/ + ArrayList getProcessSummary(boolean isForResultScreen); + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLExecuteStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLExecuteStep.java index f8d2f679..3a5b1126 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLExecuteStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLExecuteStep.java @@ -33,7 +33,6 @@ import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; import com.kingsrook.qqq.backend.core.model.data.QRecord; -import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamed.StreamedETLProcess; /******************************************************************************* @@ -80,7 +79,7 @@ public class StreamedETLExecuteStep extends BaseStreamedETLStep implements Backe () -> (consumeRecordsFromPipe(recordPipe, transformStep, loadStep, runBackendStepInput, runBackendStepOutput, loadedRecordList)) ); - runBackendStepOutput.addValue(StreamedETLProcess.FIELD_RECORD_COUNT, recordCount); + runBackendStepOutput.addValue(StreamedETLWithFrontendProcess.FIELD_RECORD_COUNT, recordCount); runBackendStepOutput.setRecords(loadedRecordList); ///////////////////// @@ -90,6 +89,15 @@ public class StreamedETLExecuteStep extends BaseStreamedETLStep implements Backe { transaction.get().commit(); } + + if(transformStep instanceof ProcessSummaryProviderInterface processSummaryProvider) + { + ////////////////////////////////////////////////////////////////////////////////////////////// + // get the process summary from the ... transform step? the load step? each knows some... // + // TODO!! // + ////////////////////////////////////////////////////////////////////////////////////////////// + runBackendStepOutput.addValue(StreamedETLWithFrontendProcess.FIELD_PROCESS_SUMMARY, processSummaryProvider.getProcessSummary(true)); + } } catch(Exception e) { @@ -121,7 +129,7 @@ public class StreamedETLExecuteStep extends BaseStreamedETLStep implements Backe *******************************************************************************/ private int consumeRecordsFromPipe(RecordPipe recordPipe, AbstractTransformStep transformStep, AbstractLoadStep loadStep, RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput, List loadedRecordList) throws QException { - Integer totalRows = runBackendStepInput.getValueInteger(StreamedETLProcess.FIELD_RECORD_COUNT); + Integer totalRows = runBackendStepInput.getValueInteger(StreamedETLWithFrontendProcess.FIELD_RECORD_COUNT); if(totalRows != null) { runBackendStepInput.getAsyncJobCallback().updateStatus(currentRowCount, totalRows); diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLPreviewStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLPreviewStep.java index ee920b3c..435480ef 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLPreviewStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLPreviewStep.java @@ -32,6 +32,8 @@ import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInpu import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamed.StreamedETLProcess; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; /******************************************************************************* @@ -39,6 +41,8 @@ import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamed.Str *******************************************************************************/ public class StreamedETLPreviewStep extends BaseStreamedETLStep implements BackendStep { + private static final Logger LOG = LogManager.getLogger(StreamedETLPreviewStep.class); + /******************************************************************************* @@ -48,29 +52,72 @@ public class StreamedETLPreviewStep extends BaseStreamedETLStep implements Backe @SuppressWarnings("checkstyle:indentation") public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException { - RecordPipe recordPipe = new RecordPipe(); - AbstractExtractStep extractStep = getExtractStep(runBackendStepInput); - extractStep.setLimit(PROCESS_OUTPUT_RECORD_LIST_LIMIT); // todo - make this an input? - extractStep.setRecordPipe(recordPipe); + Integer limit = PROCESS_OUTPUT_RECORD_LIST_LIMIT; // todo - use a field instead of hard-coded here? + + ////////////////////////////////////////////////////////////////////////////////////////////////////////// + // if the do-full-validation flag has already been set, then do the validation step instead of this one // + ////////////////////////////////////////////////////////////////////////////////////////////////////////// + boolean supportsFullValidation = runBackendStepInput.getValue_boolean(StreamedETLWithFrontendProcess.FIELD_SUPPORTS_FULL_VALIDATION); + boolean doFullValidation = runBackendStepInput.getValue_boolean(StreamedETLWithFrontendProcess.FIELD_DO_FULL_VALIDATION); + if(supportsFullValidation && doFullValidation) + { + skipToValidateStep(runBackendStepOutput); + return; + } /////////////////////////////////////////// // request a count from the extract step // /////////////////////////////////////////// - Integer recordCount = extractStep.doCount(runBackendStepInput); + AbstractExtractStep extractStep = getExtractStep(runBackendStepInput); + Integer recordCount = extractStep.doCount(runBackendStepInput); runBackendStepOutput.addValue(StreamedETLProcess.FIELD_RECORD_COUNT, recordCount); + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // if the count is less than the normal limit here, and this process supports validation, then go straight to the validation step // + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // todo - maybe some future version we do this - maybe based on a user-preference + // if(supportsFullValidation && recordCount <= limit) + // { + // skipToValidateStep(runBackendStepOutput); + // return; + // } + + //////////////////////////////////////////////////////// + // proceed with a doing a limited extract & transform // + //////////////////////////////////////////////////////// + RecordPipe recordPipe = new RecordPipe(); + extractStep.setLimit(limit); + extractStep.setRecordPipe(recordPipe); + AbstractTransformStep transformStep = getTransformStep(runBackendStepInput); - List transformedRecordList = new ArrayList<>(); + List previewRecordList = new ArrayList<>(); new AsyncRecordPipeLoop().run("StreamedETL>Preview>ExtractStep", PROCESS_OUTPUT_RECORD_LIST_LIMIT, recordPipe, (status) -> { extractStep.run(runBackendStepInput, runBackendStepOutput); return (runBackendStepOutput); }, - () -> (consumeRecordsFromPipe(recordPipe, transformStep, runBackendStepInput, runBackendStepOutput, transformedRecordList)) + () -> (consumeRecordsFromPipe(recordPipe, transformStep, runBackendStepInput, runBackendStepOutput, previewRecordList)) ); - runBackendStepOutput.setRecords(transformedRecordList); + runBackendStepOutput.setRecords(previewRecordList); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private void skipToValidateStep(RunBackendStepOutput runBackendStepOutput) + { + LOG.info("Skipping to validation step"); + runBackendStepOutput.addValue(StreamedETLWithFrontendProcess.FIELD_DO_FULL_VALIDATION, true); + ArrayList stepList = new ArrayList<>(runBackendStepOutput.getProcessState().getStepList()); + System.out.println("Step list pre: " + stepList); + stepList.removeIf(s -> s.equals(StreamedETLWithFrontendProcess.STEP_NAME_REVIEW)); + stepList.add(stepList.indexOf(StreamedETLWithFrontendProcess.STEP_NAME_VALIDATE) + 1, StreamedETLWithFrontendProcess.STEP_NAME_REVIEW); + runBackendStepOutput.getProcessState().setStepList(stepList); + System.out.println("Step list post: " + stepList); } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLValidateStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLValidateStep.java new file mode 100644 index 00000000..d5e925d6 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLValidateStep.java @@ -0,0 +1,155 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend; + + +import java.util.ArrayList; +import java.util.List; +import com.kingsrook.qqq.backend.core.actions.async.AsyncRecordPipeLoop; +import com.kingsrook.qqq.backend.core.actions.processes.BackendStep; +import com.kingsrook.qqq.backend.core.actions.reporting.RecordPipe; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + + +/******************************************************************************* + ** Backend step to do a full validation of a streamed ETL job + *******************************************************************************/ +public class StreamedETLValidateStep extends BaseStreamedETLStep implements BackendStep +{ + private static final Logger LOG = LogManager.getLogger(StreamedETLValidateStep.class); + + private int currentRowCount = 1; + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + @SuppressWarnings("checkstyle:indentation") + public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException + { + ///////////////////////////////////////////////////////////////////// + // check if we are supported in this process - if not, return noop // + ///////////////////////////////////////////////////////////////////// + boolean supportsFullValidation = runBackendStepInput.getValue_boolean(StreamedETLWithFrontendProcess.FIELD_SUPPORTS_FULL_VALIDATION); + if(!supportsFullValidation) + { + LOG.info("Process does not support validation, so skipping validation step"); + return; + } + + //////////////////////////////////////////////////////////////////////////////// + // check if we've been requested to run in this process - if not, return noop // + //////////////////////////////////////////////////////////////////////////////// + boolean doFullValidation = runBackendStepInput.getValue_boolean(StreamedETLWithFrontendProcess.FIELD_DO_FULL_VALIDATION); + if(!doFullValidation) + { + LOG.info("Not requested to do full validation, so skipping validation step"); + return; + } + + //////////////////////////////////////////////////////////////////////////////////////////////////////////// + // if we're proceeding with full validation, move the review step to be after validation in the step list // + //////////////////////////////////////////////////////////////////////////////////////////////////////////// + ArrayList stepList = new ArrayList<>(runBackendStepOutput.getProcessState().getStepList()); + System.out.println("Step list pre: " + stepList); + stepList.removeIf(s -> s.equals(StreamedETLWithFrontendProcess.STEP_NAME_REVIEW)); + stepList.add(stepList.indexOf(StreamedETLWithFrontendProcess.STEP_NAME_VALIDATE) + 1, StreamedETLWithFrontendProcess.STEP_NAME_REVIEW); + runBackendStepOutput.getProcessState().setStepList(stepList); + System.out.println("Step list post: " + stepList); + + ////////////////////////////////////////////////////////// + // basically repeat the preview step, but with no limit // + ////////////////////////////////////////////////////////// + RecordPipe recordPipe = new RecordPipe(); + AbstractExtractStep extractStep = getExtractStep(runBackendStepInput); + extractStep.setLimit(null); + extractStep.setRecordPipe(recordPipe); + + AbstractTransformStep transformStep = getTransformStep(runBackendStepInput); + if(!(transformStep instanceof ProcessSummaryProviderInterface processSummaryProvider)) + { + throw (new QException("Transform Step " + transformStep.getClass().getName() + " does not implement ProcessSummaryProviderInterface.")); + } + + List previewRecordList = new ArrayList<>(); + int recordCount = new AsyncRecordPipeLoop().run("StreamedETL>Preview>ValidateStep", PROCESS_OUTPUT_RECORD_LIST_LIMIT, recordPipe, (status) -> + { + extractStep.run(runBackendStepInput, runBackendStepOutput); + return (runBackendStepOutput); + }, + () -> (consumeRecordsFromPipe(recordPipe, transformStep, runBackendStepInput, runBackendStepOutput, previewRecordList)) + ); + + runBackendStepOutput.setRecords(previewRecordList); + runBackendStepOutput.addValue(StreamedETLWithFrontendProcess.FIELD_RECORD_COUNT, recordCount); + + ////////////////////////////////////////////////////// + // get the process summary from the validation step // + ////////////////////////////////////////////////////// + runBackendStepOutput.addValue(StreamedETLWithFrontendProcess.FIELD_VALIDATION_SUMMARY, processSummaryProvider.getProcessSummary(false)); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private int consumeRecordsFromPipe(RecordPipe recordPipe, AbstractTransformStep transformStep, RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput, List previewRecordList) throws QException + { + Integer totalRows = runBackendStepInput.getValueInteger(StreamedETLWithFrontendProcess.FIELD_RECORD_COUNT); + if(totalRows != null) + { + runBackendStepInput.getAsyncJobCallback().updateStatus(currentRowCount, totalRows); + } + + /////////////////////////////////// + // get the records from the pipe // + /////////////////////////////////// + List qRecords = recordPipe.consumeAvailableRecords(); + + ///////////////////////////////////////////////////// + // pass the records through the transform function // + ///////////////////////////////////////////////////// + transformStep.setInputRecordPage(qRecords); + transformStep.run(runBackendStepInput, runBackendStepOutput); + + /////////////////////////////////////////////////////// + // copy a small number of records to the output list // + /////////////////////////////////////////////////////// + int i = 0; + while(previewRecordList.size() < PROCESS_OUTPUT_RECORD_LIST_LIMIT && i < transformStep.getOutputRecordPage().size()) + { + previewRecordList.add(transformStep.getOutputRecordPage().get(i++)); + } + + currentRowCount += qRecords.size(); + return (qRecords.size()); + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLWithFrontendProcess.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLWithFrontendProcess.java index 20d33147..e8b2a57a 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLWithFrontendProcess.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLWithFrontendProcess.java @@ -22,11 +22,17 @@ package com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend; +import java.io.Serializable; +import java.util.HashMap; +import java.util.Map; import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QComponentType; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendComponentMetaData; import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendStepMetaData; import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionInputMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionOutputMetaData; import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; import com.kingsrook.qqq.backend.core.model.metadata.processes.QStepMetaData; @@ -51,60 +57,104 @@ import com.kingsrook.qqq.backend.core.model.metadata.processes.QStepMetaData; *******************************************************************************/ public class StreamedETLWithFrontendProcess { - public static final String PROCESS_NAME = "etl.streamedWithFrontend"; + public static final String STEP_NAME_PREVIEW = "preview"; + public static final String STEP_NAME_REVIEW = "review"; + public static final String STEP_NAME_VALIDATE = "validate"; + public static final String STEP_NAME_EXECUTE = "execute"; + public static final String STEP_NAME_RESULT = "result"; - public static final String STEP_NAME_PREVIEW = "preview"; - public static final String STEP_NAME_REVIEW = "review"; - public static final String STEP_NAME_EXECUTE = "execute"; - public static final String STEP_NAME_RESULT = "result"; + public static final String FIELD_EXTRACT_CODE = "extract"; // QCodeReference, of AbstractExtractStep + public static final String FIELD_TRANSFORM_CODE = "transform"; // QCodeReference, of AbstractTransformStep + public static final String FIELD_LOAD_CODE = "load"; // QCodeReference, of AbstractLoadStep - public static final String FIELD_EXTRACT_CODE = "extract"; - public static final String FIELD_TRANSFORM_CODE = "transform"; - public static final String FIELD_LOAD_CODE = "load"; + public static final String FIELD_SOURCE_TABLE = "sourceTable"; // String + public static final String FIELD_DESTINATION_TABLE = "destinationTable"; // String + public static final String FIELD_RECORD_COUNT = "recordCount"; // Integer + public static final String FIELD_DEFAULT_QUERY_FILTER = "defaultQueryFilter"; // QQueryFilter or String (json, of q QQueryFilter) - public static final String FIELD_SOURCE_TABLE = "sourceTable"; - public static final String FIELD_DEFAULT_QUERY_FILTER = "defaultQueryFilter"; - public static final String FIELD_DESTINATION_TABLE = "destinationTable"; + public static final String FIELD_SUPPORTS_FULL_VALIDATION = "supportsFullValidation"; // Boolean + public static final String FIELD_DO_FULL_VALIDATION = "doFullValidation"; // Boolean + public static final String FIELD_VALIDATION_SUMMARY = "validationSummary"; // List + public static final String FIELD_PROCESS_SUMMARY = "processResults"; // List /******************************************************************************* ** *******************************************************************************/ - public QProcessMetaData defineProcessMetaData( + public static QProcessMetaData defineProcessMetaData( String sourceTableName, String destinationTableName, Class extractStepClass, Class transformStepClass, Class loadStepClass ) + { + Map defaultFieldValues = new HashMap<>(); + defaultFieldValues.put(FIELD_SOURCE_TABLE, sourceTableName); + defaultFieldValues.put(FIELD_DESTINATION_TABLE, destinationTableName); + return defineProcessMetaData(extractStepClass, transformStepClass, loadStepClass, defaultFieldValues); + } + + + + /******************************************************************************* + ** @param defaultFieldValues - expected to possibly contain values for the following field names: + ** - FIELD_SOURCE_TABLE + ** - FIELD_DESTINATION_TABLE + ** - FIELD_SUPPORTS_FULL_VALIDATION + ** - FIELD_DEFAULT_QUERY_FILTER + ** - FIELD_DO_FULL_VALIDATION + *******************************************************************************/ + public static QProcessMetaData defineProcessMetaData( + Class extractStepClass, + Class transformStepClass, + Class loadStepClass, + Map defaultFieldValues + ) { QStepMetaData previewStep = new QBackendStepMetaData() .withName(STEP_NAME_PREVIEW) .withCode(new QCodeReference(StreamedETLPreviewStep.class)) .withInputData(new QFunctionInputMetaData() - .withField(new QFieldMetaData().withName(FIELD_SOURCE_TABLE).withDefaultValue(sourceTableName)) - .withField(new QFieldMetaData().withName(FIELD_DEFAULT_QUERY_FILTER)) + .withField(new QFieldMetaData().withName(FIELD_SOURCE_TABLE).withDefaultValue(defaultFieldValues.get(FIELD_SOURCE_TABLE))) + .withField(new QFieldMetaData().withName(FIELD_SUPPORTS_FULL_VALIDATION).withDefaultValue(defaultFieldValues.getOrDefault(FIELD_SUPPORTS_FULL_VALIDATION, false))) + .withField(new QFieldMetaData().withName(FIELD_DEFAULT_QUERY_FILTER).withDefaultValue(defaultFieldValues.get(FIELD_DEFAULT_QUERY_FILTER))) .withField(new QFieldMetaData().withName(FIELD_EXTRACT_CODE).withDefaultValue(new QCodeReference(extractStepClass))) - .withField(new QFieldMetaData().withName(FIELD_TRANSFORM_CODE).withDefaultValue(new QCodeReference(transformStepClass)))); + .withField(new QFieldMetaData().withName(FIELD_TRANSFORM_CODE).withDefaultValue(new QCodeReference(transformStepClass))) + ); QFrontendStepMetaData reviewStep = new QFrontendStepMetaData() - .withName(STEP_NAME_REVIEW); + .withName(STEP_NAME_REVIEW) + .withComponent(new QFrontendComponentMetaData().withType(QComponentType.VALIDATION_REVIEW_SCREEN)); + + QStepMetaData validateStep = new QBackendStepMetaData() + .withName(STEP_NAME_VALIDATE) + .withCode(new QCodeReference(StreamedETLValidateStep.class)) + .withInputData(new QFunctionInputMetaData() + .withField(new QFieldMetaData().withName(FIELD_DO_FULL_VALIDATION).withDefaultValue(defaultFieldValues.get(FIELD_DO_FULL_VALIDATION)))) + .withOutputMetaData(new QFunctionOutputMetaData() + .withField(new QFieldMetaData().withName(FIELD_VALIDATION_SUMMARY)) + ); QStepMetaData executeStep = new QBackendStepMetaData() .withName(STEP_NAME_EXECUTE) .withCode(new QCodeReference(StreamedETLExecuteStep.class)) .withInputData(new QFunctionInputMetaData() - .withField(new QFieldMetaData().withName(FIELD_DESTINATION_TABLE).withDefaultValue(destinationTableName)) - .withField(new QFieldMetaData().withName(FIELD_LOAD_CODE).withDefaultValue(new QCodeReference(loadStepClass)))); + .withField(new QFieldMetaData().withName(FIELD_DESTINATION_TABLE).withDefaultValue(defaultFieldValues.get(FIELD_DESTINATION_TABLE))) + .withField(new QFieldMetaData().withName(FIELD_LOAD_CODE).withDefaultValue(new QCodeReference(loadStepClass)))) + .withOutputMetaData(new QFunctionOutputMetaData() + .withField(new QFieldMetaData().withName(FIELD_PROCESS_SUMMARY)) + ); QFrontendStepMetaData resultStep = new QFrontendStepMetaData() - .withName(STEP_NAME_RESULT); + .withName(STEP_NAME_RESULT) + .withComponent(new QFrontendComponentMetaData().withType(QComponentType.PROCESS_SUMMARY_RESULTS)); return new QProcessMetaData() - .withName(PROCESS_NAME) .addStep(previewStep) .addStep(reviewStep) + .addStep(validateStep) .addStep(executeStep) .addStep(resultStep); } diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/processes/RunProcessTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/processes/RunProcessTest.java index 336cb636..efa5f6c6 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/processes/RunProcessTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/processes/RunProcessTest.java @@ -23,24 +23,34 @@ package com.kingsrook.qqq.backend.core.actions.processes; import java.io.Serializable; +import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.UUID; import com.kingsrook.qqq.backend.core.exceptions.QException; -import com.kingsrook.qqq.backend.core.processes.implementations.mock.MockBackendStep; import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessState; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput; import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessOutput; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; -import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData; import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendStepMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QStepMetaData; +import com.kingsrook.qqq.backend.core.processes.implementations.mock.MockBackendStep; import com.kingsrook.qqq.backend.core.state.StateType; import com.kingsrook.qqq.backend.core.state.UUIDAndTypeStateKey; import com.kingsrook.qqq.backend.core.utils.TestUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.junit.jupiter.api.Test; +import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertInstanceOf; import static org.junit.jupiter.api.Assertions.assertNotEquals; @@ -56,6 +66,9 @@ import static org.junit.jupiter.api.Assertions.fail; *******************************************************************************/ public class RunProcessTest { + private static final Logger LOG = LogManager.getLogger(RunProcessTest.class); + + /******************************************************************************* ** @@ -85,7 +98,7 @@ public class RunProcessTest @Test public void testBreakOnFrontendSteps() throws QException { - TestCallback callback = new TestCallback(); + TestCallback callback = new TestCallback(); QInstance instance = TestUtils.defineInstance(); RunProcessInput request = new RunProcessInput(instance); String processName = TestUtils.PROCESS_NAME_GREET_PEOPLE_INTERACTIVE; @@ -130,7 +143,7 @@ public class RunProcessTest @Test public void testSkipFrontendSteps() throws QException { - TestCallback callback = new TestCallback(); + TestCallback callback = new TestCallback(); QInstance instance = TestUtils.defineInstance(); RunProcessInput request = new RunProcessInput(instance); String processName = TestUtils.PROCESS_NAME_GREET_PEOPLE_INTERACTIVE; @@ -154,7 +167,7 @@ public class RunProcessTest @Test public void testFailOnFrontendSteps() { - TestCallback callback = new TestCallback(); + TestCallback callback = new TestCallback(); QInstance instance = TestUtils.defineInstance(); RunProcessInput request = new RunProcessInput(instance); String processName = TestUtils.PROCESS_NAME_GREET_PEOPLE_INTERACTIVE; @@ -188,7 +201,8 @@ public class RunProcessTest //////////////////////////////////////////////////////////////////////////////// RunProcessInput runProcessInput = new RunProcessInput(); UUIDAndTypeStateKey stateKey = new UUIDAndTypeStateKey(UUID.randomUUID(), StateType.PROCESS_STATUS); - ProcessState processState = new RunProcessAction().primeProcessState(runProcessInput, stateKey); + QProcessMetaData process = TestUtils.defineInstance().getProcess(TestUtils.PROCESS_NAME_GREET_PEOPLE); + ProcessState processState = new RunProcessAction().primeProcessState(runProcessInput, stateKey, process); assertNotNull(processState); } @@ -206,10 +220,11 @@ public class RunProcessTest RunProcessInput runProcessInput = new RunProcessInput(); runProcessInput.setStartAfterStep("setupStep"); UUIDAndTypeStateKey stateKey = new UUIDAndTypeStateKey(UUID.randomUUID(), StateType.PROCESS_STATUS); + QProcessMetaData process = TestUtils.defineInstance().getProcess(TestUtils.PROCESS_NAME_GREET_PEOPLE); assertThrows(QException.class, () -> { - new RunProcessAction().primeProcessState(runProcessInput, stateKey); + new RunProcessAction().primeProcessState(runProcessInput, stateKey, process); }); } @@ -238,7 +253,8 @@ public class RunProcessTest oldProcessState.getValues().put("foo", "fubu"); RunProcessAction.getStateProvider().put(stateKey, oldProcessState); - ProcessState primedProcessState = new RunProcessAction().primeProcessState(runProcessInput, stateKey); + QProcessMetaData process = TestUtils.defineInstance().getProcess(TestUtils.PROCESS_NAME_GREET_PEOPLE); + ProcessState primedProcessState = new RunProcessAction().primeProcessState(runProcessInput, stateKey, process); assertEquals("myValue", primedProcessState.getValues().get("key")); ///////////////////////////////////////////////////////////////////////////////////////////// @@ -250,6 +266,163 @@ public class RunProcessTest + /******************************************************************************* + ** Test a simple version of custom routing, where we just add a frontend step. + *******************************************************************************/ + @Test + void testCustomRoutingAddFrontendStep() throws QException + { + QInstance qInstance = TestUtils.defineInstance(); + + QStepMetaData back1 = new QBackendStepMetaData() + .withName("back1") + .withCode(new QCodeReference(BackendStepThatMayAddFrontendStep.class)); + + QStepMetaData front1 = new QFrontendStepMetaData() + .withName("front1"); + + String processName = "customRouting"; + qInstance.addProcess(new QProcessMetaData() + .withName(processName) + .withStepList(List.of( + back1 + ////////////////////////////////////// + // only put back1 in the step list. // + ////////////////////////////////////// + )) + .addOptionalStep(front1) + ); + + //////////////////////////////////////////////////////////// + // make sure that if we run by default, we get to the end // + //////////////////////////////////////////////////////////// + RunProcessInput request = new RunProcessInput(qInstance); + request.setSession(TestUtils.getMockSession()); + request.setProcessName(processName); + request.setCallback(new TestCallback()); + request.setFrontendStepBehavior(RunProcessInput.FrontendStepBehavior.BREAK); + RunProcessOutput result = new RunProcessAction().execute(request); + assertThat(result.getProcessState().getNextStepName()).isEmpty(); + + ///////////////////////////////////////////////////////////////////////////////////////////// + // now run again, with the field set to cause the front1 step to be added to the step list // + ///////////////////////////////////////////////////////////////////////////////////////////// + request.addValue("shouldAddFrontendStep", true); + result = new RunProcessAction().execute(request); + assertThat(result.getProcessState().getNextStepName()).hasValue("front1"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static class BackendStepThatMayAddFrontendStep implements BackendStep + { + + /******************************************************************************* + ** Execute the backend step - using the request as input, and the result as output. + ** + *******************************************************************************/ + @Override + public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException + { + LOG.info("Running " + getClass().getSimpleName()); + if(runBackendStepInput.getValue("shouldAddFrontendStep") != null) + { + List stepList = new ArrayList<>(runBackendStepOutput.getProcessState().getStepList()); + stepList.add("front1"); + runBackendStepOutput.getProcessState().setStepList(stepList); + } + } + } + + + + /******************************************************************************* + ** Test a version of custom routing, where we remove steps + *******************************************************************************/ + @Test + void testCustomRoutingRemoveSteps() throws QException + { + QInstance qInstance = TestUtils.defineInstance(); + + QStepMetaData back1 = new QBackendStepMetaData() + .withName("back1") + .withCode(new QCodeReference(BackendStepThatMayRemoveFrontendStep.class)); + + QStepMetaData front1 = new QFrontendStepMetaData() + .withName("front1"); + + QStepMetaData back2 = new QBackendStepMetaData() + .withName("back2") + .withCode(new QCodeReference(NoopBackendStep.class)); + + QStepMetaData front2 = new QFrontendStepMetaData() + .withName("front2"); + + String processName = "customRouting"; + qInstance.addProcess(new QProcessMetaData() + .withName(processName) + .withStepList(List.of( + back1, + front1, + back2, + front2 + )) + ); + + ///////////////////////////////////////////////////////////////////////////// + // make sure that if we run by default, we get stop on both frontend steps // + ///////////////////////////////////////////////////////////////////////////// + RunProcessInput request = new RunProcessInput(qInstance); + request.setSession(TestUtils.getMockSession()); + request.setProcessName(processName); + request.setCallback(new TestCallback()); + request.setFrontendStepBehavior(RunProcessInput.FrontendStepBehavior.BREAK); + RunProcessOutput result = new RunProcessAction().execute(request); + assertThat(result.getProcessState().getNextStepName()).hasValue("front1"); + + request.setStartAfterStep("front1"); + result = new RunProcessAction().execute(request); + assertThat(result.getProcessState().getNextStepName()).hasValue("front2"); + + ///////////////////////////////////////////////////////////////////////////////////////////////// + // now run again, with the field set to cause the front1 step to be removed from the step list // + ///////////////////////////////////////////////////////////////////////////////////////////////// + request.setStartAfterStep(null); + request.addValue("shouldRemoveFrontendStep", true); + result = new RunProcessAction().execute(request); + assertThat(result.getProcessState().getNextStepName()).hasValue("front2"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static class BackendStepThatMayRemoveFrontendStep implements BackendStep + { + + /******************************************************************************* + ** Execute the backend step - using the request as input, and the result as output. + ** + *******************************************************************************/ + @Override + public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException + { + LOG.info("Running " + getClass().getSimpleName()); + if(runBackendStepInput.getValue("shouldRemoveFrontendStep") != null) + { + List stepList = new ArrayList<>(runBackendStepOutput.getProcessState().getStepList()); + stepList.removeIf(s -> s.equals("front1")); + runBackendStepOutput.getProcessState().setStepList(stepList); + } + } + } + + + /******************************************************************************* ** *******************************************************************************/ @@ -287,4 +460,24 @@ public class RunProcessTest return (rs); } } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static class NoopBackendStep implements BackendStep + { + + /******************************************************************************* + ** Execute the backend step - using the request as input, and the result as output. + ** + *******************************************************************************/ + @Override + public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException + { + LOG.info("Running " + getClass().getSimpleName()); + } + } + } diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/reporting/ReportActionTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/reporting/ReportActionTest.java index 71d2807f..08d0d5ba 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/reporting/ReportActionTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/reporting/ReportActionTest.java @@ -64,7 +64,8 @@ class ReportActionTest runReport(recordCount, filename, ReportFormat.CSV, false); - File file = new File(filename); + File file = new File(filename); + @SuppressWarnings("unchecked") List fileLines = FileUtils.readLines(file, StandardCharsets.UTF_8.name()); assertEquals(recordCount + 1, fileLines.size()); assertTrue(file.delete()); @@ -85,7 +86,8 @@ class ReportActionTest runReport(recordCount, filename, ReportFormat.CSV, false); - File file = new File(filename); + File file = new File(filename); + @SuppressWarnings("unchecked") List fileLines = FileUtils.readLines(file, StandardCharsets.UTF_8.name()); assertEquals(recordCount + 1, fileLines.size()); assertTrue(file.delete()); diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLWithFrontendProcessTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLWithFrontendProcessTest.java index 2c344a49..2f1f11b6 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLWithFrontendProcessTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLWithFrontendProcessTest.java @@ -2,16 +2,19 @@ package com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwit import java.io.Serializable; +import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; import com.kingsrook.qqq.backend.core.actions.processes.QProcessCallback; import com.kingsrook.qqq.backend.core.actions.processes.RunProcessAction; import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLine; import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput; import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessOutput; +import com.kingsrook.qqq.backend.core.model.actions.processes.Status; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; @@ -155,7 +158,7 @@ class StreamedETLWithFrontendProcessTest assertThat(postList).as("Should have transformed and updated " + name).anyMatch(qr -> qr.getValue("name").equals("Transformed:" + name)); } - for(String name : new String[] { "Circle", "Triangle"}) + for(String name : new String[] { "Circle", "Triangle" }) { assertThat(postList).as("Should not have transformed and updated " + name).anyMatch(qr -> qr.getValue("name").equals(name)); } @@ -192,6 +195,47 @@ class StreamedETLWithFrontendProcessTest + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testWithValidationStep() throws QException + { + QInstance instance = TestUtils.defineInstance(); + + //////////////////////////////////////////////////////// + // define the process - an ELT from Shapes to Persons // + //////////////////////////////////////////////////////// + QProcessMetaData process = new StreamedETLWithFrontendProcess().defineProcessMetaData( + TestUtils.TABLE_NAME_SHAPE, + TestUtils.TABLE_NAME_PERSON, + ExtractViaQueryStep.class, + TestTransformShapeToPersonWithValidationStep.class, + LoadViaInsertStep.class); + process.setTableName(TestUtils.TABLE_NAME_SHAPE); + instance.addProcess(process); + + /////////////////////////////////////////////////////// + // switch the person table to use the memory backend // + /////////////////////////////////////////////////////// + instance.getTable(TestUtils.TABLE_NAME_PERSON).setBackendName(TestUtils.MEMORY_BACKEND_NAME); + + TestUtils.insertDefaultShapes(instance); + + ///////////////////// + // run the process // todo - don't skip FE steps + ///////////////////// + runProcess(instance, process); + + List postList = TestUtils.queryTable(instance, TestUtils.TABLE_NAME_PERSON); + assertThat(postList) + .as("Should have inserted Circle").anyMatch(qr -> qr.getValue("lastName").equals("Circle")) + .as("Should have inserted Triangle").anyMatch(qr -> qr.getValue("lastName").equals("Triangle")) + .as("Should have inserted Square").anyMatch(qr -> qr.getValue("lastName").equals("Square")); + } + + + /******************************************************************************* ** *******************************************************************************/ @@ -201,6 +245,7 @@ class StreamedETLWithFrontendProcessTest } + /******************************************************************************* ** *******************************************************************************/ @@ -246,6 +291,63 @@ class StreamedETLWithFrontendProcessTest + /******************************************************************************* + ** + *******************************************************************************/ + public static class TestTransformShapeToPersonWithValidationStep extends AbstractTransformStep implements ProcessSummaryProviderInterface + { + private ProcessSummaryLine okSummary = new ProcessSummaryLine(Status.OK, 0, "can be transformed into a Person"); + private ProcessSummaryLine notAPolygonSummary = new ProcessSummaryLine(Status.OK, 0, "cannot be transformed, because they are not a Polygon"); + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public ArrayList getProcessSummary(boolean isForResultScreen) + { + if(isForResultScreen) + { + okSummary.setMessage("were transformed into a Person"); + } + + ArrayList summaryList = new ArrayList<>(); + summaryList.add(okSummary); + summaryList.add(notAPolygonSummary); + return (summaryList); + } + + + + /******************************************************************************* + ** Execute the backend step - using the request as input, and the result as output. + ** + *******************************************************************************/ + @Override + public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException + { + for(QRecord qRecord : getInputRecordPage()) + { + if(qRecord.getValueString("name").equals("Circle")) + { + notAPolygonSummary.incrementCountAndAddPrimaryKey(qRecord.getValue("id")); + } + else + { + QRecord newQRecord = new QRecord(); + newQRecord.setValue("firstName", "Johnny"); + newQRecord.setValue("lastName", qRecord.getValueString("name")); + getOutputRecordPage().add(newQRecord); + + okSummary.incrementCountAndAddPrimaryKey(qRecord.getValue("id")); + } + } + } + } + + + /******************************************************************************* ** *******************************************************************************/ diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java index 9984d2e3..5d093faf 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java @@ -328,7 +328,7 @@ public class TestUtils .withOutputMetaData(new QFunctionOutputMetaData() .withRecordListMetaData(new QRecordListMetaData() .withTableName(TABLE_NAME_PERSON) - .addField(new QFieldMetaData("fullGreeting", QFieldType.STRING)) + .withField(new QFieldMetaData("fullGreeting", QFieldType.STRING)) ) .withFieldList(List.of(new QFieldMetaData("outputMessage", QFieldType.STRING)))) ); @@ -366,7 +366,7 @@ public class TestUtils .withOutputMetaData(new QFunctionOutputMetaData() .withRecordListMetaData(new QRecordListMetaData() .withTableName(TABLE_NAME_PERSON) - .addField(new QFieldMetaData("fullGreeting", QFieldType.STRING)) + .withField(new QFieldMetaData("fullGreeting", QFieldType.STRING)) ) .withFieldList(List.of(new QFieldMetaData("outputMessage", QFieldType.STRING)))) ) @@ -403,7 +403,7 @@ public class TestUtils .withOutputMetaData(new QFunctionOutputMetaData() .withRecordListMetaData(new QRecordListMetaData() .withTableName(TABLE_NAME_PERSON) - .addField(new QFieldMetaData("age", QFieldType.INTEGER))) + .withField(new QFieldMetaData("age", QFieldType.INTEGER))) .withFieldList(List.of( new QFieldMetaData("minAge", QFieldType.INTEGER), new QFieldMetaData("maxAge", QFieldType.INTEGER))))) @@ -418,7 +418,7 @@ public class TestUtils .withOutputMetaData(new QFunctionOutputMetaData() .withRecordListMetaData(new QRecordListMetaData() .withTableName(TABLE_NAME_PERSON) - .addField(new QFieldMetaData("newAge", QFieldType.INTEGER))))); + .withField(new QFieldMetaData("newAge", QFieldType.INTEGER))))); } diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/model/metadata/AbstractFilesystemBackendMetaData.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/model/metadata/AbstractFilesystemBackendMetaData.java index 9002bdb8..5cf43010 100644 --- a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/model/metadata/AbstractFilesystemBackendMetaData.java +++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/model/metadata/AbstractFilesystemBackendMetaData.java @@ -70,11 +70,10 @@ public class AbstractFilesystemBackendMetaData extends QBackendMetaData ** Fluent setter for basePath ** *******************************************************************************/ - @SuppressWarnings("unchecked") - public T withBasePath(String basePath) + public AbstractFilesystemBackendMetaData withBasePath(String basePath) { this.basePath = basePath; - return (T) this; + return (this); } } diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/model/metadata/FilesystemBackendMetaData.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/model/metadata/FilesystemBackendMetaData.java index 02b3950e..51cbdb6a 100644 --- a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/model/metadata/FilesystemBackendMetaData.java +++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/model/metadata/FilesystemBackendMetaData.java @@ -42,4 +42,28 @@ public class FilesystemBackendMetaData extends AbstractFilesystemBackendMetaData setBackendType(FilesystemBackendModule.class); } + + + /******************************************************************************* + ** Fluent setter for basePath + ** + *******************************************************************************/ + public FilesystemBackendMetaData withBasePath(String basePath) + { + setBasePath(basePath); + return (this); + } + + + + /******************************************************************************* + ** Fluent setter for name + ** + *******************************************************************************/ + public FilesystemBackendMetaData withName(String name) + { + setName(name); + return this; + } + } diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/basic/BasicETLCollectSourceFileNamesStep.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/basic/BasicETLCollectSourceFileNamesStep.java index b444a42d..fd1f56b7 100644 --- a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/basic/BasicETLCollectSourceFileNamesStep.java +++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/basic/BasicETLCollectSourceFileNamesStep.java @@ -79,6 +79,6 @@ public class BasicETLCollectSourceFileNamesStep implements BackendStep .withCodeType(QCodeType.JAVA) .withCodeUsage(QCodeUsage.BACKEND_STEP)) .withOutputMetaData(new QFunctionOutputMetaData() - .addField(new QFieldMetaData(FIELD_SOURCE_FILE_PATHS, QFieldType.STRING)))); + .withField(new QFieldMetaData(FIELD_SOURCE_FILE_PATHS, QFieldType.STRING)))); } } diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/model/metadata/S3BackendMetaData.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/model/metadata/S3BackendMetaData.java index 92dfccc1..2475b6b5 100644 --- a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/model/metadata/S3BackendMetaData.java +++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/model/metadata/S3BackendMetaData.java @@ -76,11 +76,10 @@ public class S3BackendMetaData extends AbstractFilesystemBackendMetaData ** Fluent setter for bucketName ** *******************************************************************************/ - @SuppressWarnings("unchecked") - public T withBucketName(String bucketName) + public S3BackendMetaData withBucketName(String bucketName) { this.bucketName = bucketName; - return (T) this; + return (this); } @@ -111,11 +110,10 @@ public class S3BackendMetaData extends AbstractFilesystemBackendMetaData ** Fluent setter for accessKey ** *******************************************************************************/ - @SuppressWarnings("unchecked") - public T withAccessKey(String accessKey) + public S3BackendMetaData withAccessKey(String accessKey) { this.accessKey = accessKey; - return (T) this; + return (this); } @@ -146,11 +144,10 @@ public class S3BackendMetaData extends AbstractFilesystemBackendMetaData ** Fluent setter for secretKey ** *******************************************************************************/ - @SuppressWarnings("unchecked") - public T withSecretKey(String secretKey) + public S3BackendMetaData withSecretKey(String secretKey) { this.secretKey = secretKey; - return (T) this; + return (this); } @@ -181,11 +178,10 @@ public class S3BackendMetaData extends AbstractFilesystemBackendMetaData ** Fluent setter for region ** *******************************************************************************/ - @SuppressWarnings("unchecked") - public T withRegion(String region) + public S3BackendMetaData withRegion(String region) { this.region = region; - return (T) this; + return (this); } @@ -204,4 +200,28 @@ public class S3BackendMetaData extends AbstractFilesystemBackendMetaData secretKey = interpreter.interpret(secretKey); } + + + /******************************************************************************* + ** Fluent setter for basePath + ** + *******************************************************************************/ + public S3BackendMetaData withBasePath(String basePath) + { + setBasePath(basePath); + return (this); + } + + + + /******************************************************************************* + ** Fluent setter for name + ** + *******************************************************************************/ + public S3BackendMetaData withName(String name) + { + setName(name); + return this; + } + } diff --git a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/jdbc/QueryManager.java b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/jdbc/QueryManager.java index 0a90182f..ce761755 100644 --- a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/jdbc/QueryManager.java +++ b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/jdbc/QueryManager.java @@ -746,12 +746,14 @@ public class QueryManager } else if(value instanceof LocalDate ld) { + @SuppressWarnings("deprecation") java.sql.Date date = new java.sql.Date(ld.getYear() - 1900, ld.getMonthValue() - 1, ld.getDayOfMonth()); statement.setDate(index, date); return (1); } else if(value instanceof LocalTime lt) { + @SuppressWarnings("deprecation") java.sql.Time time = new java.sql.Time(lt.getHour(), lt.getMinute(), lt.getSecond()); statement.setTime(index, time); return (1); diff --git a/qqq-sample-project/src/main/java/com/kingsrook/sampleapp/SampleMetaDataProvider.java b/qqq-sample-project/src/main/java/com/kingsrook/sampleapp/SampleMetaDataProvider.java index 7c0e5c00..09817033 100644 --- a/qqq-sample-project/src/main/java/com/kingsrook/sampleapp/SampleMetaDataProvider.java +++ b/qqq-sample-project/src/main/java/com/kingsrook/sampleapp/SampleMetaDataProvider.java @@ -22,7 +22,10 @@ package com.kingsrook.sampleapp; +import java.io.Serializable; +import java.util.HashMap; import java.util.List; +import java.util.Map; import com.kingsrook.qqq.backend.core.actions.processes.BackendStep; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.exceptions.QValueException; @@ -49,6 +52,9 @@ import com.kingsrook.qqq.backend.core.model.metadata.tables.QFieldSection; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.Tier; import com.kingsrook.qqq.backend.core.modules.authentication.metadata.QAuthenticationMetaData; +import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.ExtractViaQueryStep; +import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.LoadViaInsertStep; +import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess; import com.kingsrook.qqq.backend.core.processes.implementations.general.LoadInitialRecordsStep; import com.kingsrook.qqq.backend.core.processes.implementations.mock.MockBackendStep; import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.Cardinality; @@ -56,6 +62,7 @@ import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.RecordFor import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemBackendMetaData; import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemTableBackendDetails; import com.kingsrook.qqq.backend.module.rdbms.model.metadata.RDBMSBackendMetaData; +import com.kingsrook.sampleapp.processes.clonepeople.ClonePeopleTransformStep; import io.github.cdimascio.dotenv.Dotenv; @@ -79,6 +86,7 @@ public class SampleMetaDataProvider public static final String PROCESS_NAME_GREET = "greet"; public static final String PROCESS_NAME_GREET_INTERACTIVE = "greetInteractive"; + public static final String PROCESS_NAME_CLONE_PEOPLE = "clonePeople"; public static final String PROCESS_NAME_SIMPLE_SLEEP = "simpleSleep"; public static final String PROCESS_NAME_SIMPLE_THROW = "simpleThrow"; public static final String PROCESS_NAME_SLEEP_INTERACTIVE = "sleepInteractive"; @@ -110,6 +118,7 @@ public class SampleMetaDataProvider qInstance.addTable(defineTableCityFile()); qInstance.addProcess(defineProcessGreetPeople()); qInstance.addProcess(defineProcessGreetPeopleInteractive()); + qInstance.addProcess(defineProcessClonePeople()); qInstance.addProcess(defineProcessSimpleSleep()); qInstance.addProcess(defineProcessScreenThenSleep()); qInstance.addProcess(defineProcessSimpleThrow()); @@ -129,19 +138,18 @@ public class SampleMetaDataProvider qInstance.addApp(new QAppMetaData() .withName(APP_NAME_GREETINGS) .withIcon(new QIcon().withName("emoji_people")) - .withChild(qInstance.getProcess(PROCESS_NAME_GREET) - .withIcon(new QIcon().withName("emoji_people"))) - .withChild(qInstance.getTable(TABLE_NAME_PERSON) - .withIcon(new QIcon().withName("person"))) - .withChild(qInstance.getTable(TABLE_NAME_CITY) - .withIcon(new QIcon().withName("location_city"))) - .withChild(qInstance.getProcess(PROCESS_NAME_GREET_INTERACTIVE)) - .withIcon(new QIcon().withName("waving_hand"))); + .withChild(qInstance.getProcess(PROCESS_NAME_GREET).withIcon(new QIcon().withName("emoji_people"))) + .withChild(qInstance.getTable(TABLE_NAME_PERSON).withIcon(new QIcon().withName("person"))) + .withChild(qInstance.getTable(TABLE_NAME_CITY).withIcon(new QIcon().withName("location_city"))) + .withChild(qInstance.getProcess(PROCESS_NAME_GREET_INTERACTIVE).withIcon(new QIcon().withName("waving_hand"))) + ); qInstance.addApp(new QAppMetaData() .withName(APP_NAME_PEOPLE) .withIcon(new QIcon().withName("person")) - .withChild(qInstance.getApp(APP_NAME_GREETINGS))); + .withChild(qInstance.getApp(APP_NAME_GREETINGS)) + .withChild(qInstance.getProcess(PROCESS_NAME_CLONE_PEOPLE).withIcon(new QIcon().withName("content_copy"))) + ); qInstance.addApp(new QAppMetaData() .withName(APP_NAME_MISCELLANEOUS) @@ -231,8 +239,8 @@ public class SampleMetaDataProvider .withBackendName("company_code")); table.addField(new QFieldMetaData("service_level", QFieldType.STRING) // todo PVS - .withLabel("Service Level") - .withIsRequired(true)); + .withLabel("Service Level") + .withIsRequired(true)); table.addSection(new QFieldSection("identity", "Identity", new QIcon("badge"), Tier.T1, List.of("id", "name"))); table.addSection(new QFieldSection("basicInfo", "Basic Info", new QIcon("dataset"), Tier.T2, List.of("company_code", "service_level"))); @@ -325,7 +333,7 @@ public class SampleMetaDataProvider .withOutputMetaData(new QFunctionOutputMetaData() .withRecordListMetaData(new QRecordListMetaData() .withTableName(TABLE_NAME_PERSON) - .addField(new QFieldMetaData("fullGreeting", QFieldType.STRING)) + .withField(new QFieldMetaData("fullGreeting", QFieldType.STRING)) ) .withFieldList(List.of(new QFieldMetaData("outputMessage", QFieldType.STRING)))) ); @@ -365,7 +373,7 @@ public class SampleMetaDataProvider .withOutputMetaData(new QFunctionOutputMetaData() .withRecordListMetaData(new QRecordListMetaData() .withTableName(TABLE_NAME_PERSON) - .addField(new QFieldMetaData("fullGreeting", QFieldType.STRING)) + .withField(new QFieldMetaData("fullGreeting", QFieldType.STRING)) ) .withFieldList(List.of(new QFieldMetaData("outputMessage", QFieldType.STRING)))) ) @@ -383,6 +391,35 @@ public class SampleMetaDataProvider + /******************************************************************************* + ** + *******************************************************************************/ + private static QProcessMetaData defineProcessClonePeople() + { + Map values = new HashMap<>(); + values.put(StreamedETLWithFrontendProcess.FIELD_SOURCE_TABLE, TABLE_NAME_PERSON); + values.put(StreamedETLWithFrontendProcess.FIELD_DESTINATION_TABLE, TABLE_NAME_PERSON); + values.put(StreamedETLWithFrontendProcess.FIELD_SUPPORTS_FULL_VALIDATION, true); + + QProcessMetaData process = StreamedETLWithFrontendProcess.defineProcessMetaData( + ExtractViaQueryStep.class, + ClonePeopleTransformStep.class, + LoadViaInsertStep.class, + values + ); + process.setName(PROCESS_NAME_CLONE_PEOPLE); + process.setTableName(TABLE_NAME_PERSON); + + process.getFrontendStep(StreamedETLWithFrontendProcess.STEP_NAME_REVIEW) + .withRecordListField(new QFieldMetaData("firstName", QFieldType.STRING)) + .withRecordListField(new QFieldMetaData("lastName", QFieldType.STRING)) + ; + + return (process); + } + + + /******************************************************************************* ** Define a process with just one step that sleeps *******************************************************************************/ @@ -467,7 +504,7 @@ public class SampleMetaDataProvider .withCodeType(QCodeType.JAVA) .withCodeUsage(QCodeUsage.BACKEND_STEP)) .withInputData(new QFunctionInputMetaData() - .addField(new QFieldMetaData(SleeperStep.FIELD_SLEEP_MILLIS, QFieldType.INTEGER)))); + .withField(new QFieldMetaData(SleeperStep.FIELD_SLEEP_MILLIS, QFieldType.INTEGER)))); } } diff --git a/qqq-sample-project/src/main/java/com/kingsrook/sampleapp/processes/clonepeople/ClonePeopleTransformStep.java b/qqq-sample-project/src/main/java/com/kingsrook/sampleapp/processes/clonepeople/ClonePeopleTransformStep.java new file mode 100644 index 00000000..a8874be4 --- /dev/null +++ b/qqq-sample-project/src/main/java/com/kingsrook/sampleapp/processes/clonepeople/ClonePeopleTransformStep.java @@ -0,0 +1,88 @@ +package com.kingsrook.sampleapp.processes.clonepeople; + + +import java.io.Serializable; +import java.util.ArrayList; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLine; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; +import com.kingsrook.qqq.backend.core.model.actions.processes.Status; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.AbstractTransformStep; +import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.ProcessSummaryProviderInterface; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class ClonePeopleTransformStep extends AbstractTransformStep implements ProcessSummaryProviderInterface +{ + private ProcessSummaryLine okSummary = new ProcessSummaryLine(Status.OK, 0, "can be cloned with no issues."); + private ProcessSummaryLine warningCloneSummary = new ProcessSummaryLine(Status.WARNING, 0, "can be cloned, but because are already a clone, their clone cannot be cloned in the future."); + private ProcessSummaryLine refuseCloningSummary = new ProcessSummaryLine(Status.ERROR, 0, "say they don't want to be cloned (probably a Garret...)"); + private ProcessSummaryLine nestedCloneSummary = new ProcessSummaryLine(Status.ERROR, 0, "are already a clone of a clone, so they can't be cloned again."); + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public ArrayList getProcessSummary(boolean isForResultScreen) + { + if(isForResultScreen) + { + okSummary.setMessage("were cloned"); + warningCloneSummary.setMessage("were already a clone, so they were cloned again now, but their clones cannot be cloned after this."); + nestedCloneSummary.setMessage("are already a clone of a clone, so they weren't cloned again."); + } + + ArrayList rs = new ArrayList<>(); + okSummary.addSelfToListIfAnyCount(rs); + warningCloneSummary.addSelfToListIfAnyCount(rs); + refuseCloningSummary.addSelfToListIfAnyCount(rs); + nestedCloneSummary.addSelfToListIfAnyCount(rs); + return (rs); + } + + + + /******************************************************************************* + ** Execute the backend step - using the request as input, and the result as output. + ** + *******************************************************************************/ + @Override + public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException + { + for(QRecord inputPerson : getInputRecordPage()) + { + Serializable id = inputPerson.getValue("id"); + if("Garret".equals(inputPerson.getValueString("firstName"))) + { + refuseCloningSummary.incrementCountAndAddPrimaryKey(id); + } + else if(inputPerson.getValueString("firstName").matches("Clone of.*Clone of.*")) + { + nestedCloneSummary.incrementCountAndAddPrimaryKey(id); + } + else + { + QRecord outputPerson = new QRecord(inputPerson); + outputPerson.setValue("id", null); + outputPerson.setValue("firstName", "Clone of: " + inputPerson.getValueString("firstName")); + getOutputRecordPage().add(outputPerson); + + if(inputPerson.getValueString("firstName").matches("Clone of.*")) + { + warningCloneSummary.incrementCountAndAddPrimaryKey(id); + } + else + { + okSummary.incrementCountAndAddPrimaryKey(id); + } + } + } + } + +}