diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/scripts/RunAdHocRecordScriptAction.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/scripts/RunAdHocRecordScriptAction.java index 90ea3482..19824ec0 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/scripts/RunAdHocRecordScriptAction.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/scripts/RunAdHocRecordScriptAction.java @@ -145,6 +145,7 @@ public class RunAdHocRecordScriptAction new ExecuteCodeAction().run(executeCodeInput, executeCodeOutput); output.setOutput(executeCodeOutput.getOutput()); + output.setLogger(executionLogger); } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/scripts/RunAdHocRecordScriptOutput.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/scripts/RunAdHocRecordScriptOutput.java index d070916e..90f761f8 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/scripts/RunAdHocRecordScriptOutput.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/scripts/RunAdHocRecordScriptOutput.java @@ -23,6 +23,7 @@ package com.kingsrook.qqq.backend.core.model.actions.scripts; import java.io.Serializable; +import com.kingsrook.qqq.backend.core.actions.scripts.logging.QCodeExecutionLoggerInterface; import com.kingsrook.qqq.backend.core.model.actions.AbstractActionOutput; @@ -33,6 +34,8 @@ public class RunAdHocRecordScriptOutput extends AbstractActionOutput { private Serializable output; + private QCodeExecutionLoggerInterface logger; + /******************************************************************************* @@ -67,4 +70,35 @@ public class RunAdHocRecordScriptOutput extends AbstractActionOutput return (this); } + + + /******************************************************************************* + ** Getter for logger + *******************************************************************************/ + public QCodeExecutionLoggerInterface getLogger() + { + return (this.logger); + } + + + + /******************************************************************************* + ** Setter for logger + *******************************************************************************/ + public void setLogger(QCodeExecutionLoggerInterface logger) + { + this.logger = logger; + } + + + + /******************************************************************************* + ** Fluent setter for logger + *******************************************************************************/ + public RunAdHocRecordScriptOutput withLogger(QCodeExecutionLoggerInterface logger) + { + this.logger = logger; + return (this); + } + } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/scripts/Script.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/scripts/Script.java index 430abb0f..3a716e7b 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/scripts/Script.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/scripts/Script.java @@ -55,6 +55,9 @@ public class Script extends QRecordEntity @QField(possibleValueSourceName = TablesPossibleValueSourceMetaDataProvider.NAME) private String tableName; + @QField() + private Integer maxBatchSize; + @QField(possibleValueSourceName = "scriptRevision") private Integer currentScriptRevisionId; @@ -314,4 +317,35 @@ public class Script extends QRecordEntity return (this); } + + + /******************************************************************************* + ** Getter for maxBatchSize + *******************************************************************************/ + public Integer getMaxBatchSize() + { + return (this.maxBatchSize); + } + + + + /******************************************************************************* + ** Setter for maxBatchSize + *******************************************************************************/ + public void setMaxBatchSize(Integer maxBatchSize) + { + this.maxBatchSize = maxBatchSize; + } + + + + /******************************************************************************* + ** Fluent setter for maxBatchSize + *******************************************************************************/ + public Script withMaxBatchSize(Integer maxBatchSize) + { + this.maxBatchSize = maxBatchSize; + return (this); + } + } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/scripts/ScriptsMetaDataProvider.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/scripts/ScriptsMetaDataProvider.java index b2d51a95..346e8fa7 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/scripts/ScriptsMetaDataProvider.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/scripts/ScriptsMetaDataProvider.java @@ -57,10 +57,10 @@ import com.kingsrook.qqq.backend.core.model.metadata.tables.QFieldSection; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.TablesPossibleValueSourceMetaDataProvider; import com.kingsrook.qqq.backend.core.model.metadata.tables.Tier; -import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.NoopTransformStep; import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess; import com.kingsrook.qqq.backend.core.processes.implementations.scripts.RunRecordScriptExtractStep; import com.kingsrook.qqq.backend.core.processes.implementations.scripts.RunRecordScriptLoadStep; +import com.kingsrook.qqq.backend.core.processes.implementations.scripts.RunRecordScriptTransformStep; import com.kingsrook.qqq.backend.core.processes.implementations.scripts.StoreScriptRevisionProcessStep; import com.kingsrook.qqq.backend.core.processes.implementations.scripts.TestScriptProcessStep; @@ -138,7 +138,7 @@ public class ScriptsMetaDataProvider .withIcon(new QIcon().withName("data_object")) .withSupportsFullValidation(false) .withExtractStepClass(RunRecordScriptExtractStep.class) - .withTransformStepClass(NoopTransformStep.class) + .withTransformStepClass(RunRecordScriptTransformStep.class) .withLoadStepClass(RunRecordScriptLoadStep.class) .getProcessMetaData(); @@ -165,6 +165,10 @@ public class ScriptsMetaDataProvider .withLabel("Log Lines") .getWidgetMetaData()); + instance.addWidget(ChildRecordListRenderer.widgetMetaDataBuilder(instance.getJoin(QJoinMetaData.makeInferredJoinName(Script.TABLE_NAME, ScriptLog.TABLE_NAME))).withMaxRows(50) + .withLabel("Recent Logs") + .getWidgetMetaData()); + instance.addWidget(new QWidgetMetaData() .withName("scriptViewer") .withLabel("Contents") @@ -211,6 +215,14 @@ public class ScriptsMetaDataProvider .withOrderBy(new QFilterOrderBy("id")) .withInferredName()); + instance.addJoin(new QJoinMetaData() + .withType(JoinType.ONE_TO_MANY) + .withLeftTable(Script.TABLE_NAME) + .withRightTable(ScriptLog.TABLE_NAME) + .withJoinOn(new JoinOn("id", "scriptId")) + .withOrderBy(new QFilterOrderBy("id")) + .withInferredName()); + } @@ -308,11 +320,17 @@ public class ScriptsMetaDataProvider *******************************************************************************/ private QTableMetaData defineTableTriggerTable(String backendName) throws QException { - return (defineStandardTable(backendName, TableTrigger.TABLE_NAME, TableTrigger.class) + QTableMetaData tableMetaData = defineStandardTable(backendName, TableTrigger.TABLE_NAME, TableTrigger.class) .withRecordLabelFields("id") .withSection(new QFieldSection("identity", new QIcon().withName("badge"), Tier.T1, List.of("id"))) .withSection(new QFieldSection("contents", new QIcon().withName("data_object"), Tier.T2, List.of("tableName", "filterId", "scriptId", "priority", "postInsert", "postUpdate"))) - .withSection(new QFieldSection("dates", new QIcon().withName("calendar_month"), Tier.T3, List.of("createDate", "modifyDate")))); + .withSection(new QFieldSection("dates", new QIcon().withName("calendar_month"), Tier.T3, List.of("createDate", "modifyDate"))); + + tableMetaData.getField("scriptId").withPossibleValueSourceFilter(new QQueryFilter( + new QFilterCriteria("scriptType.name", QCriteriaOperator.EQUALS, SCRIPT_TYPE_NAME_RECORD) + )); + + return tableMetaData; } @@ -323,12 +341,15 @@ public class ScriptsMetaDataProvider private QTableMetaData defineScriptTable(String backendName) throws QException { QTableMetaData tableMetaData = defineStandardTable(backendName, Script.TABLE_NAME, Script.class) - .withSection(new QFieldSection("identity", new QIcon().withName("badge"), Tier.T1, List.of("id", "name", "scriptTypeId", "tableName", "currentScriptRevisionId"))) + .withSection(new QFieldSection("identity", new QIcon().withName("badge"), Tier.T1, List.of("id", "name", "scriptTypeId", "currentScriptRevisionId"))) + .withSection(new QFieldSection("recordScriptSettings", new QIcon().withName("table_rows"), Tier.T2, List.of("tableName", "maxBatchSize"))) .withSection(new QFieldSection("contents", new QIcon().withName("data_object"), Tier.T2).withWidgetName("scriptViewer")) - .withSection(new QFieldSection("dates", new QIcon().withName("calendar_month"), Tier.T3, List.of("createDate", "modifyDate"))); + .withSection(new QFieldSection("dates", new QIcon().withName("calendar_month"), Tier.T3, List.of("createDate", "modifyDate"))) + .withSection(new QFieldSection("lines", new QIcon().withName("horizontal_rule"), Tier.T2).withWidgetName(QJoinMetaData.makeInferredJoinName(Script.TABLE_NAME, ScriptLog.TABLE_NAME))); tableMetaData.getField("name").withFieldAdornment(AdornmentType.Size.LARGE.toAdornment()); tableMetaData.getField("currentScriptRevisionId").withFieldAdornment(AdornmentType.Size.LARGE.toAdornment()); + tableMetaData.getField("currentScriptRevisionId").withIsEditable(false); return (tableMetaData); } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/scripts/RunRecordScriptExtractStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/scripts/RunRecordScriptExtractStep.java index 227611f9..4599b0c8 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/scripts/RunRecordScriptExtractStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/scripts/RunRecordScriptExtractStep.java @@ -22,9 +22,13 @@ package com.kingsrook.qqq.backend.core.processes.implementations.scripts; +import com.kingsrook.qqq.backend.core.actions.tables.GetAction; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; +import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetOutput; +import com.kingsrook.qqq.backend.core.model.scripts.Script; import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.ExtractViaQueryStep; import com.kingsrook.qqq.backend.core.utils.StringUtils; @@ -53,6 +57,16 @@ public class RunRecordScriptExtractStep extends ExtractViaQueryStep runBackendStepInput.addValue(FIELD_SOURCE_TABLE, tableName); + Integer scriptId = runBackendStepInput.getValueInteger("scriptId"); + GetInput getInput = new GetInput(); + getInput.setTableName(Script.TABLE_NAME); + getInput.setPrimaryKey(scriptId); + GetOutput getOutput = new GetAction().execute(getInput); + if(getOutput.getRecord() != null) + { + runBackendStepOutput.addValue("scriptName", getOutput.getRecord().getValueString("name")); + } + super.preRun(runBackendStepInput, runBackendStepOutput); } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/scripts/RunRecordScriptLoadStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/scripts/RunRecordScriptLoadStep.java index 598f20fb..74ecdc40 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/scripts/RunRecordScriptLoadStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/scripts/RunRecordScriptLoadStep.java @@ -22,30 +22,147 @@ package com.kingsrook.qqq.backend.core.processes.implementations.scripts; +import java.io.Serializable; +import java.util.ArrayList; +import java.util.List; import com.kingsrook.qqq.backend.core.actions.scripts.RunAdHocRecordScriptAction; +import com.kingsrook.qqq.backend.core.actions.scripts.logging.StoreScriptLogAndScriptLogLineExecutionLogger; +import com.kingsrook.qqq.backend.core.actions.tables.GetAction; import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.logging.QLogger; +import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryFilterLink; +import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLine; +import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLineInterface; import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; +import com.kingsrook.qqq.backend.core.model.actions.processes.Status; import com.kingsrook.qqq.backend.core.model.actions.scripts.RunAdHocRecordScriptInput; import com.kingsrook.qqq.backend.core.model.actions.scripts.RunAdHocRecordScriptOutput; +import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetOutput; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; import com.kingsrook.qqq.backend.core.model.metadata.code.AdHocScriptCodeReference; +import com.kingsrook.qqq.backend.core.model.scripts.Script; +import com.kingsrook.qqq.backend.core.model.scripts.ScriptLog; import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.AbstractLoadStep; +import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.ProcessSummaryProviderInterface; +import com.kingsrook.qqq.backend.core.utils.CollectionUtils; +import com.kingsrook.qqq.backend.core.utils.StringUtils; +import org.apache.commons.lang.BooleanUtils; +import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; +import static com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator.IN; /******************************************************************************* ** Load step for the runRecordScript process - runs the script on a page of records *******************************************************************************/ -public class RunRecordScriptLoadStep extends AbstractLoadStep +public class RunRecordScriptLoadStep extends AbstractLoadStep implements ProcessSummaryProviderInterface { + private static final QLogger LOG = QLogger.getLogger(RunRecordScriptLoadStep.class); + + private ProcessSummaryLine okLine = new ProcessSummaryLine(Status.OK) + .withSingularPastMessage("had the script ran against it.") + .withPluralPastMessage("had the script ran against them."); + + private List okScriptLogIds = new ArrayList<>(); + private List errorScriptLogIds = new ArrayList<>(); + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public Integer getOverrideRecordPipeCapacity(RunBackendStepInput runBackendStepInput) + { + Integer scriptId = runBackendStepInput.getValueInteger("scriptId"); + try + { + GetInput getInput = new GetInput(); + getInput.setTableName(Script.TABLE_NAME); + getInput.setPrimaryKey(scriptId); + GetOutput getOutput = new GetAction().execute(getInput); + if(getOutput.getRecord() != null) + { + Integer scriptMaxBatchSize = getOutput.getRecord().getValueInteger("maxBatchSize"); + if(scriptMaxBatchSize != null) + { + return (scriptMaxBatchSize); + } + } + } + catch(Exception e) + { + LOG.warn("Error getting script by id: " + scriptId); + } + + return super.getOverrideRecordPipeCapacity(runBackendStepInput); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ @Override public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException { runBackendStepInput.getAsyncJobCallback().updateStatus("Running script"); - RunAdHocRecordScriptInput input = new RunAdHocRecordScriptInput(); - input.setRecordList(runBackendStepInput.getRecords()); - input.setCodeReference(new AdHocScriptCodeReference().withScriptId(runBackendStepInput.getValueInteger("scriptId"))); - RunAdHocRecordScriptOutput output = new RunAdHocRecordScriptOutput(); - new RunAdHocRecordScriptAction().run(input, output); + okLine.incrementCount(runBackendStepInput.getRecords().size()); + + Integer scriptId = runBackendStepInput.getValueInteger("scriptId"); + StoreScriptLogAndScriptLogLineExecutionLogger scriptLogger = new StoreScriptLogAndScriptLogLineExecutionLogger(null, null); // downstream these will get set! + + try + { + RunAdHocRecordScriptInput input = new RunAdHocRecordScriptInput(); + input.setRecordList(runBackendStepInput.getRecords()); + input.setCodeReference(new AdHocScriptCodeReference().withScriptId(scriptId)); + input.setLogger(scriptLogger); + RunAdHocRecordScriptOutput output = new RunAdHocRecordScriptOutput(); + new RunAdHocRecordScriptAction().run(input, output); + } + catch(Exception e) + { + LOG.info("Exception running record script", e, logPair("scriptId", scriptId)); + } + + if(scriptLogger.getScriptLog() != null) + { + Integer id = scriptLogger.getScriptLog().getValueInteger("id"); + if(id != null) + { + boolean hadError = BooleanUtils.isTrue(scriptLogger.getScriptLog().getValueBoolean("hadError")); + (hadError ? errorScriptLogIds : okScriptLogIds).add(id); + } + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public ArrayList getProcessSummary(RunBackendStepOutput runBackendStepOutput, boolean isForResultScreen) + { + ArrayList summary = new ArrayList<>(); + summary.add(okLine); + + if(CollectionUtils.nullSafeHasContents(okScriptLogIds)) + { + summary.add(new ProcessSummaryFilterLink(Status.OK, ScriptLog.TABLE_NAME, new QQueryFilter(new QFilterCriteria("id", IN, okScriptLogIds))) + .withLinkText("Created " + String.format("%,d", okScriptLogIds.size()) + " Successful Script Log" + StringUtils.plural(okScriptLogIds))); + } + + if(CollectionUtils.nullSafeHasContents(errorScriptLogIds)) + { + summary.add(new ProcessSummaryFilterLink(Status.ERROR, ScriptLog.TABLE_NAME, new QQueryFilter(new QFilterCriteria("id", IN, errorScriptLogIds))) + .withLinkText("Created " + String.format("%,d", errorScriptLogIds.size()) + " Script Log" + StringUtils.plural(errorScriptLogIds) + " with Errors")); + } + + return (summary); } }