Add script maxBatchSize, to influence pipe capacity to avoid pipe-full-too-long errors; add link to script logs after running process; add logs to script view screen

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/scripts/RunAdHocRecordScriptAction.java

@@ -145,6 +145,7 @@ public class RunAdHocRecordScriptAction
       new ExecuteCodeAction().run(executeCodeInput, executeCodeOutput);
 
       output.setOutput(executeCodeOutput.getOutput());
+      output.setLogger(executionLogger);
    }
 
 

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/scripts/RunAdHocRecordScriptOutput.java

@@ -23,6 +23,7 @@ package com.kingsrook.qqq.backend.core.model.actions.scripts;
 
 
 import java.io.Serializable;
+import com.kingsrook.qqq.backend.core.actions.scripts.logging.QCodeExecutionLoggerInterface;
 import com.kingsrook.qqq.backend.core.model.actions.AbstractActionOutput;
 
 
@@ -33,6 +34,8 @@ public class RunAdHocRecordScriptOutput extends AbstractActionOutput
 {
    private Serializable output;
 
+   private QCodeExecutionLoggerInterface logger;
+
 
 
    /*******************************************************************************
@@ -67,4 +70,35 @@ public class RunAdHocRecordScriptOutput extends AbstractActionOutput
       return (this);
    }
 
+
+
+   /*******************************************************************************
+    ** Getter for logger
+    *******************************************************************************/
+   public QCodeExecutionLoggerInterface getLogger()
+   {
+      return (this.logger);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for logger
+    *******************************************************************************/
+   public void setLogger(QCodeExecutionLoggerInterface logger)
+   {
+      this.logger = logger;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for logger
+    *******************************************************************************/
+   public RunAdHocRecordScriptOutput withLogger(QCodeExecutionLoggerInterface logger)
+   {
+      this.logger = logger;
+      return (this);
+   }
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/scripts/Script.java

@@ -55,6 +55,9 @@ public class Script extends QRecordEntity
    @QField(possibleValueSourceName = TablesPossibleValueSourceMetaDataProvider.NAME)
    private String tableName;
 
+   @QField()
+   private Integer maxBatchSize;
+
    @QField(possibleValueSourceName = "scriptRevision")
    private Integer currentScriptRevisionId;
 
@@ -314,4 +317,35 @@ public class Script extends QRecordEntity
       return (this);
    }
 
+
+
+   /*******************************************************************************
+    ** Getter for maxBatchSize
+    *******************************************************************************/
+   public Integer getMaxBatchSize()
+   {
+      return (this.maxBatchSize);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for maxBatchSize
+    *******************************************************************************/
+   public void setMaxBatchSize(Integer maxBatchSize)
+   {
+      this.maxBatchSize = maxBatchSize;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for maxBatchSize
+    *******************************************************************************/
+   public Script withMaxBatchSize(Integer maxBatchSize)
+   {
+      this.maxBatchSize = maxBatchSize;
+      return (this);
+   }
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/scripts/ScriptsMetaDataProvider.java

@@ -57,10 +57,10 @@ import com.kingsrook.qqq.backend.core.model.metadata.tables.QFieldSection;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.TablesPossibleValueSourceMetaDataProvider;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.Tier;
-import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.NoopTransformStep;
 import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess;
 import com.kingsrook.qqq.backend.core.processes.implementations.scripts.RunRecordScriptExtractStep;
 import com.kingsrook.qqq.backend.core.processes.implementations.scripts.RunRecordScriptLoadStep;
+import com.kingsrook.qqq.backend.core.processes.implementations.scripts.RunRecordScriptTransformStep;
 import com.kingsrook.qqq.backend.core.processes.implementations.scripts.StoreScriptRevisionProcessStep;
 import com.kingsrook.qqq.backend.core.processes.implementations.scripts.TestScriptProcessStep;
 
@@ -138,7 +138,7 @@ public class ScriptsMetaDataProvider
          .withIcon(new QIcon().withName("data_object"))
          .withSupportsFullValidation(false)
          .withExtractStepClass(RunRecordScriptExtractStep.class)
-         .withTransformStepClass(NoopTransformStep.class)
+         .withTransformStepClass(RunRecordScriptTransformStep.class)
          .withLoadStepClass(RunRecordScriptLoadStep.class)
          .getProcessMetaData();
 
@@ -165,6 +165,10 @@ public class ScriptsMetaDataProvider
          .withLabel("Log Lines")
          .getWidgetMetaData());
 
+      instance.addWidget(ChildRecordListRenderer.widgetMetaDataBuilder(instance.getJoin(QJoinMetaData.makeInferredJoinName(Script.TABLE_NAME, ScriptLog.TABLE_NAME))).withMaxRows(50)
+         .withLabel("Recent Logs")
+         .getWidgetMetaData());
+
       instance.addWidget(new QWidgetMetaData()
          .withName("scriptViewer")
          .withLabel("Contents")
@@ -211,6 +215,14 @@ public class ScriptsMetaDataProvider
          .withOrderBy(new QFilterOrderBy("id"))
          .withInferredName());
 
+      instance.addJoin(new QJoinMetaData()
+         .withType(JoinType.ONE_TO_MANY)
+         .withLeftTable(Script.TABLE_NAME)
+         .withRightTable(ScriptLog.TABLE_NAME)
+         .withJoinOn(new JoinOn("id", "scriptId"))
+         .withOrderBy(new QFilterOrderBy("id"))
+         .withInferredName());
+
    }
 
 
@@ -308,11 +320,17 @@ public class ScriptsMetaDataProvider
     *******************************************************************************/
    private QTableMetaData defineTableTriggerTable(String backendName) throws QException
    {
-      return (defineStandardTable(backendName, TableTrigger.TABLE_NAME, TableTrigger.class)
+      QTableMetaData tableMetaData = defineStandardTable(backendName, TableTrigger.TABLE_NAME, TableTrigger.class)
          .withRecordLabelFields("id")
          .withSection(new QFieldSection("identity", new QIcon().withName("badge"), Tier.T1, List.of("id")))
          .withSection(new QFieldSection("contents", new QIcon().withName("data_object"), Tier.T2, List.of("tableName", "filterId", "scriptId", "priority", "postInsert", "postUpdate")))
-         .withSection(new QFieldSection("dates", new QIcon().withName("calendar_month"), Tier.T3, List.of("createDate", "modifyDate"))));
+         .withSection(new QFieldSection("dates", new QIcon().withName("calendar_month"), Tier.T3, List.of("createDate", "modifyDate")));
+
+      tableMetaData.getField("scriptId").withPossibleValueSourceFilter(new QQueryFilter(
+         new QFilterCriteria("scriptType.name", QCriteriaOperator.EQUALS, SCRIPT_TYPE_NAME_RECORD)
+      ));
+
+      return tableMetaData;
    }
 
 
@@ -323,12 +341,15 @@ public class ScriptsMetaDataProvider
    private QTableMetaData defineScriptTable(String backendName) throws QException
    {
       QTableMetaData tableMetaData = defineStandardTable(backendName, Script.TABLE_NAME, Script.class)
-         .withSection(new QFieldSection("identity", new QIcon().withName("badge"), Tier.T1, List.of("id", "name", "scriptTypeId", "tableName", "currentScriptRevisionId")))
+         .withSection(new QFieldSection("identity", new QIcon().withName("badge"), Tier.T1, List.of("id", "name", "scriptTypeId", "currentScriptRevisionId")))
+         .withSection(new QFieldSection("recordScriptSettings", new QIcon().withName("table_rows"), Tier.T2, List.of("tableName", "maxBatchSize")))
          .withSection(new QFieldSection("contents", new QIcon().withName("data_object"), Tier.T2).withWidgetName("scriptViewer"))
-         .withSection(new QFieldSection("dates", new QIcon().withName("calendar_month"), Tier.T3, List.of("createDate", "modifyDate")));
+         .withSection(new QFieldSection("dates", new QIcon().withName("calendar_month"), Tier.T3, List.of("createDate", "modifyDate")))
+         .withSection(new QFieldSection("lines", new QIcon().withName("horizontal_rule"), Tier.T2).withWidgetName(QJoinMetaData.makeInferredJoinName(Script.TABLE_NAME, ScriptLog.TABLE_NAME)));
 
       tableMetaData.getField("name").withFieldAdornment(AdornmentType.Size.LARGE.toAdornment());
       tableMetaData.getField("currentScriptRevisionId").withFieldAdornment(AdornmentType.Size.LARGE.toAdornment());
+      tableMetaData.getField("currentScriptRevisionId").withIsEditable(false);
 
       return (tableMetaData);
    }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/scripts/RunRecordScriptExtractStep.java

@@ -22,9 +22,13 @@
 package com.kingsrook.qqq.backend.core.processes.implementations.scripts;
 
 
+import com.kingsrook.qqq.backend.core.actions.tables.GetAction;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
 import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetOutput;
+import com.kingsrook.qqq.backend.core.model.scripts.Script;
 import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.ExtractViaQueryStep;
 import com.kingsrook.qqq.backend.core.utils.StringUtils;
 
@@ -53,6 +57,16 @@ public class RunRecordScriptExtractStep extends ExtractViaQueryStep
 
       runBackendStepInput.addValue(FIELD_SOURCE_TABLE, tableName);
 
+      Integer  scriptId = runBackendStepInput.getValueInteger("scriptId");
+      GetInput getInput = new GetInput();
+      getInput.setTableName(Script.TABLE_NAME);
+      getInput.setPrimaryKey(scriptId);
+      GetOutput getOutput = new GetAction().execute(getInput);
+      if(getOutput.getRecord() != null)
+      {
+         runBackendStepOutput.addValue("scriptName", getOutput.getRecord().getValueString("name"));
+      }
+
       super.preRun(runBackendStepInput, runBackendStepOutput);
    }
 

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/scripts/RunRecordScriptLoadStep.java

@@ -22,30 +22,147 @@
 package com.kingsrook.qqq.backend.core.processes.implementations.scripts;
 
 
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.List;
 import com.kingsrook.qqq.backend.core.actions.scripts.RunAdHocRecordScriptAction;
+import com.kingsrook.qqq.backend.core.actions.scripts.logging.StoreScriptLogAndScriptLogLineExecutionLogger;
+import com.kingsrook.qqq.backend.core.actions.tables.GetAction;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.logging.QLogger;
+import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryFilterLink;
+import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLine;
+import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLineInterface;
 import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
 import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
+import com.kingsrook.qqq.backend.core.model.actions.processes.Status;
 import com.kingsrook.qqq.backend.core.model.actions.scripts.RunAdHocRecordScriptInput;
 import com.kingsrook.qqq.backend.core.model.actions.scripts.RunAdHocRecordScriptOutput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetOutput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
 import com.kingsrook.qqq.backend.core.model.metadata.code.AdHocScriptCodeReference;
+import com.kingsrook.qqq.backend.core.model.scripts.Script;
+import com.kingsrook.qqq.backend.core.model.scripts.ScriptLog;
 import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.AbstractLoadStep;
+import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.ProcessSummaryProviderInterface;
+import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
+import com.kingsrook.qqq.backend.core.utils.StringUtils;
+import org.apache.commons.lang.BooleanUtils;
+import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
+import static com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator.IN;
 
 
 /*******************************************************************************
  ** Load step for the runRecordScript process - runs the script on a page of records
  *******************************************************************************/
-public class RunRecordScriptLoadStep extends AbstractLoadStep
+public class RunRecordScriptLoadStep extends AbstractLoadStep implements ProcessSummaryProviderInterface
 {
+   private static final QLogger LOG = QLogger.getLogger(RunRecordScriptLoadStep.class);
+
+   private ProcessSummaryLine okLine = new ProcessSummaryLine(Status.OK)
+      .withSingularPastMessage("had the script ran against it.")
+      .withPluralPastMessage("had the script ran against them.");
+
+   private List<Serializable> okScriptLogIds    = new ArrayList<>();
+   private List<Serializable> errorScriptLogIds = new ArrayList<>();
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public Integer getOverrideRecordPipeCapacity(RunBackendStepInput runBackendStepInput)
+   {
+      Integer scriptId = runBackendStepInput.getValueInteger("scriptId");
+      try
+      {
+         GetInput getInput = new GetInput();
+         getInput.setTableName(Script.TABLE_NAME);
+         getInput.setPrimaryKey(scriptId);
+         GetOutput getOutput = new GetAction().execute(getInput);
+         if(getOutput.getRecord() != null)
+         {
+            Integer scriptMaxBatchSize = getOutput.getRecord().getValueInteger("maxBatchSize");
+            if(scriptMaxBatchSize != null)
+            {
+               return (scriptMaxBatchSize);
+            }
+         }
+      }
+      catch(Exception e)
+      {
+         LOG.warn("Error getting script by id: " + scriptId);
+      }
+
+      return super.getOverrideRecordPipeCapacity(runBackendStepInput);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
    @Override
    public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException
    {
       runBackendStepInput.getAsyncJobCallback().updateStatus("Running script");
 
-      RunAdHocRecordScriptInput input = new RunAdHocRecordScriptInput();
+      okLine.incrementCount(runBackendStepInput.getRecords().size());
-      input.setRecordList(runBackendStepInput.getRecords());
+
-      input.setCodeReference(new AdHocScriptCodeReference().withScriptId(runBackendStepInput.getValueInteger("scriptId")));
+      Integer                                       scriptId     = runBackendStepInput.getValueInteger("scriptId");
-      RunAdHocRecordScriptOutput output = new RunAdHocRecordScriptOutput();
+      StoreScriptLogAndScriptLogLineExecutionLogger scriptLogger = new StoreScriptLogAndScriptLogLineExecutionLogger(null, null); // downstream these will get set!
-      new RunAdHocRecordScriptAction().run(input, output);
+
+      try
+      {
+         RunAdHocRecordScriptInput input = new RunAdHocRecordScriptInput();
+         input.setRecordList(runBackendStepInput.getRecords());
+         input.setCodeReference(new AdHocScriptCodeReference().withScriptId(scriptId));
+         input.setLogger(scriptLogger);
+         RunAdHocRecordScriptOutput output = new RunAdHocRecordScriptOutput();
+         new RunAdHocRecordScriptAction().run(input, output);
+      }
+      catch(Exception e)
+      {
+         LOG.info("Exception running record script", e, logPair("scriptId", scriptId));
+      }
+
+      if(scriptLogger.getScriptLog() != null)
+      {
+         Integer id = scriptLogger.getScriptLog().getValueInteger("id");
+         if(id != null)
+         {
+            boolean hadError = BooleanUtils.isTrue(scriptLogger.getScriptLog().getValueBoolean("hadError"));
+            (hadError ? errorScriptLogIds : okScriptLogIds).add(id);
+         }
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public ArrayList<ProcessSummaryLineInterface> getProcessSummary(RunBackendStepOutput runBackendStepOutput, boolean isForResultScreen)
+   {
+      ArrayList<ProcessSummaryLineInterface> summary = new ArrayList<>();
+      summary.add(okLine);
+
+      if(CollectionUtils.nullSafeHasContents(okScriptLogIds))
+      {
+         summary.add(new ProcessSummaryFilterLink(Status.OK, ScriptLog.TABLE_NAME, new QQueryFilter(new QFilterCriteria("id", IN, okScriptLogIds)))
+            .withLinkText("Created " + String.format("%,d", okScriptLogIds.size()) + " Successful Script Log" + StringUtils.plural(okScriptLogIds)));
+      }
+
+      if(CollectionUtils.nullSafeHasContents(errorScriptLogIds))
+      {
+         summary.add(new ProcessSummaryFilterLink(Status.ERROR, ScriptLog.TABLE_NAME, new QQueryFilter(new QFilterCriteria("id", IN, errorScriptLogIds)))
+            .withLinkText("Created " + String.format("%,d", errorScriptLogIds.size()) + " Script Log" + StringUtils.plural(errorScriptLogIds) + " with Errors"));
+      }
+
+      return (summary);
    }
 }