mirror of
https://github.com/Kingsrook/qqq.git
synced 2025-07-18 05:01:07 +00:00
Add script maxBatchSize, to influence pipe capacity to avoid pipe-full-too-long errors; add link to script logs after running process; add logs to script view screen
This commit is contained in:
@ -145,6 +145,7 @@ public class RunAdHocRecordScriptAction
|
||||
new ExecuteCodeAction().run(executeCodeInput, executeCodeOutput);
|
||||
|
||||
output.setOutput(executeCodeOutput.getOutput());
|
||||
output.setLogger(executionLogger);
|
||||
}
|
||||
|
||||
|
||||
|
||||
@ -23,6 +23,7 @@ package com.kingsrook.qqq.backend.core.model.actions.scripts;
|
||||
|
||||
|
||||
import java.io.Serializable;
|
||||
import com.kingsrook.qqq.backend.core.actions.scripts.logging.QCodeExecutionLoggerInterface;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.AbstractActionOutput;
|
||||
|
||||
|
||||
@ -33,6 +34,8 @@ public class RunAdHocRecordScriptOutput extends AbstractActionOutput
|
||||
{
|
||||
private Serializable output;
|
||||
|
||||
private QCodeExecutionLoggerInterface logger;
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
@ -67,4 +70,35 @@ public class RunAdHocRecordScriptOutput extends AbstractActionOutput
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for logger
|
||||
*******************************************************************************/
|
||||
public QCodeExecutionLoggerInterface getLogger()
|
||||
{
|
||||
return (this.logger);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Setter for logger
|
||||
*******************************************************************************/
|
||||
public void setLogger(QCodeExecutionLoggerInterface logger)
|
||||
{
|
||||
this.logger = logger;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Fluent setter for logger
|
||||
*******************************************************************************/
|
||||
public RunAdHocRecordScriptOutput withLogger(QCodeExecutionLoggerInterface logger)
|
||||
{
|
||||
this.logger = logger;
|
||||
return (this);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -55,6 +55,9 @@ public class Script extends QRecordEntity
|
||||
@QField(possibleValueSourceName = TablesPossibleValueSourceMetaDataProvider.NAME)
|
||||
private String tableName;
|
||||
|
||||
@QField()
|
||||
private Integer maxBatchSize;
|
||||
|
||||
@QField(possibleValueSourceName = "scriptRevision")
|
||||
private Integer currentScriptRevisionId;
|
||||
|
||||
@ -314,4 +317,35 @@ public class Script extends QRecordEntity
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for maxBatchSize
|
||||
*******************************************************************************/
|
||||
public Integer getMaxBatchSize()
|
||||
{
|
||||
return (this.maxBatchSize);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Setter for maxBatchSize
|
||||
*******************************************************************************/
|
||||
public void setMaxBatchSize(Integer maxBatchSize)
|
||||
{
|
||||
this.maxBatchSize = maxBatchSize;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Fluent setter for maxBatchSize
|
||||
*******************************************************************************/
|
||||
public Script withMaxBatchSize(Integer maxBatchSize)
|
||||
{
|
||||
this.maxBatchSize = maxBatchSize;
|
||||
return (this);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -57,10 +57,10 @@ import com.kingsrook.qqq.backend.core.model.metadata.tables.QFieldSection;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.tables.TablesPossibleValueSourceMetaDataProvider;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.tables.Tier;
|
||||
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.NoopTransformStep;
|
||||
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess;
|
||||
import com.kingsrook.qqq.backend.core.processes.implementations.scripts.RunRecordScriptExtractStep;
|
||||
import com.kingsrook.qqq.backend.core.processes.implementations.scripts.RunRecordScriptLoadStep;
|
||||
import com.kingsrook.qqq.backend.core.processes.implementations.scripts.RunRecordScriptTransformStep;
|
||||
import com.kingsrook.qqq.backend.core.processes.implementations.scripts.StoreScriptRevisionProcessStep;
|
||||
import com.kingsrook.qqq.backend.core.processes.implementations.scripts.TestScriptProcessStep;
|
||||
|
||||
@ -138,7 +138,7 @@ public class ScriptsMetaDataProvider
|
||||
.withIcon(new QIcon().withName("data_object"))
|
||||
.withSupportsFullValidation(false)
|
||||
.withExtractStepClass(RunRecordScriptExtractStep.class)
|
||||
.withTransformStepClass(NoopTransformStep.class)
|
||||
.withTransformStepClass(RunRecordScriptTransformStep.class)
|
||||
.withLoadStepClass(RunRecordScriptLoadStep.class)
|
||||
.getProcessMetaData();
|
||||
|
||||
@ -165,6 +165,10 @@ public class ScriptsMetaDataProvider
|
||||
.withLabel("Log Lines")
|
||||
.getWidgetMetaData());
|
||||
|
||||
instance.addWidget(ChildRecordListRenderer.widgetMetaDataBuilder(instance.getJoin(QJoinMetaData.makeInferredJoinName(Script.TABLE_NAME, ScriptLog.TABLE_NAME))).withMaxRows(50)
|
||||
.withLabel("Recent Logs")
|
||||
.getWidgetMetaData());
|
||||
|
||||
instance.addWidget(new QWidgetMetaData()
|
||||
.withName("scriptViewer")
|
||||
.withLabel("Contents")
|
||||
@ -211,6 +215,14 @@ public class ScriptsMetaDataProvider
|
||||
.withOrderBy(new QFilterOrderBy("id"))
|
||||
.withInferredName());
|
||||
|
||||
instance.addJoin(new QJoinMetaData()
|
||||
.withType(JoinType.ONE_TO_MANY)
|
||||
.withLeftTable(Script.TABLE_NAME)
|
||||
.withRightTable(ScriptLog.TABLE_NAME)
|
||||
.withJoinOn(new JoinOn("id", "scriptId"))
|
||||
.withOrderBy(new QFilterOrderBy("id"))
|
||||
.withInferredName());
|
||||
|
||||
}
|
||||
|
||||
|
||||
@ -308,11 +320,17 @@ public class ScriptsMetaDataProvider
|
||||
*******************************************************************************/
|
||||
private QTableMetaData defineTableTriggerTable(String backendName) throws QException
|
||||
{
|
||||
return (defineStandardTable(backendName, TableTrigger.TABLE_NAME, TableTrigger.class)
|
||||
QTableMetaData tableMetaData = defineStandardTable(backendName, TableTrigger.TABLE_NAME, TableTrigger.class)
|
||||
.withRecordLabelFields("id")
|
||||
.withSection(new QFieldSection("identity", new QIcon().withName("badge"), Tier.T1, List.of("id")))
|
||||
.withSection(new QFieldSection("contents", new QIcon().withName("data_object"), Tier.T2, List.of("tableName", "filterId", "scriptId", "priority", "postInsert", "postUpdate")))
|
||||
.withSection(new QFieldSection("dates", new QIcon().withName("calendar_month"), Tier.T3, List.of("createDate", "modifyDate"))));
|
||||
.withSection(new QFieldSection("dates", new QIcon().withName("calendar_month"), Tier.T3, List.of("createDate", "modifyDate")));
|
||||
|
||||
tableMetaData.getField("scriptId").withPossibleValueSourceFilter(new QQueryFilter(
|
||||
new QFilterCriteria("scriptType.name", QCriteriaOperator.EQUALS, SCRIPT_TYPE_NAME_RECORD)
|
||||
));
|
||||
|
||||
return tableMetaData;
|
||||
}
|
||||
|
||||
|
||||
@ -323,12 +341,15 @@ public class ScriptsMetaDataProvider
|
||||
private QTableMetaData defineScriptTable(String backendName) throws QException
|
||||
{
|
||||
QTableMetaData tableMetaData = defineStandardTable(backendName, Script.TABLE_NAME, Script.class)
|
||||
.withSection(new QFieldSection("identity", new QIcon().withName("badge"), Tier.T1, List.of("id", "name", "scriptTypeId", "tableName", "currentScriptRevisionId")))
|
||||
.withSection(new QFieldSection("identity", new QIcon().withName("badge"), Tier.T1, List.of("id", "name", "scriptTypeId", "currentScriptRevisionId")))
|
||||
.withSection(new QFieldSection("recordScriptSettings", new QIcon().withName("table_rows"), Tier.T2, List.of("tableName", "maxBatchSize")))
|
||||
.withSection(new QFieldSection("contents", new QIcon().withName("data_object"), Tier.T2).withWidgetName("scriptViewer"))
|
||||
.withSection(new QFieldSection("dates", new QIcon().withName("calendar_month"), Tier.T3, List.of("createDate", "modifyDate")));
|
||||
.withSection(new QFieldSection("dates", new QIcon().withName("calendar_month"), Tier.T3, List.of("createDate", "modifyDate")))
|
||||
.withSection(new QFieldSection("lines", new QIcon().withName("horizontal_rule"), Tier.T2).withWidgetName(QJoinMetaData.makeInferredJoinName(Script.TABLE_NAME, ScriptLog.TABLE_NAME)));
|
||||
|
||||
tableMetaData.getField("name").withFieldAdornment(AdornmentType.Size.LARGE.toAdornment());
|
||||
tableMetaData.getField("currentScriptRevisionId").withFieldAdornment(AdornmentType.Size.LARGE.toAdornment());
|
||||
tableMetaData.getField("currentScriptRevisionId").withIsEditable(false);
|
||||
|
||||
return (tableMetaData);
|
||||
}
|
||||
|
||||
@ -22,9 +22,13 @@
|
||||
package com.kingsrook.qqq.backend.core.processes.implementations.scripts;
|
||||
|
||||
|
||||
import com.kingsrook.qqq.backend.core.actions.tables.GetAction;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetOutput;
|
||||
import com.kingsrook.qqq.backend.core.model.scripts.Script;
|
||||
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.ExtractViaQueryStep;
|
||||
import com.kingsrook.qqq.backend.core.utils.StringUtils;
|
||||
|
||||
@ -53,6 +57,16 @@ public class RunRecordScriptExtractStep extends ExtractViaQueryStep
|
||||
|
||||
runBackendStepInput.addValue(FIELD_SOURCE_TABLE, tableName);
|
||||
|
||||
Integer scriptId = runBackendStepInput.getValueInteger("scriptId");
|
||||
GetInput getInput = new GetInput();
|
||||
getInput.setTableName(Script.TABLE_NAME);
|
||||
getInput.setPrimaryKey(scriptId);
|
||||
GetOutput getOutput = new GetAction().execute(getInput);
|
||||
if(getOutput.getRecord() != null)
|
||||
{
|
||||
runBackendStepOutput.addValue("scriptName", getOutput.getRecord().getValueString("name"));
|
||||
}
|
||||
|
||||
super.preRun(runBackendStepInput, runBackendStepOutput);
|
||||
}
|
||||
|
||||
|
||||
@ -22,30 +22,147 @@
|
||||
package com.kingsrook.qqq.backend.core.processes.implementations.scripts;
|
||||
|
||||
|
||||
import java.io.Serializable;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import com.kingsrook.qqq.backend.core.actions.scripts.RunAdHocRecordScriptAction;
|
||||
import com.kingsrook.qqq.backend.core.actions.scripts.logging.StoreScriptLogAndScriptLogLineExecutionLogger;
|
||||
import com.kingsrook.qqq.backend.core.actions.tables.GetAction;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.logging.QLogger;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryFilterLink;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLine;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLineInterface;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.processes.Status;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.scripts.RunAdHocRecordScriptInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.scripts.RunAdHocRecordScriptOutput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetOutput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.code.AdHocScriptCodeReference;
|
||||
import com.kingsrook.qqq.backend.core.model.scripts.Script;
|
||||
import com.kingsrook.qqq.backend.core.model.scripts.ScriptLog;
|
||||
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.AbstractLoadStep;
|
||||
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.ProcessSummaryProviderInterface;
|
||||
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
|
||||
import com.kingsrook.qqq.backend.core.utils.StringUtils;
|
||||
import org.apache.commons.lang.BooleanUtils;
|
||||
import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
|
||||
import static com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator.IN;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Load step for the runRecordScript process - runs the script on a page of records
|
||||
*******************************************************************************/
|
||||
public class RunRecordScriptLoadStep extends AbstractLoadStep
|
||||
public class RunRecordScriptLoadStep extends AbstractLoadStep implements ProcessSummaryProviderInterface
|
||||
{
|
||||
private static final QLogger LOG = QLogger.getLogger(RunRecordScriptLoadStep.class);
|
||||
|
||||
private ProcessSummaryLine okLine = new ProcessSummaryLine(Status.OK)
|
||||
.withSingularPastMessage("had the script ran against it.")
|
||||
.withPluralPastMessage("had the script ran against them.");
|
||||
|
||||
private List<Serializable> okScriptLogIds = new ArrayList<>();
|
||||
private List<Serializable> errorScriptLogIds = new ArrayList<>();
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public Integer getOverrideRecordPipeCapacity(RunBackendStepInput runBackendStepInput)
|
||||
{
|
||||
Integer scriptId = runBackendStepInput.getValueInteger("scriptId");
|
||||
try
|
||||
{
|
||||
GetInput getInput = new GetInput();
|
||||
getInput.setTableName(Script.TABLE_NAME);
|
||||
getInput.setPrimaryKey(scriptId);
|
||||
GetOutput getOutput = new GetAction().execute(getInput);
|
||||
if(getOutput.getRecord() != null)
|
||||
{
|
||||
Integer scriptMaxBatchSize = getOutput.getRecord().getValueInteger("maxBatchSize");
|
||||
if(scriptMaxBatchSize != null)
|
||||
{
|
||||
return (scriptMaxBatchSize);
|
||||
}
|
||||
}
|
||||
}
|
||||
catch(Exception e)
|
||||
{
|
||||
LOG.warn("Error getting script by id: " + scriptId);
|
||||
}
|
||||
|
||||
return super.getOverrideRecordPipeCapacity(runBackendStepInput);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException
|
||||
{
|
||||
runBackendStepInput.getAsyncJobCallback().updateStatus("Running script");
|
||||
|
||||
RunAdHocRecordScriptInput input = new RunAdHocRecordScriptInput();
|
||||
input.setRecordList(runBackendStepInput.getRecords());
|
||||
input.setCodeReference(new AdHocScriptCodeReference().withScriptId(runBackendStepInput.getValueInteger("scriptId")));
|
||||
RunAdHocRecordScriptOutput output = new RunAdHocRecordScriptOutput();
|
||||
new RunAdHocRecordScriptAction().run(input, output);
|
||||
okLine.incrementCount(runBackendStepInput.getRecords().size());
|
||||
|
||||
Integer scriptId = runBackendStepInput.getValueInteger("scriptId");
|
||||
StoreScriptLogAndScriptLogLineExecutionLogger scriptLogger = new StoreScriptLogAndScriptLogLineExecutionLogger(null, null); // downstream these will get set!
|
||||
|
||||
try
|
||||
{
|
||||
RunAdHocRecordScriptInput input = new RunAdHocRecordScriptInput();
|
||||
input.setRecordList(runBackendStepInput.getRecords());
|
||||
input.setCodeReference(new AdHocScriptCodeReference().withScriptId(scriptId));
|
||||
input.setLogger(scriptLogger);
|
||||
RunAdHocRecordScriptOutput output = new RunAdHocRecordScriptOutput();
|
||||
new RunAdHocRecordScriptAction().run(input, output);
|
||||
}
|
||||
catch(Exception e)
|
||||
{
|
||||
LOG.info("Exception running record script", e, logPair("scriptId", scriptId));
|
||||
}
|
||||
|
||||
if(scriptLogger.getScriptLog() != null)
|
||||
{
|
||||
Integer id = scriptLogger.getScriptLog().getValueInteger("id");
|
||||
if(id != null)
|
||||
{
|
||||
boolean hadError = BooleanUtils.isTrue(scriptLogger.getScriptLog().getValueBoolean("hadError"));
|
||||
(hadError ? errorScriptLogIds : okScriptLogIds).add(id);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public ArrayList<ProcessSummaryLineInterface> getProcessSummary(RunBackendStepOutput runBackendStepOutput, boolean isForResultScreen)
|
||||
{
|
||||
ArrayList<ProcessSummaryLineInterface> summary = new ArrayList<>();
|
||||
summary.add(okLine);
|
||||
|
||||
if(CollectionUtils.nullSafeHasContents(okScriptLogIds))
|
||||
{
|
||||
summary.add(new ProcessSummaryFilterLink(Status.OK, ScriptLog.TABLE_NAME, new QQueryFilter(new QFilterCriteria("id", IN, okScriptLogIds)))
|
||||
.withLinkText("Created " + String.format("%,d", okScriptLogIds.size()) + " Successful Script Log" + StringUtils.plural(okScriptLogIds)));
|
||||
}
|
||||
|
||||
if(CollectionUtils.nullSafeHasContents(errorScriptLogIds))
|
||||
{
|
||||
summary.add(new ProcessSummaryFilterLink(Status.ERROR, ScriptLog.TABLE_NAME, new QQueryFilter(new QFilterCriteria("id", IN, errorScriptLogIds)))
|
||||
.withLinkText("Created " + String.format("%,d", errorScriptLogIds.size()) + " Script Log" + StringUtils.plural(errorScriptLogIds) + " with Errors"));
|
||||
}
|
||||
|
||||
return (summary);
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user