QRunIO/qqq
2
0
Fork 0
mirror of https://github.com/Kingsrook/qqq.git synced 2025-07-18 05:01:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

Update to a workable MVP version of running scripts as table automations

Browse Source
This commit is contained in:
darin.kelkhoff
2023-03-07 15:18:08 -06:00
parent eafa82eb85
commit d0fb8c33e9
5 changed files with 211 additions and 66 deletions
Download Patch File Download Diff File Expand all files Collapse all files

63
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/automation/RecordAutomationStatusUpdater.java
View File

@ -25,10 +25,18 @@ package com.kingsrook.qqq.backend.core.actions.automation;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
import com.kingsrook.qqq.backend.core.actions.tables.CountAction;
import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.logging.QLogger;
import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountOutput;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
import com.kingsrook.qqq.backend.core.model.automation.TableTrigger;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
@ -119,14 +127,63 @@ public class RecordAutomationStatusUpdater
if(automationStatus.equals(AutomationStatus.PENDING_INSERT_AUTOMATIONS))
{
return tableActions.stream().noneMatch(a -> TriggerEvent.POST_INSERT.equals(a.getTriggerEvent()));
if(tableActions.stream().anyMatch(a -> TriggerEvent.POST_INSERT.equals(a.getTriggerEvent())))
{
return (false);
}
else if(areThereTableTriggersForTable(table, TriggerEvent.POST_INSERT))
{
return (false);
}
}
else if(automationStatus.equals(AutomationStatus.PENDING_UPDATE_AUTOMATIONS))
{
return tableActions.stream().noneMatch(a -> TriggerEvent.POST_UPDATE.equals(a.getTriggerEvent()));
if(tableActions.stream().anyMatch(a -> TriggerEvent.POST_UPDATE.equals(a.getTriggerEvent())))
{
return (false);
}
else if(areThereTableTriggersForTable(table, TriggerEvent.POST_UPDATE))
{
return (false);
}
}
return (false);
return (true);
}
/*******************************************************************************
**
*******************************************************************************/
private static boolean areThereTableTriggersForTable(QTableMetaData table, TriggerEvent triggerEvent)
{
if(QContext.getQInstance().getTable(TableTrigger.TABLE_NAME) == null)
{
return (false);
}
try
{
///////////////////
// todo - cache? //
///////////////////
CountInput countInput = new CountInput();
countInput.setTableName(TableTrigger.TABLE_NAME);
countInput.setFilter(new QQueryFilter(
new QFilterCriteria("tableName", QCriteriaOperator.EQUALS, table.getName()),
new QFilterCriteria(triggerEvent.equals(TriggerEvent.POST_INSERT) ? "postInsert" : "postUpdate", QCriteriaOperator.EQUALS, true)
));
CountOutput countOutput = new CountAction().execute(countInput);
return (countOutput.getCount() != null && countOutput.getCount() > 0);
}
catch(Exception e)
{
///////////////////////////////////////////////////////////////////////////////////////////////////////////
// if the count query failed, we're a bit safer to err on the side of "yeah, there might be automations" //
///////////////////////////////////////////////////////////////////////////////////////////////////////////
return (true);
}
}

52
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/automation/RunRecordScriptsAutomationHandler.java → qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/automation/RunRecordScriptAutomationHandler.java
View File

@ -22,6 +22,8 @@
package com.kingsrook.qqq.backend.core.actions.automation;
import java.io.Serializable;
import java.util.Map;
import com.kingsrook.qqq.backend.core.actions.scripts.RunAdHocRecordScriptAction;
import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
import com.kingsrook.qqq.backend.core.context.QContext;
@ -40,17 +42,17 @@ import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.code.AdHocScriptCodeReference;
import com.kingsrook.qqq.backend.core.model.scripts.Script;
import com.kingsrook.qqq.backend.core.model.scripts.ScriptRevision;
import com.kingsrook.qqq.backend.core.model.scripts.ScriptType;
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
import com.kingsrook.qqq.backend.core.utils.ValueUtils;
import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
/*******************************************************************************
**
*******************************************************************************/
public class RunRecordScriptsAutomationHandler extends RecordAutomationHandler
public class RunRecordScriptAutomationHandler extends RecordAutomationHandler
{
private static final QLogger LOG = QLogger.getLogger(RunRecordScriptsAutomationHandler.class);
private static final QLogger LOG = QLogger.getLogger(RunRecordScriptAutomationHandler.class);
@ -60,30 +62,34 @@ public class RunRecordScriptsAutomationHandler extends RecordAutomationHandler
@Override
public void execute(RecordAutomationInput recordAutomationInput) throws QException
{
String tableName = recordAutomationInput.getTableName();
QueryInput queryInput = new QueryInput();
queryInput.setTableName(ScriptRevision.TABLE_NAME);
queryInput.setFilter(new QQueryFilter(
new QFilterCriteria("script.tableName", QCriteriaOperator.EQUALS, tableName),
new QFilterCriteria("scriptType.name", QCriteriaOperator.EQUALS, "Record Script") // todo... no. something about post-insert/update?
));
queryInput.withQueryJoin(new QueryJoin(Script.TABLE_NAME).withBaseTableOrAlias(ScriptRevision.TABLE_NAME).withJoinMetaData(QContext.getQInstance().getJoin("currentScriptRevision")));
queryInput.withQueryJoin(new QueryJoin(ScriptType.TABLE_NAME).withBaseTableOrAlias(Script.TABLE_NAME));
String tableName = recordAutomationInput.getTableName();
Map<String, Serializable> values = recordAutomationInput.getAction().getValues();
Integer scriptId = ValueUtils.getValueAsInteger(values.get("scriptId"));
QueryOutput queryOutput = new QueryAction().execute(queryInput);
for(QRecord scriptRevision : CollectionUtils.nonNullList(queryOutput.getRecords()))
if(scriptId == null)
{
// todo - refresh the records if more than 1 script
LOG.info("Running script against records", logPair("scriptRevisionId", scriptRevision.getValue("id")), logPair("scriptId", scriptRevision.getValue("scriptIdd")));
RunAdHocRecordScriptInput input = new RunAdHocRecordScriptInput();
input.setCodeReference(new AdHocScriptCodeReference().withScriptRevisionRecord(scriptRevision));
input.setTableName(tableName);
input.setRecordList(recordAutomationInput.getRecordList());
RunAdHocRecordScriptOutput output = new RunAdHocRecordScriptOutput();
new RunAdHocRecordScriptAction().run(input, output);
throw (new QException("ScriptId was not provided in values map for record automations on table: " + tableName));
}
QueryInput queryInput = new QueryInput();
queryInput.setTableName(ScriptRevision.TABLE_NAME);
queryInput.setFilter(new QQueryFilter(new QFilterCriteria("scriptId", QCriteriaOperator.EQUALS, scriptId)));
queryInput.withQueryJoin(new QueryJoin(Script.TABLE_NAME).withBaseTableOrAlias(ScriptRevision.TABLE_NAME).withJoinMetaData(QContext.getQInstance().getJoin("currentScriptRevision")));
QueryOutput queryOutput = new QueryAction().execute(queryInput);
if(CollectionUtils.nullSafeIsEmpty(queryOutput.getRecords()))
{
throw (new QException("Could not find current revision for scriptId: " + scriptId + " on table " + tableName));
}
QRecord scriptRevision = queryOutput.getRecords().get(0);
LOG.info("Running script against records", logPair("scriptRevisionId", scriptRevision.getValue("id")), logPair("scriptId", scriptRevision.getValue("scriptIdd")));
RunAdHocRecordScriptInput input = new RunAdHocRecordScriptInput();
input.setCodeReference(new AdHocScriptCodeReference().withScriptRevisionRecord(scriptRevision));
input.setTableName(tableName);
input.setRecordList(recordAutomationInput.getRecordList());
RunAdHocRecordScriptOutput output = new RunAdHocRecordScriptOutput();
new RunAdHocRecordScriptAction().run(input, output);
}
}

116
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/automation/polling/PollingAutomationPerTableRunner.java
View File

@ -25,15 +25,16 @@ package com.kingsrook.qqq.backend.core.actions.automation.polling;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.function.Supplier;
import java.util.stream.Collectors;
import com.kingsrook.qqq.backend.core.actions.async.AsyncRecordPipeLoop;
import com.kingsrook.qqq.backend.core.actions.automation.AutomationStatus;
import com.kingsrook.qqq.backend.core.actions.automation.RecordAutomationHandler;
import com.kingsrook.qqq.backend.core.actions.automation.RecordAutomationStatusUpdater;
import com.kingsrook.qqq.backend.core.actions.automation.RunRecordScriptAutomationHandler;
import com.kingsrook.qqq.backend.core.actions.customizers.QCodeLoader;
import com.kingsrook.qqq.backend.core.actions.processes.QProcessCallback;
import com.kingsrook.qqq.backend.core.actions.processes.RunProcessAction;
@ -49,9 +50,12 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput;
import com.kingsrook.qqq.backend.core.model.automation.RecordAutomationInput;
import com.kingsrook.qqq.backend.core.model.automation.TableTrigger;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.AutomationStatusTrackingType;
import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.QTableAutomationDetails;
@ -60,6 +64,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.TriggerEv
import com.kingsrook.qqq.backend.core.model.session.QSession;
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
import com.kingsrook.qqq.backend.core.utils.StringUtils;
import com.kingsrook.qqq.backend.core.utils.collections.MapBuilder;
import org.apache.commons.lang.NotImplementedException;
@ -87,6 +92,11 @@ public class PollingAutomationPerTableRunner implements Runnable
TriggerEvent.POST_UPDATE, AutomationStatus.PENDING_UPDATE_AUTOMATIONS
);
private static Map<AutomationStatus, TriggerEvent> automationStatusTriggerEventMap = Map.of(
AutomationStatus.PENDING_INSERT_AUTOMATIONS, TriggerEvent.POST_INSERT,
AutomationStatus.PENDING_UPDATE_AUTOMATIONS, TriggerEvent.POST_UPDATE
);
private static Map<AutomationStatus, AutomationStatus> pendingToRunningStatusMap = Map.of(
AutomationStatus.PENDING_INSERT_AUTOMATIONS, AutomationStatus.RUNNING_INSERT_AUTOMATIONS,
AutomationStatus.PENDING_UPDATE_AUTOMATIONS, AutomationStatus.RUNNING_UPDATE_AUTOMATIONS
@ -102,51 +112,26 @@ public class PollingAutomationPerTableRunner implements Runnable
/*******************************************************************************
**
*******************************************************************************/
public record TableActions(String tableName, AutomationStatus status, List<TableAutomationAction> actions)
public record TableActions(String tableName, AutomationStatus status)
{
}
/*******************************************************************************
**
** basically just get a list of tables which at least *could* have automations
** run - either meta-data automations, or table-triggers (data/user defined).
*******************************************************************************/
public static List<TableActions> getTableActions(QInstance instance, String providerName)
{
Map<String, Map<AutomationStatus, List<TableAutomationAction>>> workingTableActionMap = new HashMap<>();
List<TableActions> tableActionList = new ArrayList<>();
List<TableActions> tableActionList = new ArrayList<>();
//////////////////////////////////////////////////////////////////////
// todo - share logic like this among any automation implementation //
//////////////////////////////////////////////////////////////////////
for(QTableMetaData table : instance.getTables().values())
{
if(table.getAutomationDetails() != null && providerName.equals(table.getAutomationDetails().getProviderName()))
{
///////////////////////////////////////////////////////////////////////////
// organize the table's actions by type //
// todo - in future, need user-defined actions here too (and refreshed!) //
///////////////////////////////////////////////////////////////////////////
for(TableAutomationAction action : table.getAutomationDetails().getActions())
{
AutomationStatus automationStatus = triggerEventAutomationStatusMap.get(action.getTriggerEvent());
workingTableActionMap.putIfAbsent(table.getName(), new HashMap<>());
workingTableActionMap.get(table.getName()).putIfAbsent(automationStatus, new ArrayList<>());
workingTableActionMap.get(table.getName()).get(automationStatus).add(action);
}
////////////////////////////////////////////
// convert the map to tableAction records //
////////////////////////////////////////////
for(Map.Entry<AutomationStatus, List<TableAutomationAction>> entry : workingTableActionMap.get(table.getName()).entrySet())
{
AutomationStatus automationStatus = entry.getKey();
List<TableAutomationAction> actionList = entry.getValue();
actionList.sort(Comparator.comparing(TableAutomationAction::getPriority));
tableActionList.add(new TableActions(table.getName(), automationStatus, actionList));
}
tableActionList.add(new TableActions(table.getName(), AutomationStatus.PENDING_INSERT_AUTOMATIONS));
tableActionList.add(new TableActions(table.getName(), AutomationStatus.PENDING_UPDATE_AUTOMATIONS));
}
}
@ -183,7 +168,7 @@ public class PollingAutomationPerTableRunner implements Runnable
try
{
QSession session = sessionSupplier != null ? sessionSupplier.get() : new QSession();
processTableInsertOrUpdate(instance.getTable(tableActions.tableName()), session, tableActions.status(), tableActions.actions());
processTableInsertOrUpdate(instance.getTable(tableActions.tableName()), session, tableActions.status());
}
catch(Exception e)
{
@ -201,8 +186,12 @@ public class PollingAutomationPerTableRunner implements Runnable
/*******************************************************************************
** Query for and process records that have a PENDING_INSERT or PENDING_UPDATE status on a given table.
*******************************************************************************/
public void processTableInsertOrUpdate(QTableMetaData table, QSession session, AutomationStatus automationStatus, List<TableAutomationAction> actions) throws QException
public void processTableInsertOrUpdate(QTableMetaData table, QSession session, AutomationStatus automationStatus) throws QException
{
/////////////////////////////////////////////////////////////////////////
// get the actions to run against this table in this automation status //
/////////////////////////////////////////////////////////////////////////
List<TableAutomationAction> actions = getTableActions(table, automationStatus);
if(CollectionUtils.nullSafeIsEmpty(actions))
{
return;
@ -247,6 +236,56 @@ public class PollingAutomationPerTableRunner implements Runnable
/*******************************************************************************
** get the actions to run against a table in an automation status. both from
** metaData and tableTriggers/data.
*******************************************************************************/
private List<TableAutomationAction> getTableActions(QTableMetaData table, AutomationStatus automationStatus) throws QException
{
List<TableAutomationAction> rs = new ArrayList<>();
TriggerEvent triggerEvent = automationStatusTriggerEventMap.get(automationStatus);
///////////////////////////////////////////////////////////
// start with any actions defined in the table meta data //
///////////////////////////////////////////////////////////
for(TableAutomationAction action : table.getAutomationDetails().getActions())
{
if(action.getTriggerEvent().equals(triggerEvent))
{
rs.add(action);
}
}
/////////////////////////////////////////////////
// next add any tableTriggers, defined in data //
/////////////////////////////////////////////////
QueryInput queryInput = new QueryInput();
queryInput.setTableName(TableTrigger.TABLE_NAME);
queryInput.setFilter(new QQueryFilter(
new QFilterCriteria("tableName", QCriteriaOperator.EQUALS, table.getName()),
new QFilterCriteria(triggerEvent.equals(TriggerEvent.POST_INSERT) ? "postInsert" : "postUpdate", QCriteriaOperator.EQUALS, true)
));
QueryOutput queryOutput = new QueryAction().execute(queryInput);
for(QRecord record : queryOutput.getRecords())
{
// todo - get filter if there is/was one
rs.add(new TableAutomationAction()
.withName("Script:" + record.getValue("scriptId"))
.withFilter(null)
.withTriggerEvent(triggerEvent)
.withPriority(record.getValueInteger("priority"))
.withCodeReference(new QCodeReference(RunRecordScriptAutomationHandler.class))
.withValues(MapBuilder.of("scriptId", record.getValue("scriptId")))
);
}
rs.sort(Comparator.comparing(taa -> Objects.requireNonNullElse(taa.getPriority(), Integer.MAX_VALUE)));
return (rs);
}
/*******************************************************************************
** For a set of records that were found to be in a PENDING state - run all the
** table's actions against them - IF they are found to match the action's filter
@ -275,12 +314,12 @@ public class PollingAutomationPerTableRunner implements Runnable
//////////////////////////////////////////////////////////////////////////////////////////////////////////////
// note - this method - will re-query the objects, so we should have confidence that their data is fresh... //
//////////////////////////////////////////////////////////////////////////////////////////////////////////////
List<QRecord> matchingQRecords = getRecordsMatchingActionFilter(session, table, records, action);
List<QRecord> matchingQRecords = getRecordsMatchingActionFilter(table, records, action);
LOG.debug("Of the {} records that were pending automations, {} of them match the filter on the action {}", records.size(), matchingQRecords.size(), action);
if(CollectionUtils.nullSafeHasContents(matchingQRecords))
{
LOG.debug(" Processing " + matchingQRecords.size() + " records in " + table + " for action " + action);
applyActionToMatchingRecords(instance, session, table, matchingQRecords, action);
applyActionToMatchingRecords(table, matchingQRecords, action);
}
}
catch(Exception e)
@ -317,7 +356,7 @@ public class PollingAutomationPerTableRunner implements Runnable
** but that will almost certainly give potentially different results than a true
** backend - e.g., just consider if the DB is case-sensitive for strings...
*******************************************************************************/
private List<QRecord> getRecordsMatchingActionFilter(QSession session, QTableMetaData table, List<QRecord> records, TableAutomationAction action) throws QException
private List<QRecord> getRecordsMatchingActionFilter(QTableMetaData table, List<QRecord> records, TableAutomationAction action) throws QException
{
QueryInput queryInput = new QueryInput();
queryInput.setTableName(table.getName());
@ -359,7 +398,7 @@ public class PollingAutomationPerTableRunner implements Runnable
** Finally, actually run action code against a list of known matching records.
** todo not commit - move to somewhere genericer
*******************************************************************************/
public static void applyActionToMatchingRecords(QInstance instance, QSession session, QTableMetaData table, List<QRecord> records, TableAutomationAction action) throws Exception
public static void applyActionToMatchingRecords(QTableMetaData table, List<QRecord> records, TableAutomationAction action) throws Exception
{
if(StringUtils.hasContent(action.getProcessName()))
{
@ -403,6 +442,7 @@ public class PollingAutomationPerTableRunner implements Runnable
RecordAutomationInput input = new RecordAutomationInput();
input.setTableName(table.getName());
input.setRecordList(records);
input.setAction(action);
RecordAutomationHandler recordAutomationHandler = QCodeLoader.getRecordAutomationHandler(action);
recordAutomationHandler.execute(input);

44
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/automation/TableAutomationAction.java
View File

@ -22,6 +22,8 @@
package com.kingsrook.qqq.backend.core.model.metadata.tables.automation;
import java.io.Serializable;
import java.util.Map;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
@ -36,6 +38,8 @@ public class TableAutomationAction
private Integer priority = 500;
private QQueryFilter filter;
private Map<String, Serializable> values;
////////////////////////////////
// mutually-exclusive options //
////////////////////////////////
@ -50,7 +54,8 @@ public class TableAutomationAction
@Override
public String toString()
{
return "TableAutomationAction{name='" + name + "'}";}
return "TableAutomationAction{name='" + name + "'}";
}
@ -75,6 +80,7 @@ public class TableAutomationAction
}
/*******************************************************************************
** Fluent setter for name
**
@ -108,6 +114,7 @@ public class TableAutomationAction
}
/*******************************************************************************
** Fluent setter for triggerEvent
**
@ -141,6 +148,7 @@ public class TableAutomationAction
}
/*******************************************************************************
** Fluent setter for priority
**
@ -174,6 +182,7 @@ public class TableAutomationAction
}
/*******************************************************************************
** Fluent setter for filter
**
@ -207,6 +216,7 @@ public class TableAutomationAction
}
/*******************************************************************************
** Fluent setter for codeReference
**
@ -240,6 +250,7 @@ public class TableAutomationAction
}
/*******************************************************************************
** Fluent setter for processName
**
@ -250,4 +261,35 @@ public class TableAutomationAction
return (this);
}
/*******************************************************************************
** Getter for values
*******************************************************************************/
public Map<String, Serializable> getValues()
{
return (this.values);
}
/*******************************************************************************
** Setter for values
*******************************************************************************/
public void setValues(Map<String, Serializable> values)
{
this.values = values;
}
/*******************************************************************************
** Fluent setter for values
*******************************************************************************/
public TableAutomationAction withValues(Map<String, Serializable> values)
{
this.values = values;
return (this);
}
}

2
qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/automation/polling/PollingAutomationPerTableRunnerTest.java
View File

@ -164,7 +164,7 @@ class PollingAutomationPerTableRunnerTest extends BaseTest
/////////////////////////////////////////////////////////////////////////////////////////////////////
// note - don't call run - it is meant to be called async - e.g., it sets & clears thread context. //
/////////////////////////////////////////////////////////////////////////////////////////////////////
pollingAutomationPerTableRunner.processTableInsertOrUpdate(qInstance.getTable(tableAction.tableName()), QContext.getQSession(), tableAction.status(), tableAction.actions());
pollingAutomationPerTableRunner.processTableInsertOrUpdate(qInstance.getTable(tableAction.tableName()), QContext.getQSession(), tableAction.status());
}
}