From d54010e89d7c1d55c1a259f2f378e8ead2268b92 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Mon, 19 Feb 2024 10:27:24 -0600 Subject: [PATCH] Add order-by to the query used for running automations; updated logs. --- .../PollingAutomationPerTableRunner.java | 53 +++++++++++++-- .../PollingAutomationPerTableRunnerTest.java | 67 +++++++++++++++++++ 2 files changed, 113 insertions(+), 7 deletions(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/automation/polling/PollingAutomationPerTableRunner.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/automation/polling/PollingAutomationPerTableRunner.java index 88602fc4..b6ba394f 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/automation/polling/PollingAutomationPerTableRunner.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/automation/polling/PollingAutomationPerTableRunner.java @@ -28,6 +28,7 @@ import java.util.Comparator; import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.Optional; import java.util.function.Supplier; import java.util.stream.Collectors; import com.kingsrook.qqq.backend.core.actions.async.AsyncRecordPipeLoop; @@ -60,6 +61,8 @@ import com.kingsrook.qqq.backend.core.model.automation.TableTrigger; import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; +import com.kingsrook.qqq.backend.core.model.metadata.fields.DynamicDefaultValueBehavior; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.AutomationStatusTrackingType; import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.QTableAutomationDetails; @@ -257,7 +260,7 @@ public class PollingAutomationPerTableRunner implements Runnable } catch(Exception e) { - LOG.warn("Error running automations", e); + LOG.warn("Error running automations", e, logPair("tableName", tableActions.tableName()), logPair("status", tableActions.status())); } finally { @@ -301,7 +304,9 @@ public class PollingAutomationPerTableRunner implements Runnable AutomationStatusTrackingType statusTrackingType = automationDetails.getStatusTracking().getType(); if(AutomationStatusTrackingType.FIELD_IN_TABLE.equals(statusTrackingType)) { - queryInput.setFilter(new QQueryFilter().withCriteria(new QFilterCriteria(automationDetails.getStatusTracking().getFieldName(), QCriteriaOperator.EQUALS, List.of(automationStatus.getId())))); + QQueryFilter filter = new QQueryFilter().withCriteria(new QFilterCriteria(automationDetails.getStatusTracking().getFieldName(), QCriteriaOperator.EQUALS, List.of(automationStatus.getId()))); + addOrderByToQueryFilter(table, automationStatus, filter); + queryInput.setFilter(filter); } else { @@ -330,6 +335,38 @@ public class PollingAutomationPerTableRunner implements Runnable + /******************************************************************************* + ** + *******************************************************************************/ + static void addOrderByToQueryFilter(QTableMetaData table, AutomationStatus automationStatus, QQueryFilter filter) + { + //////////////////////////////////////////////////////////////////////////////////// + // look for a field in the table with either create-date or modify-date behavior, // + // based on if doing insert or update automations // + //////////////////////////////////////////////////////////////////////////////////// + DynamicDefaultValueBehavior dynamicDefaultValueBehavior = automationStatus.equals(AutomationStatus.PENDING_INSERT_AUTOMATIONS) ? DynamicDefaultValueBehavior.CREATE_DATE : DynamicDefaultValueBehavior.MODIFY_DATE; + Optional field = table.getFields().values().stream() + .filter(f -> dynamicDefaultValueBehavior.equals(f.getBehaviorOrDefault(QContext.getQInstance(), DynamicDefaultValueBehavior.class))) + .findFirst(); + + if(field.isPresent()) + { + ////////////////////////////////////////////////////////////////////// + // if a create/modify date field was found, order by it (ascending) // + ////////////////////////////////////////////////////////////////////// + filter.addOrderBy(new QFilterOrderBy(field.get().getName())); + } + else + { + //////////////////////////////////// + // else, order by the primary key // + //////////////////////////////////// + filter.addOrderBy(new QFilterOrderBy(table.getPrimaryKeyField())); + } + } + + + /******************************************************************************* ** get the actions to run against a table in an automation status. both from ** metaData and tableTriggers/data. @@ -458,13 +495,15 @@ public class PollingAutomationPerTableRunner implements Runnable //////////////////////////////////////// // update status on all these records // //////////////////////////////////////// - if(anyActionsFailed) + AutomationStatus statusToUpdateTo = anyActionsFailed ? pendingToFailedStatusMap.get(automationStatus) : AutomationStatus.OK; + try { - RecordAutomationStatusUpdater.setAutomationStatusInRecordsAndUpdate(instance, session, table, records, pendingToFailedStatusMap.get(automationStatus)); + RecordAutomationStatusUpdater.setAutomationStatusInRecordsAndUpdate(instance, session, table, records, statusToUpdateTo); } - else + catch(Exception e) { - RecordAutomationStatusUpdater.setAutomationStatusInRecordsAndUpdate(instance, session, table, records, AutomationStatus.OK); + LOG.warn("Error updating automationStatus after running automations", logPair("tableName", table), logPair("count", records.size()), logPair("status", statusToUpdateTo)); + throw (e); } } @@ -494,7 +533,7 @@ public class PollingAutomationPerTableRunner implements Runnable } catch(Exception e) { - LOG.warn("Caught exception processing records on " + table + " for action " + action, e); + LOG.warn("Caught exception processing automations", e, logPair("tableName", table), logPair("action", action.getName())); return (true); } } diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/automation/polling/PollingAutomationPerTableRunnerTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/automation/polling/PollingAutomationPerTableRunnerTest.java index 6ce3868c..f1dc131a 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/automation/polling/PollingAutomationPerTableRunnerTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/automation/polling/PollingAutomationPerTableRunnerTest.java @@ -46,6 +46,7 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput; import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput; import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.fields.DynamicDefaultValueBehavior; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; @@ -593,4 +594,70 @@ class PollingAutomationPerTableRunnerTest extends BaseTest new PollingAutomationPerTableRunner.ShardedTableActions(null, null, null, null, null).noopToFakeTestCoverage(); } + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testAddOrderByToQueryFilter() + { + ////////////////////////////////////////////////////////////////////////// + // make a table we'll test with. just put a primary-key id on it first // + ////////////////////////////////////////////////////////////////////////// + QTableMetaData table = new QTableMetaData() + .withPrimaryKeyField("id") + .withField(new QFieldMetaData("id", QFieldType.INTEGER)); + { + QQueryFilter filter = new QQueryFilter(); + PollingAutomationPerTableRunner.addOrderByToQueryFilter(table, AutomationStatus.PENDING_INSERT_AUTOMATIONS, filter); + assertEquals("id", filter.getOrderBys().get(0).getFieldName()); + } + + { + QQueryFilter filter = new QQueryFilter(); + PollingAutomationPerTableRunner.addOrderByToQueryFilter(table, AutomationStatus.PENDING_UPDATE_AUTOMATIONS, filter); + assertEquals("id", filter.getOrderBys().get(0).getFieldName()); + } + + //////////////////////////////////////////////////////////////////////////////// + // add createDate & modifyDate fields, but not with dynamic-default-behaviors // + // so should still sort by id // + //////////////////////////////////////////////////////////////////////////////// + QFieldMetaData createDate = new QFieldMetaData("createDate", QFieldType.DATE_TIME); + QFieldMetaData modifyDate = new QFieldMetaData("modifyDate", QFieldType.DATE_TIME); + table.addField(createDate); + table.addField(modifyDate); + { + QQueryFilter filter = new QQueryFilter(); + PollingAutomationPerTableRunner.addOrderByToQueryFilter(table, AutomationStatus.PENDING_INSERT_AUTOMATIONS, filter); + assertEquals("id", filter.getOrderBys().get(0).getFieldName()); + } + + { + QQueryFilter filter = new QQueryFilter(); + PollingAutomationPerTableRunner.addOrderByToQueryFilter(table, AutomationStatus.PENDING_UPDATE_AUTOMATIONS, filter); + assertEquals("id", filter.getOrderBys().get(0).getFieldName()); + } + + ///////////////////////////////////////////////////////////////////////////////////// + // add dynamic default value behaviors, confirm create/modify date fields are used // + ///////////////////////////////////////////////////////////////////////////////////// + createDate.withBehavior(DynamicDefaultValueBehavior.CREATE_DATE); + modifyDate.withBehavior(DynamicDefaultValueBehavior.MODIFY_DATE); + + { + QQueryFilter filter = new QQueryFilter(); + PollingAutomationPerTableRunner.addOrderByToQueryFilter(table, AutomationStatus.PENDING_INSERT_AUTOMATIONS, filter); + assertEquals("createDate", filter.getOrderBys().get(0).getFieldName()); + } + + { + QQueryFilter filter = new QQueryFilter(); + PollingAutomationPerTableRunner.addOrderByToQueryFilter(table, AutomationStatus.PENDING_UPDATE_AUTOMATIONS, filter); + assertEquals("modifyDate", filter.getOrderBys().get(0).getFieldName()); + } + + } + } \ No newline at end of file