diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/automation/RecordAutomationStatusUpdater.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/automation/RecordAutomationStatusUpdater.java index beb22b30..ce942c25 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/automation/RecordAutomationStatusUpdater.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/automation/RecordAutomationStatusUpdater.java @@ -22,9 +22,8 @@ package com.kingsrook.qqq.backend.core.actions.automation; -import java.util.ArrayList; +import java.util.Collections; import java.util.List; -import java.util.Objects; import com.kingsrook.qqq.backend.core.actions.tables.CountAction; import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction; import com.kingsrook.qqq.backend.core.context.QContext; @@ -90,6 +89,10 @@ public class RecordAutomationStatusUpdater } } + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // Avoid setting records to PENDING_INSERT or PENDING_UPDATE even if they don't have any insert or update automations or triggers // + // such records should go straight to OK status. // + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// if(canWeSkipPendingAndGoToOkay(table, automationStatus)) { automationStatus = AutomationStatus.OK; @@ -121,9 +124,13 @@ public class RecordAutomationStatusUpdater ** being asked to set status to PENDING_INSERT (or PENDING_UPDATE), then just ** move the status straight to OK. *******************************************************************************/ - private static boolean canWeSkipPendingAndGoToOkay(QTableMetaData table, AutomationStatus automationStatus) + static boolean canWeSkipPendingAndGoToOkay(QTableMetaData table, AutomationStatus automationStatus) { - List tableActions = Objects.requireNonNullElse(table.getAutomationDetails().getActions(), new ArrayList<>()); + List tableActions = Collections.emptyList(); + if(table.getAutomationDetails() != null && table.getAutomationDetails().getActions() != null) + { + tableActions = table.getAutomationDetails().getActions(); + } if(automationStatus.equals(AutomationStatus.PENDING_INSERT_AUTOMATIONS)) { @@ -135,6 +142,12 @@ public class RecordAutomationStatusUpdater { return (false); } + + //////////////////////////////////////////////////////////////////////////////////////// + // if we're going to pending-insert, and there are no insert automations or triggers, // + // then we may skip pending and go to okay. // + //////////////////////////////////////////////////////////////////////////////////////// + return (true); } else if(automationStatus.equals(AutomationStatus.PENDING_UPDATE_AUTOMATIONS)) { @@ -146,9 +159,21 @@ public class RecordAutomationStatusUpdater { return (false); } - } - return (true); + //////////////////////////////////////////////////////////////////////////////////////// + // if we're going to pending-update, and there are no insert automations or triggers, // + // then we may skip pending and go to okay. // + //////////////////////////////////////////////////////////////////////////////////////// + return (true); + } + else + { + /////////////////////////////////////////////////////////////////////////////////////////////////////// + // if we're going to any other automation status - then we may never "skip pending" and go to okay - // + // because we weren't asked to go to pending! // + /////////////////////////////////////////////////////////////////////////////////////////////////////// + return (false); + } } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/automation/polling/PollingAutomationPerTableRunner.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/automation/polling/PollingAutomationPerTableRunner.java index 52cc7646..8d37403e 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/automation/polling/PollingAutomationPerTableRunner.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/automation/polling/PollingAutomationPerTableRunner.java @@ -342,22 +342,9 @@ public class PollingAutomationPerTableRunner implements Runnable boolean anyActionsFailed = false; for(TableAutomationAction action : actions) { - try + boolean hadError = applyActionToRecords(table, records, action); + if(hadError) { - ////////////////////////////////////////////////////////////////////////////////////////////////////////////// - // note - this method - will re-query the objects, so we should have confidence that their data is fresh... // - ////////////////////////////////////////////////////////////////////////////////////////////////////////////// - List matchingQRecords = getRecordsMatchingActionFilter(table, records, action); - LOG.debug("Of the {} records that were pending automations, {} of them match the filter on the action {}", records.size(), matchingQRecords.size(), action); - if(CollectionUtils.nullSafeHasContents(matchingQRecords)) - { - LOG.debug(" Processing " + matchingQRecords.size() + " records in " + table + " for action " + action); - applyActionToMatchingRecords(table, matchingQRecords, action); - } - } - catch(Exception e) - { - LOG.warn("Caught exception processing records on " + table + " for action " + action, e); anyActionsFailed = true; } } @@ -377,6 +364,37 @@ public class PollingAutomationPerTableRunner implements Runnable + /******************************************************************************* + ** Run one action over a list of records (if they match the action's filter). + ** + ** @return hadError - true if an exception was caught; false if all OK. + *******************************************************************************/ + protected boolean applyActionToRecords(QTableMetaData table, List records, TableAutomationAction action) + { + try + { + ////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // note - this method - will re-query the objects, so we should have confidence that their data is fresh... // + ////////////////////////////////////////////////////////////////////////////////////////////////////////////// + List matchingQRecords = getRecordsMatchingActionFilter(table, records, action); + LOG.debug("Of the {} records that were pending automations, {} of them match the filter on the action {}", records.size(), matchingQRecords.size(), action); + if(CollectionUtils.nullSafeHasContents(matchingQRecords)) + { + LOG.debug(" Processing " + matchingQRecords.size() + " records in " + table + " for action " + action); + applyActionToMatchingRecords(table, matchingQRecords, action); + } + + return (false); + } + catch(Exception e) + { + LOG.warn("Caught exception processing records on " + table + " for action " + action, e); + return (true); + } + } + + + /******************************************************************************* ** For a given action, and a list of records - return a new list, of the ones ** which match the action's filter (if there is one - if not, then all match). diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/scripts/ScriptsMetaDataProvider.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/scripts/ScriptsMetaDataProvider.java index bd7f3231..b7282874 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/scripts/ScriptsMetaDataProvider.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/scripts/ScriptsMetaDataProvider.java @@ -383,7 +383,7 @@ public class ScriptsMetaDataProvider /******************************************************************************* ** *******************************************************************************/ - private QTableMetaData defineTableTriggerTable(String backendName) throws QException + public QTableMetaData defineTableTriggerTable(String backendName) throws QException { QTableMetaData tableMetaData = defineStandardTable(backendName, TableTrigger.TABLE_NAME, TableTrigger.class) .withRecordLabelFields("id") diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/automation/RecordAutomationStatusUpdaterTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/automation/RecordAutomationStatusUpdaterTest.java new file mode 100644 index 00000000..81a1a34f --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/automation/RecordAutomationStatusUpdaterTest.java @@ -0,0 +1,128 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2023. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.actions.automation; + + +import java.util.List; +import com.kingsrook.qqq.backend.core.BaseTest; +import com.kingsrook.qqq.backend.core.actions.tables.InsertAction; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; +import com.kingsrook.qqq.backend.core.model.automation.TableTrigger; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.QTableAutomationDetails; +import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.TableAutomationAction; +import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.TriggerEvent; +import com.kingsrook.qqq.backend.core.model.scripts.ScriptsMetaDataProvider; +import com.kingsrook.qqq.backend.core.utils.TestUtils; +import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + + +/******************************************************************************* + ** Unit test for RecordAutomationStatusUpdater + *******************************************************************************/ +class RecordAutomationStatusUpdaterTest extends BaseTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testCanWeSkipPendingAndGoToOkay() throws QException + { + QContext.getQInstance() + .addTable(new ScriptsMetaDataProvider().defineTableTriggerTable(TestUtils.MEMORY_BACKEND_NAME)); + + //////////////////////////////////////////////////////////// + // define tables with various automations and/or triggers // + //////////////////////////////////////////////////////////// + QTableMetaData tableWithNoAutomations = new QTableMetaData() + .withName("tableWithNoAutomations"); + + QTableMetaData tableWithInsertAutomation = new QTableMetaData() + .withName("tableWithInsertAutomation") + .withAutomationDetails(new QTableAutomationDetails() + .withAction(new TableAutomationAction().withTriggerEvent(TriggerEvent.POST_INSERT))); + + QTableMetaData tableWithUpdateAutomation = new QTableMetaData() + .withName("tableWithUpdateAutomation") + .withAutomationDetails(new QTableAutomationDetails() + .withAction(new TableAutomationAction() + .withTriggerEvent(TriggerEvent.POST_UPDATE))); + + QTableMetaData tableWithInsertAndUpdateAutomations = new QTableMetaData() + .withName("tableWithInsertAndUpdateAutomations ") + .withAutomationDetails(new QTableAutomationDetails() + .withAction(new TableAutomationAction().withTriggerEvent(TriggerEvent.POST_INSERT)) + .withAction(new TableAutomationAction().withTriggerEvent(TriggerEvent.POST_UPDATE))); + + QTableMetaData tableWithInsertTrigger = new QTableMetaData() + .withName("tableWithInsertTrigger"); + new InsertAction().execute(new InsertInput(TableTrigger.TABLE_NAME) + .withRecordEntity(new TableTrigger().withTableName(tableWithInsertTrigger.getName()).withPostInsert(true).withPostUpdate(false))); + + QTableMetaData tableWithUpdateTrigger = new QTableMetaData() + .withName("tableWithUpdateTrigger"); + new InsertAction().execute(new InsertInput(TableTrigger.TABLE_NAME) + .withRecordEntity(new TableTrigger().withTableName(tableWithUpdateTrigger.getName()).withPostInsert(false).withPostUpdate(true))); + + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // tests for going to PENDING_INSERT. // + // we should be allowed to skip and go to OK (return true) if the table does not have insert automations or triggers // + // we should NOT be allowed to skip and go to OK (return false) if the table does NOT have insert automations or triggers // + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + assertTrue(RecordAutomationStatusUpdater.canWeSkipPendingAndGoToOkay(tableWithNoAutomations, AutomationStatus.PENDING_INSERT_AUTOMATIONS)); + assertFalse(RecordAutomationStatusUpdater.canWeSkipPendingAndGoToOkay(tableWithInsertAutomation, AutomationStatus.PENDING_INSERT_AUTOMATIONS)); + assertTrue(RecordAutomationStatusUpdater.canWeSkipPendingAndGoToOkay(tableWithUpdateAutomation, AutomationStatus.PENDING_INSERT_AUTOMATIONS)); + assertFalse(RecordAutomationStatusUpdater.canWeSkipPendingAndGoToOkay(tableWithInsertAndUpdateAutomations, AutomationStatus.PENDING_INSERT_AUTOMATIONS)); + assertFalse(RecordAutomationStatusUpdater.canWeSkipPendingAndGoToOkay(tableWithInsertTrigger, AutomationStatus.PENDING_INSERT_AUTOMATIONS)); + assertTrue(RecordAutomationStatusUpdater.canWeSkipPendingAndGoToOkay(tableWithUpdateTrigger, AutomationStatus.PENDING_INSERT_AUTOMATIONS)); + + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // tests for going to PENDING_UPDATE. // + // we should be allowed to skip and go to OK (return true) if the table does not have update automations or triggers // + // we should NOT be allowed to skip and go to OK (return false) if the table does NOT have insert automations or triggers // + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + assertTrue(RecordAutomationStatusUpdater.canWeSkipPendingAndGoToOkay(tableWithNoAutomations, AutomationStatus.PENDING_UPDATE_AUTOMATIONS)); + assertTrue(RecordAutomationStatusUpdater.canWeSkipPendingAndGoToOkay(tableWithInsertAutomation, AutomationStatus.PENDING_UPDATE_AUTOMATIONS)); + assertFalse(RecordAutomationStatusUpdater.canWeSkipPendingAndGoToOkay(tableWithUpdateAutomation, AutomationStatus.PENDING_UPDATE_AUTOMATIONS)); + assertFalse(RecordAutomationStatusUpdater.canWeSkipPendingAndGoToOkay(tableWithInsertAndUpdateAutomations, AutomationStatus.PENDING_UPDATE_AUTOMATIONS)); + assertTrue(RecordAutomationStatusUpdater.canWeSkipPendingAndGoToOkay(tableWithInsertTrigger, AutomationStatus.PENDING_UPDATE_AUTOMATIONS)); + assertFalse(RecordAutomationStatusUpdater.canWeSkipPendingAndGoToOkay(tableWithUpdateTrigger, AutomationStatus.PENDING_UPDATE_AUTOMATIONS)); + + ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // tests for going to non-PENDING states // + // this function should NEVER return true for skipping pending if the target state (2nd arg) isn't a pending state. // + ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + for(AutomationStatus automationStatus : List.of(AutomationStatus.RUNNING_INSERT_AUTOMATIONS, AutomationStatus.RUNNING_UPDATE_AUTOMATIONS, AutomationStatus.FAILED_INSERT_AUTOMATIONS, AutomationStatus.FAILED_UPDATE_AUTOMATIONS, AutomationStatus.OK)) + { + for(QTableMetaData table : List.of(tableWithNoAutomations, tableWithInsertAutomation, tableWithUpdateAutomation, tableWithInsertAndUpdateAutomations, tableWithInsertTrigger, tableWithUpdateTrigger)) + { + assertFalse(RecordAutomationStatusUpdater.canWeSkipPendingAndGoToOkay(table, automationStatus), "Should never be okay to skip pending and go to OK (because we weren't going to pending). table=[" + table.getName() + "], status=[" + automationStatus + "]"); + } + } + } + +} \ No newline at end of file diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/automation/polling/PollingAutomationPerTableRunnerTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/automation/polling/PollingAutomationPerTableRunnerTest.java index a0cb010d..d4dcb1d5 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/automation/polling/PollingAutomationPerTableRunnerTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/automation/polling/PollingAutomationPerTableRunnerTest.java @@ -22,21 +22,27 @@ package com.kingsrook.qqq.backend.core.actions.automation.polling; +import java.io.Serializable; import java.time.LocalDate; import java.time.Month; import java.util.ArrayList; import java.util.List; import java.util.Optional; +import java.util.function.Supplier; import com.kingsrook.qqq.backend.core.BaseTest; import com.kingsrook.qqq.backend.core.actions.automation.AutomationStatus; import com.kingsrook.qqq.backend.core.actions.tables.InsertAction; +import com.kingsrook.qqq.backend.core.actions.tables.QueryAction; import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction; import com.kingsrook.qqq.backend.core.context.QContext; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertOutput; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput; import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput; import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; @@ -60,7 +66,9 @@ import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; /******************************************************************************* @@ -155,6 +163,40 @@ class PollingAutomationPerTableRunnerTest extends BaseTest + /******************************************************************************* + ** Test that if an automation has an error that we get error status + *******************************************************************************/ + @Test + void testAutomationWithError() throws QException + { + QInstance qInstance = QContext.getQInstance(); + + ///////////////////////////////////////////////////////////////////////////////////////////////////////////// + // insert 2 person records, both updated by the insert action, and 1 logged by logger-on-update automation // + ///////////////////////////////////////////////////////////////////////////////////////////////////////////// + InsertInput insertInput = new InsertInput(); + insertInput.setTableName(TestUtils.TABLE_NAME_PERSON_MEMORY); + insertInput.setRecords(List.of( + new QRecord().withValue("id", 1).withValue("firstName", "Han").withValue("lastName", "Solo").withValue("birthDate", LocalDate.parse("1977-05-25")), + new QRecord().withValue("id", 2).withValue("firstName", "Luke").withValue("lastName", "Skywalker").withValue("birthDate", LocalDate.parse("1977-05-25")), + new QRecord().withValue("id", 3).withValue("firstName", "Darth").withValue("lastName", "Vader").withValue("birthDate", LocalDate.parse("1977-05-25")) + )); + new InsertAction().execute(insertInput); + assertAllRecordsAutomationStatus(AutomationStatus.PENDING_INSERT_AUTOMATIONS); + + ///////////////////////// + // run the automations // + ///////////////////////// + runAllTableActions(qInstance); + + /////////////////////////////////////////////////////////////////////////////////////////////////////////// + // make sure all records are in status ERROR (even though only 1 threw, it breaks the page that it's in) // + /////////////////////////////////////////////////////////////////////////////////////////////////////////// + assertAllRecordsAutomationStatus(AutomationStatus.FAILED_INSERT_AUTOMATIONS); + } + + + /******************************************************************************* ** *******************************************************************************/ @@ -169,7 +211,6 @@ class PollingAutomationPerTableRunnerTest extends BaseTest // note - don't call run - it is meant to be called async - e.g., it sets & clears thread context. // ///////////////////////////////////////////////////////////////////////////////////////////////////// pollingAutomationPerTableRunner.processTableInsertOrUpdate(qInstance.getTable(tableAction.tableName()), QContext.getQSession(), tableAction.status()); - } } @@ -228,6 +269,77 @@ class PollingAutomationPerTableRunnerTest extends BaseTest + /******************************************************************************* + ** Test a large-ish number - to demonstrate paging working - and how it deals + ** with intermittent errors + ** + *******************************************************************************/ + @Test + void testMultiPagesWithSomeFailures() throws QException + { + QInstance qInstance = QContext.getQInstance(); + + ////////////////////////////////////////////////////////////////////////////////////////////////////// + // adjust table's automations batch size - as any exceptions thrown put the whole batch into error. // + // so we'll make batches (pages) of 100 - run for 500 records, and make just a couple bad records // + // that'll cause errors - so we should get a few failed pages, and the rest ok. // + ////////////////////////////////////////////////////////////////////////////////////////////////////// + int pageSize = 100; + qInstance.getTable(TestUtils.TABLE_NAME_PERSON_MEMORY) + .getAutomationDetails() + .setOverrideBatchSize(pageSize); + + ////////////////////////////////////////////////////////////////////////////////// + // insert many people - half who should be updated by the AgeChecker automation // + ////////////////////////////////////////////////////////////////////////////////// + InsertInput insertInput = new InsertInput(); + insertInput.setTableName(TestUtils.TABLE_NAME_PERSON_MEMORY); + + insertInput.setRecords(new ArrayList<>()); + int SIZE = 500; + for(int i = 0; i < SIZE; i++) + { + insertInput.getRecords().add(new QRecord().withValue("firstName", "Qui Gon").withValue("lastName", "Jinn " + i).withValue("birthDate", LocalDate.now())); + insertInput.getRecords().add(new QRecord().withValue("firstName", "Obi Wan").withValue("lastName", "Kenobi " + i)); + + ///////////////////////////////// + // throw 2 Darths into the mix // + ///////////////////////////////// + if(i == 101 || i == 301) + { + insertInput.getRecords().add(new QRecord().withValue("firstName", "Darth").withValue("lastName", "Maul " + i)); + } + } + + InsertOutput insertOutput = new InsertAction().execute(insertInput); + List insertedIds = insertOutput.getRecords().stream().map(r -> r.getValue("id")).toList(); + + assertAllRecordsAutomationStatus(AutomationStatus.PENDING_INSERT_AUTOMATIONS); + + ///////////////////////// + // run the automations // + ///////////////////////// + runAllTableActions(qInstance); + + //////////////////////////////////////////////////////////////////// + // make sure that some records became ok, but others became error // + //////////////////////////////////////////////////////////////////// + QueryOutput queryOutput = new QueryAction().execute(new QueryInput(TestUtils.TABLE_NAME_PERSON_MEMORY).withFilter(new QQueryFilter(new QFilterCriteria("id", QCriteriaOperator.IN, insertedIds)))); + List okRecords = queryOutput.getRecords().stream().filter(r -> AutomationStatus.OK.getId().equals(r.getValueInteger(TestUtils.standardQqqAutomationStatusField().getName()))).toList(); + List failedRecords = queryOutput.getRecords().stream().filter(r -> AutomationStatus.FAILED_INSERT_AUTOMATIONS.getId().equals(r.getValueInteger(TestUtils.standardQqqAutomationStatusField().getName()))).toList(); + + assertFalse(okRecords.isEmpty(), "Some inserted records should be automation status OK"); + assertFalse(failedRecords.isEmpty(), "Some inserted records should be automation status Failed"); + assertEquals(insertedIds.size(), okRecords.size() + failedRecords.size(), "All inserted records should be OK or Failed"); + + /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // make sure that only 2 pages failed - meaning our number of failedRecords is < pageSize * 2 (as any page may be smaller than the pageSize) // + /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + assertThat(failedRecords.size()).isLessThanOrEqualTo(pageSize * 2).describedAs("No more than 2 pages should be in failed status."); + } + + + /******************************************************************************* ** Test running a process for automation, instead of a code ref. *******************************************************************************/ @@ -367,6 +479,61 @@ class PollingAutomationPerTableRunnerTest extends BaseTest + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testServerShutdownMidRunLeavesRecordsInRunningStatus() throws QException + { + QInstance qInstance = QContext.getQInstance(); + + ///////////////////////////////////////////////////////////////////////////// + // insert 2 person records that should have insert action ran against them // + ///////////////////////////////////////////////////////////////////////////// + InsertInput insertInput = new InsertInput(); + insertInput.setTableName(TestUtils.TABLE_NAME_PERSON_MEMORY); + insertInput.setRecords(List.of( + new QRecord().withValue("id", 1).withValue("firstName", "Tim").withValue("birthDate", LocalDate.now()), + new QRecord().withValue("id", 2).withValue("firstName", "Darin").withValue("birthDate", LocalDate.now()) + )); + new InsertAction().execute(insertInput); + assertAllRecordsAutomationStatus(AutomationStatus.PENDING_INSERT_AUTOMATIONS); + + ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // duplicate the runAllTableActions method - but using the subclass of PollingAutomationPerTableRunner that will throw. // + ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + assertThatThrownBy(() -> + { + List tableActions = PollingAutomationPerTableRunner.getTableActions(qInstance, TestUtils.POLLING_AUTOMATION); + for(PollingAutomationPerTableRunner.TableActions tableAction : tableActions) + { + PollingAutomationPerTableRunner pollingAutomationPerTableRunner = new PollingAutomationPerTableRunnerThatShouldSimulateServerShutdownMidRun(qInstance, TestUtils.POLLING_AUTOMATION, QSession::new, tableAction); + + ///////////////////////////////////////////////////////////////////////////////////////////////////// + // note - don't call run - it is meant to be called async - e.g., it sets & clears thread context. // + ///////////////////////////////////////////////////////////////////////////////////////////////////// + pollingAutomationPerTableRunner.processTableInsertOrUpdate(qInstance.getTable(tableAction.tableName()), QContext.getQSession(), tableAction.status()); + } + }).hasMessage(PollingAutomationPerTableRunnerThatShouldSimulateServerShutdownMidRun.EXCEPTION_MESSAGE); + + ////////////////////////////////////////////////// + // records should be "leaked" in running status // + ////////////////////////////////////////////////// + assertAllRecordsAutomationStatus(AutomationStatus.RUNNING_INSERT_AUTOMATIONS); + + ///////////////////////////////////// + // simulate another run of the job // + ///////////////////////////////////// + runAllTableActions(qInstance); + + //////////////////////////////////////////////////////////////////////////////// + // it should NOT have updated those records - they're officially "leaked" now // + //////////////////////////////////////////////////////////////////////////////// + assertAllRecordsAutomationStatus(AutomationStatus.RUNNING_INSERT_AUTOMATIONS); + } + + + /******************************************************************************* ** *******************************************************************************/ @@ -376,4 +543,42 @@ class PollingAutomationPerTableRunnerTest extends BaseTest .isNotEmpty() .allMatch(r -> pendingInsertAutomations.getId().equals(r.getValue(TestUtils.standardQqqAutomationStatusField().getName()))); } + + + + /******************************************************************************* + ** this subclass of the class under test allows us to simulate: + ** + ** what happens if, after records have been marked as running-updates, if, + ** for example, a server shuts down? + ** + ** It does this by overriding a method that runs between those points in time, + ** and throwing a runtime exception. + *******************************************************************************/ + public static class PollingAutomationPerTableRunnerThatShouldSimulateServerShutdownMidRun extends PollingAutomationPerTableRunner + { + private static String EXCEPTION_MESSAGE = "Throwing outside of catch here, to simulate a server shutdown mid-run"; + + + + /******************************************************************************* + ** + *******************************************************************************/ + public PollingAutomationPerTableRunnerThatShouldSimulateServerShutdownMidRun(QInstance instance, String providerName, Supplier sessionSupplier, TableActions tableActions) + { + super(instance, providerName, sessionSupplier, tableActions); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + protected boolean applyActionToRecords(QTableMetaData table, List records, TableAutomationAction action) + { + throw (new RuntimeException(EXCEPTION_MESSAGE)); + } + } + } \ No newline at end of file diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java index c019aae5..c9c89e2f 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java @@ -34,6 +34,7 @@ import com.kingsrook.qqq.backend.core.actions.processes.BackendStep; import com.kingsrook.qqq.backend.core.actions.processes.person.addtopeoplesage.AddAge; import com.kingsrook.qqq.backend.core.actions.processes.person.addtopeoplesage.GetAgeStatistics; import com.kingsrook.qqq.backend.core.actions.tables.CountAction; +import com.kingsrook.qqq.backend.core.actions.tables.GetAction; import com.kingsrook.qqq.backend.core.actions.tables.InsertAction; import com.kingsrook.qqq.backend.core.actions.tables.QueryAction; import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction; @@ -45,6 +46,7 @@ import com.kingsrook.qqq.backend.core.logging.QLogger; import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetInput; import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; @@ -813,6 +815,11 @@ public class TestUtils .withFilter(new QQueryFilter().withCriteria(new QFilterCriteria("birthDate", QCriteriaOperator.GREATER_THAN, List.of(youngPersonLimitDate)))) .withCodeReference(new QCodeReference(CheckAge.class)) ) + .withAction(new TableAutomationAction() + .withName("failAutomationForSith") + .withTriggerEvent(TriggerEvent.POST_INSERT) + .withCodeReference(new QCodeReference(FailAutomationForSith.class)) + ) .withAction(new TableAutomationAction() .withName("increaseBirthdate") .withTriggerEvent(TriggerEvent.POST_INSERT) @@ -918,6 +925,15 @@ public class TestUtils List recordsToUpdate = new ArrayList<>(); for(QRecord record : recordAutomationInput.getRecordList()) { + //////////////////////////////////////////////////////////////////////// + // get the record - its automation status should currently be RUNNING // + //////////////////////////////////////////////////////////////////////// + QRecord freshlyFetchedRecord = new GetAction().executeForRecord(new GetInput(TABLE_NAME_PERSON_MEMORY).withPrimaryKey(record.getValue("id"))); + assertEquals(AutomationStatus.RUNNING_INSERT_AUTOMATIONS.getId(), freshlyFetchedRecord.getValueInteger(TestUtils.standardQqqAutomationStatusField().getName())); + + /////////////////////////////////////////// + // do whatever business logic we do here // + /////////////////////////////////////////// LocalDate birthDate = record.getValueLocalDate("birthDate"); if(birthDate != null && birthDate.isAfter(limitDate)) { @@ -940,6 +956,29 @@ public class TestUtils + /******************************************************************************* + ** + *******************************************************************************/ + public static class FailAutomationForSith extends RecordAutomationHandler + { + + /******************************************************************************* + ** + *******************************************************************************/ + public void execute(RecordAutomationInput recordAutomationInput) throws QException + { + for(QRecord record : recordAutomationInput.getRecordList()) + { + if("Darth".equals(record.getValue("firstName"))) + { + throw new QException("Oops, you look like a Sith!"); + } + } + } + } + + + /******************************************************************************* ** *******************************************************************************/