Fix canWeSkipPendingAndGoToOkay to only ever return true if its input status is a Pending status.

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/automation/RecordAutomationStatusUpdater.java

@@ -22,9 +22,8 @@
 package com.kingsrook.qqq.backend.core.actions.automation;
 
 
-import java.util.ArrayList;
+import java.util.Collections;
 import java.util.List;
-import java.util.Objects;
 import com.kingsrook.qqq.backend.core.actions.tables.CountAction;
 import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction;
 import com.kingsrook.qqq.backend.core.context.QContext;
@@ -90,6 +89,10 @@ public class RecordAutomationStatusUpdater
          }
       }
 
+      ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      // Avoid setting records to PENDING_INSERT or PENDING_UPDATE even if they don't have any insert or update automations or triggers //
+      // such records should go straight to OK status.                                                                                  //
+      ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
       if(canWeSkipPendingAndGoToOkay(table, automationStatus))
       {
          automationStatus = AutomationStatus.OK;
@@ -121,9 +124,13 @@ public class RecordAutomationStatusUpdater
     ** being asked to set status to PENDING_INSERT (or PENDING_UPDATE), then just
     ** move the status straight to OK.
     *******************************************************************************/
-   private static boolean canWeSkipPendingAndGoToOkay(QTableMetaData table, AutomationStatus automationStatus)
+   static boolean canWeSkipPendingAndGoToOkay(QTableMetaData table, AutomationStatus automationStatus)
    {
-      List<TableAutomationAction> tableActions = Objects.requireNonNullElse(table.getAutomationDetails().getActions(), new ArrayList<>());
+      List<TableAutomationAction> tableActions = Collections.emptyList();
+      if(table.getAutomationDetails() != null && table.getAutomationDetails().getActions() != null)
+      {
+         tableActions = table.getAutomationDetails().getActions();
+      }
 
       if(automationStatus.equals(AutomationStatus.PENDING_INSERT_AUTOMATIONS))
       {
@@ -135,6 +142,12 @@ public class RecordAutomationStatusUpdater
          {
             return (false);
          }
+
+         ////////////////////////////////////////////////////////////////////////////////////////
+         // if we're going to pending-insert, and there are no insert automations or triggers, //
+         // then we may skip pending and go to okay.                                           //
+         ////////////////////////////////////////////////////////////////////////////////////////
+         return (true);
       }
       else if(automationStatus.equals(AutomationStatus.PENDING_UPDATE_AUTOMATIONS))
       {
@@ -146,10 +159,22 @@ public class RecordAutomationStatusUpdater
          {
             return (false);
          }
-      }
 
+         ////////////////////////////////////////////////////////////////////////////////////////
+         // if we're going to pending-update, and there are no insert automations or triggers, //
+         // then we may skip pending and go to okay.                                           //
+         ////////////////////////////////////////////////////////////////////////////////////////
          return (true);
       }
+      else
+      {
+         ///////////////////////////////////////////////////////////////////////////////////////////////////////
+         // if we're going to any other automation status - then we may never "skip pending" and go to okay - //
+         // because we weren't asked to go to pending!                                                        //
+         ///////////////////////////////////////////////////////////////////////////////////////////////////////
+         return (false);
+      }
+   }
 
 
 

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/automation/polling/PollingAutomationPerTableRunner.java

@@ -342,22 +342,9 @@ public class PollingAutomationPerTableRunner implements Runnable
       boolean anyActionsFailed = false;
       for(TableAutomationAction action : actions)
       {
-         try
+         boolean hadError = applyActionToRecords(table, records, action);
+         if(hadError)
          {
-            //////////////////////////////////////////////////////////////////////////////////////////////////////////////
-            // note - this method - will re-query the objects, so we should have confidence that their data is fresh... //
-            //////////////////////////////////////////////////////////////////////////////////////////////////////////////
-            List<QRecord> matchingQRecords = getRecordsMatchingActionFilter(table, records, action);
-            LOG.debug("Of the {} records that were pending automations, {} of them match the filter on the action {}", records.size(), matchingQRecords.size(), action);
-            if(CollectionUtils.nullSafeHasContents(matchingQRecords))
-            {
-               LOG.debug("  Processing " + matchingQRecords.size() + " records in " + table + " for action " + action);
-               applyActionToMatchingRecords(table, matchingQRecords, action);
-            }
-         }
-         catch(Exception e)
-         {
-            LOG.warn("Caught exception processing records on " + table + " for action " + action, e);
             anyActionsFailed = true;
          }
       }
@@ -377,6 +364,37 @@ public class PollingAutomationPerTableRunner implements Runnable
 
 
 
+   /*******************************************************************************
+    ** Run one action over a list of records (if they match the action's filter).
+    **
+    ** @return hadError - true if an exception was caught; false if all OK.
+    *******************************************************************************/
+   protected boolean applyActionToRecords(QTableMetaData table, List<QRecord> records, TableAutomationAction action)
+   {
+      try
+      {
+         //////////////////////////////////////////////////////////////////////////////////////////////////////////////
+         // note - this method - will re-query the objects, so we should have confidence that their data is fresh... //
+         //////////////////////////////////////////////////////////////////////////////////////////////////////////////
+         List<QRecord> matchingQRecords = getRecordsMatchingActionFilter(table, records, action);
+         LOG.debug("Of the {} records that were pending automations, {} of them match the filter on the action {}", records.size(), matchingQRecords.size(), action);
+         if(CollectionUtils.nullSafeHasContents(matchingQRecords))
+         {
+            LOG.debug("  Processing " + matchingQRecords.size() + " records in " + table + " for action " + action);
+            applyActionToMatchingRecords(table, matchingQRecords, action);
+         }
+
+         return (false);
+      }
+      catch(Exception e)
+      {
+         LOG.warn("Caught exception processing records on " + table + " for action " + action, e);
+         return (true);
+      }
+   }
+
+
+
    /*******************************************************************************
     ** For a given action, and a list of records - return a new list, of the ones
     ** which match the action's filter (if there is one - if not, then all match).

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/scripts/ScriptsMetaDataProvider.java

@@ -383,7 +383,7 @@ public class ScriptsMetaDataProvider
    /*******************************************************************************
     **
     *******************************************************************************/
-   private QTableMetaData defineTableTriggerTable(String backendName) throws QException
+   public QTableMetaData defineTableTriggerTable(String backendName) throws QException
    {
       QTableMetaData tableMetaData = defineStandardTable(backendName, TableTrigger.TABLE_NAME, TableTrigger.class)
          .withRecordLabelFields("id")

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/automation/RecordAutomationStatusUpdaterTest.java

@@ -0,0 +1,128 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2023.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.actions.automation;
+
+
+import java.util.List;
+import com.kingsrook.qqq.backend.core.BaseTest;
+import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
+import com.kingsrook.qqq.backend.core.context.QContext;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
+import com.kingsrook.qqq.backend.core.model.automation.TableTrigger;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.QTableAutomationDetails;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.TableAutomationAction;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.TriggerEvent;
+import com.kingsrook.qqq.backend.core.model.scripts.ScriptsMetaDataProvider;
+import com.kingsrook.qqq.backend.core.utils.TestUtils;
+import org.junit.jupiter.api.Test;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+
+/*******************************************************************************
+ ** Unit test for RecordAutomationStatusUpdater 
+ *******************************************************************************/
+class RecordAutomationStatusUpdaterTest extends BaseTest
+{
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testCanWeSkipPendingAndGoToOkay() throws QException
+   {
+      QContext.getQInstance()
+         .addTable(new ScriptsMetaDataProvider().defineTableTriggerTable(TestUtils.MEMORY_BACKEND_NAME));
+
+      ////////////////////////////////////////////////////////////
+      // define tables with various automations and/or triggers //
+      ////////////////////////////////////////////////////////////
+      QTableMetaData tableWithNoAutomations = new QTableMetaData()
+         .withName("tableWithNoAutomations");
+
+      QTableMetaData tableWithInsertAutomation = new QTableMetaData()
+         .withName("tableWithInsertAutomation")
+         .withAutomationDetails(new QTableAutomationDetails()
+            .withAction(new TableAutomationAction().withTriggerEvent(TriggerEvent.POST_INSERT)));
+
+      QTableMetaData tableWithUpdateAutomation = new QTableMetaData()
+         .withName("tableWithUpdateAutomation")
+         .withAutomationDetails(new QTableAutomationDetails()
+            .withAction(new TableAutomationAction()
+               .withTriggerEvent(TriggerEvent.POST_UPDATE)));
+
+      QTableMetaData tableWithInsertAndUpdateAutomations = new QTableMetaData()
+         .withName("tableWithInsertAndUpdateAutomations ")
+         .withAutomationDetails(new QTableAutomationDetails()
+            .withAction(new TableAutomationAction().withTriggerEvent(TriggerEvent.POST_INSERT))
+            .withAction(new TableAutomationAction().withTriggerEvent(TriggerEvent.POST_UPDATE)));
+
+      QTableMetaData tableWithInsertTrigger = new QTableMetaData()
+         .withName("tableWithInsertTrigger");
+      new InsertAction().execute(new InsertInput(TableTrigger.TABLE_NAME)
+         .withRecordEntity(new TableTrigger().withTableName(tableWithInsertTrigger.getName()).withPostInsert(true).withPostUpdate(false)));
+
+      QTableMetaData tableWithUpdateTrigger = new QTableMetaData()
+         .withName("tableWithUpdateTrigger");
+      new InsertAction().execute(new InsertInput(TableTrigger.TABLE_NAME)
+         .withRecordEntity(new TableTrigger().withTableName(tableWithUpdateTrigger.getName()).withPostInsert(false).withPostUpdate(true)));
+
+      ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      // tests for going to PENDING_INSERT.                                                                                     //
+      // we should be allowed to skip and go to OK (return true) if the table does not have insert automations or triggers      //
+      // we should NOT be allowed to skip and go to OK (return false) if the table does NOT have insert automations or triggers //
+      ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      assertTrue(RecordAutomationStatusUpdater.canWeSkipPendingAndGoToOkay(tableWithNoAutomations, AutomationStatus.PENDING_INSERT_AUTOMATIONS));
+      assertFalse(RecordAutomationStatusUpdater.canWeSkipPendingAndGoToOkay(tableWithInsertAutomation, AutomationStatus.PENDING_INSERT_AUTOMATIONS));
+      assertTrue(RecordAutomationStatusUpdater.canWeSkipPendingAndGoToOkay(tableWithUpdateAutomation, AutomationStatus.PENDING_INSERT_AUTOMATIONS));
+      assertFalse(RecordAutomationStatusUpdater.canWeSkipPendingAndGoToOkay(tableWithInsertAndUpdateAutomations, AutomationStatus.PENDING_INSERT_AUTOMATIONS));
+      assertFalse(RecordAutomationStatusUpdater.canWeSkipPendingAndGoToOkay(tableWithInsertTrigger, AutomationStatus.PENDING_INSERT_AUTOMATIONS));
+      assertTrue(RecordAutomationStatusUpdater.canWeSkipPendingAndGoToOkay(tableWithUpdateTrigger, AutomationStatus.PENDING_INSERT_AUTOMATIONS));
+
+      ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      // tests for going to PENDING_UPDATE.                                                                                     //
+      // we should be allowed to skip and go to OK (return true) if the table does not have update automations or triggers      //
+      // we should NOT be allowed to skip and go to OK (return false) if the table does NOT have insert automations or triggers //
+      ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      assertTrue(RecordAutomationStatusUpdater.canWeSkipPendingAndGoToOkay(tableWithNoAutomations, AutomationStatus.PENDING_UPDATE_AUTOMATIONS));
+      assertTrue(RecordAutomationStatusUpdater.canWeSkipPendingAndGoToOkay(tableWithInsertAutomation, AutomationStatus.PENDING_UPDATE_AUTOMATIONS));
+      assertFalse(RecordAutomationStatusUpdater.canWeSkipPendingAndGoToOkay(tableWithUpdateAutomation, AutomationStatus.PENDING_UPDATE_AUTOMATIONS));
+      assertFalse(RecordAutomationStatusUpdater.canWeSkipPendingAndGoToOkay(tableWithInsertAndUpdateAutomations, AutomationStatus.PENDING_UPDATE_AUTOMATIONS));
+      assertTrue(RecordAutomationStatusUpdater.canWeSkipPendingAndGoToOkay(tableWithInsertTrigger, AutomationStatus.PENDING_UPDATE_AUTOMATIONS));
+      assertFalse(RecordAutomationStatusUpdater.canWeSkipPendingAndGoToOkay(tableWithUpdateTrigger, AutomationStatus.PENDING_UPDATE_AUTOMATIONS));
+
+      //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      // tests for going to non-PENDING states                                                                            //
+      // this function should NEVER return true for skipping pending if the target state (2nd arg) isn't a pending state. //
+      //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      for(AutomationStatus automationStatus : List.of(AutomationStatus.RUNNING_INSERT_AUTOMATIONS, AutomationStatus.RUNNING_UPDATE_AUTOMATIONS, AutomationStatus.FAILED_INSERT_AUTOMATIONS, AutomationStatus.FAILED_UPDATE_AUTOMATIONS, AutomationStatus.OK))
+      {
+         for(QTableMetaData table : List.of(tableWithNoAutomations, tableWithInsertAutomation, tableWithUpdateAutomation, tableWithInsertAndUpdateAutomations, tableWithInsertTrigger, tableWithUpdateTrigger))
+         {
+            assertFalse(RecordAutomationStatusUpdater.canWeSkipPendingAndGoToOkay(table, automationStatus), "Should never be okay to skip pending and go to OK (because we weren't going to pending).  table=[" + table.getName() + "], status=[" + automationStatus + "]");
+         }
+      }
+   }
+
+}

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/automation/polling/PollingAutomationPerTableRunnerTest.java

@@ -22,21 +22,27 @@
 package com.kingsrook.qqq.backend.core.actions.automation.polling;
 
 
+import java.io.Serializable;
 import java.time.LocalDate;
 import java.time.Month;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Optional;
+import java.util.function.Supplier;
 import com.kingsrook.qqq.backend.core.BaseTest;
 import com.kingsrook.qqq.backend.core.actions.automation.AutomationStatus;
 import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
+import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
 import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction;
 import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertOutput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
 import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
@@ -60,7 +66,9 @@ import org.junit.jupiter.api.AfterEach;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
 import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
 import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
 
 
 /*******************************************************************************
@@ -155,6 +163,40 @@ class PollingAutomationPerTableRunnerTest extends BaseTest
 
 
 
+   /*******************************************************************************
+    ** Test that if an automation has an error that we get error status
+    *******************************************************************************/
+   @Test
+   void testAutomationWithError() throws QException
+   {
+      QInstance qInstance = QContext.getQInstance();
+
+      /////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      // insert 2 person records, both updated by the insert action, and 1 logged by logger-on-update automation //
+      /////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      InsertInput insertInput = new InsertInput();
+      insertInput.setTableName(TestUtils.TABLE_NAME_PERSON_MEMORY);
+      insertInput.setRecords(List.of(
+         new QRecord().withValue("id", 1).withValue("firstName", "Han").withValue("lastName", "Solo").withValue("birthDate", LocalDate.parse("1977-05-25")),
+         new QRecord().withValue("id", 2).withValue("firstName", "Luke").withValue("lastName", "Skywalker").withValue("birthDate", LocalDate.parse("1977-05-25")),
+         new QRecord().withValue("id", 3).withValue("firstName", "Darth").withValue("lastName", "Vader").withValue("birthDate", LocalDate.parse("1977-05-25"))
+      ));
+      new InsertAction().execute(insertInput);
+      assertAllRecordsAutomationStatus(AutomationStatus.PENDING_INSERT_AUTOMATIONS);
+
+      /////////////////////////
+      // run the automations //
+      /////////////////////////
+      runAllTableActions(qInstance);
+
+      ///////////////////////////////////////////////////////////////////////////////////////////////////////////
+      // make sure all records are in status ERROR (even though only 1 threw, it breaks the page that it's in) //
+      ///////////////////////////////////////////////////////////////////////////////////////////////////////////
+      assertAllRecordsAutomationStatus(AutomationStatus.FAILED_INSERT_AUTOMATIONS);
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/
@@ -169,7 +211,6 @@ class PollingAutomationPerTableRunnerTest extends BaseTest
          // note - don't call run - it is meant to be called async - e.g., it sets & clears thread context. //
          /////////////////////////////////////////////////////////////////////////////////////////////////////
          pollingAutomationPerTableRunner.processTableInsertOrUpdate(qInstance.getTable(tableAction.tableName()), QContext.getQSession(), tableAction.status());
-
       }
    }
 
@@ -228,6 +269,77 @@ class PollingAutomationPerTableRunnerTest extends BaseTest
 
 
 
+   /*******************************************************************************
+    ** Test a large-ish number - to demonstrate paging working - and how it deals
+    ** with intermittent errors
+    **
+    *******************************************************************************/
+   @Test
+   void testMultiPagesWithSomeFailures() throws QException
+   {
+      QInstance qInstance = QContext.getQInstance();
+
+      //////////////////////////////////////////////////////////////////////////////////////////////////////
+      // adjust table's automations batch size - as any exceptions thrown put the whole batch into error. //
+      // so we'll make batches (pages) of 100 - run for 500 records, and make just a couple bad records   //
+      // that'll cause errors - so we should get a few failed pages, and the rest ok.                     //
+      //////////////////////////////////////////////////////////////////////////////////////////////////////
+      int pageSize = 100;
+      qInstance.getTable(TestUtils.TABLE_NAME_PERSON_MEMORY)
+         .getAutomationDetails()
+         .setOverrideBatchSize(pageSize);
+
+      //////////////////////////////////////////////////////////////////////////////////
+      // insert many people - half who should be updated by the AgeChecker automation //
+      //////////////////////////////////////////////////////////////////////////////////
+      InsertInput insertInput = new InsertInput();
+      insertInput.setTableName(TestUtils.TABLE_NAME_PERSON_MEMORY);
+
+      insertInput.setRecords(new ArrayList<>());
+      int SIZE = 500;
+      for(int i = 0; i < SIZE; i++)
+      {
+         insertInput.getRecords().add(new QRecord().withValue("firstName", "Qui Gon").withValue("lastName", "Jinn " + i).withValue("birthDate", LocalDate.now()));
+         insertInput.getRecords().add(new QRecord().withValue("firstName", "Obi Wan").withValue("lastName", "Kenobi " + i));
+
+         /////////////////////////////////
+         // throw 2 Darths into the mix //
+         /////////////////////////////////
+         if(i == 101 || i == 301)
+         {
+            insertInput.getRecords().add(new QRecord().withValue("firstName", "Darth").withValue("lastName", "Maul " + i));
+         }
+      }
+
+      InsertOutput       insertOutput = new InsertAction().execute(insertInput);
+      List<Serializable> insertedIds  = insertOutput.getRecords().stream().map(r -> r.getValue("id")).toList();
+
+      assertAllRecordsAutomationStatus(AutomationStatus.PENDING_INSERT_AUTOMATIONS);
+
+      /////////////////////////
+      // run the automations //
+      /////////////////////////
+      runAllTableActions(qInstance);
+
+      ////////////////////////////////////////////////////////////////////
+      // make sure that some records became ok, but others became error //
+      ////////////////////////////////////////////////////////////////////
+      QueryOutput   queryOutput   = new QueryAction().execute(new QueryInput(TestUtils.TABLE_NAME_PERSON_MEMORY).withFilter(new QQueryFilter(new QFilterCriteria("id", QCriteriaOperator.IN, insertedIds))));
+      List<QRecord> okRecords     = queryOutput.getRecords().stream().filter(r -> AutomationStatus.OK.getId().equals(r.getValueInteger(TestUtils.standardQqqAutomationStatusField().getName()))).toList();
+      List<QRecord> failedRecords = queryOutput.getRecords().stream().filter(r -> AutomationStatus.FAILED_INSERT_AUTOMATIONS.getId().equals(r.getValueInteger(TestUtils.standardQqqAutomationStatusField().getName()))).toList();
+
+      assertFalse(okRecords.isEmpty(), "Some inserted records should be automation status OK");
+      assertFalse(failedRecords.isEmpty(), "Some inserted records should be automation status Failed");
+      assertEquals(insertedIds.size(), okRecords.size() + failedRecords.size(), "All inserted records should be OK or Failed");
+
+      ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      // make sure that only 2 pages failed - meaning our number of failedRecords is < pageSize * 2 (as any page may be smaller than the pageSize) //
+      ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      assertThat(failedRecords.size()).isLessThanOrEqualTo(pageSize * 2).describedAs("No more than 2 pages should be in failed status.");
+   }
+
+
+
    /*******************************************************************************
     ** Test running a process for automation, instead of a code ref.
     *******************************************************************************/
@@ -367,6 +479,61 @@ class PollingAutomationPerTableRunnerTest extends BaseTest
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testServerShutdownMidRunLeavesRecordsInRunningStatus() throws QException
+   {
+      QInstance qInstance = QContext.getQInstance();
+
+      /////////////////////////////////////////////////////////////////////////////
+      // insert 2 person records that should have insert action ran against them //
+      /////////////////////////////////////////////////////////////////////////////
+      InsertInput insertInput = new InsertInput();
+      insertInput.setTableName(TestUtils.TABLE_NAME_PERSON_MEMORY);
+      insertInput.setRecords(List.of(
+         new QRecord().withValue("id", 1).withValue("firstName", "Tim").withValue("birthDate", LocalDate.now()),
+         new QRecord().withValue("id", 2).withValue("firstName", "Darin").withValue("birthDate", LocalDate.now())
+      ));
+      new InsertAction().execute(insertInput);
+      assertAllRecordsAutomationStatus(AutomationStatus.PENDING_INSERT_AUTOMATIONS);
+
+      //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      // duplicate the runAllTableActions method - but using the subclass of PollingAutomationPerTableRunner that will throw. //
+      //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      assertThatThrownBy(() ->
+      {
+         List<PollingAutomationPerTableRunner.TableActions> tableActions = PollingAutomationPerTableRunner.getTableActions(qInstance, TestUtils.POLLING_AUTOMATION);
+         for(PollingAutomationPerTableRunner.TableActions tableAction : tableActions)
+         {
+            PollingAutomationPerTableRunner pollingAutomationPerTableRunner = new PollingAutomationPerTableRunnerThatShouldSimulateServerShutdownMidRun(qInstance, TestUtils.POLLING_AUTOMATION, QSession::new, tableAction);
+
+            /////////////////////////////////////////////////////////////////////////////////////////////////////
+            // note - don't call run - it is meant to be called async - e.g., it sets & clears thread context. //
+            /////////////////////////////////////////////////////////////////////////////////////////////////////
+            pollingAutomationPerTableRunner.processTableInsertOrUpdate(qInstance.getTable(tableAction.tableName()), QContext.getQSession(), tableAction.status());
+         }
+      }).hasMessage(PollingAutomationPerTableRunnerThatShouldSimulateServerShutdownMidRun.EXCEPTION_MESSAGE);
+
+      //////////////////////////////////////////////////
+      // records should be "leaked" in running status //
+      //////////////////////////////////////////////////
+      assertAllRecordsAutomationStatus(AutomationStatus.RUNNING_INSERT_AUTOMATIONS);
+
+      /////////////////////////////////////
+      // simulate another run of the job //
+      /////////////////////////////////////
+      runAllTableActions(qInstance);
+
+      ////////////////////////////////////////////////////////////////////////////////
+      // it should NOT have updated those records - they're officially "leaked" now //
+      ////////////////////////////////////////////////////////////////////////////////
+      assertAllRecordsAutomationStatus(AutomationStatus.RUNNING_INSERT_AUTOMATIONS);
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/
@@ -376,4 +543,42 @@ class PollingAutomationPerTableRunnerTest extends BaseTest
          .isNotEmpty()
          .allMatch(r -> pendingInsertAutomations.getId().equals(r.getValue(TestUtils.standardQqqAutomationStatusField().getName())));
    }
+
+
+
+   /*******************************************************************************
+    ** this subclass of the class under test allows us to simulate:
+    **
+    ** what happens if, after records have been marked as running-updates, if,
+    ** for example, a server shuts down?
+    **
+    ** It does this by overriding a method that runs between those points in time,
+    ** and throwing a runtime exception.
+    *******************************************************************************/
+   public static class PollingAutomationPerTableRunnerThatShouldSimulateServerShutdownMidRun extends PollingAutomationPerTableRunner
+   {
+      private static String EXCEPTION_MESSAGE = "Throwing outside of catch here, to simulate a server shutdown mid-run";
+
+
+
+      /*******************************************************************************
+       **
+       *******************************************************************************/
+      public PollingAutomationPerTableRunnerThatShouldSimulateServerShutdownMidRun(QInstance instance, String providerName, Supplier<QSession> sessionSupplier, TableActions tableActions)
+      {
+         super(instance, providerName, sessionSupplier, tableActions);
+      }
+
+
+
+      /*******************************************************************************
+       **
+       *******************************************************************************/
+      @Override
+      protected boolean applyActionToRecords(QTableMetaData table, List<QRecord> records, TableAutomationAction action)
+      {
+         throw (new RuntimeException(EXCEPTION_MESSAGE));
+      }
+   }
+
 }

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java

@@ -34,6 +34,7 @@ import com.kingsrook.qqq.backend.core.actions.processes.BackendStep;
 import com.kingsrook.qqq.backend.core.actions.processes.person.addtopeoplesage.AddAge;
 import com.kingsrook.qqq.backend.core.actions.processes.person.addtopeoplesage.GetAgeStatistics;
 import com.kingsrook.qqq.backend.core.actions.tables.CountAction;
+import com.kingsrook.qqq.backend.core.actions.tables.GetAction;
 import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
 import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
 import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction;
@@ -45,6 +46,7 @@ import com.kingsrook.qqq.backend.core.logging.QLogger;
 import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
 import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
@@ -813,6 +815,11 @@ public class TestUtils
                .withFilter(new QQueryFilter().withCriteria(new QFilterCriteria("birthDate", QCriteriaOperator.GREATER_THAN, List.of(youngPersonLimitDate))))
                .withCodeReference(new QCodeReference(CheckAge.class))
             )
+            .withAction(new TableAutomationAction()
+               .withName("failAutomationForSith")
+               .withTriggerEvent(TriggerEvent.POST_INSERT)
+               .withCodeReference(new QCodeReference(FailAutomationForSith.class))
+            )
             .withAction(new TableAutomationAction()
                .withName("increaseBirthdate")
                .withTriggerEvent(TriggerEvent.POST_INSERT)
@@ -918,6 +925,15 @@ public class TestUtils
          List<QRecord> recordsToUpdate = new ArrayList<>();
          for(QRecord record : recordAutomationInput.getRecordList())
          {
+            ////////////////////////////////////////////////////////////////////////
+            // get the record - its automation status should currently be RUNNING //
+            ////////////////////////////////////////////////////////////////////////
+            QRecord freshlyFetchedRecord = new GetAction().executeForRecord(new GetInput(TABLE_NAME_PERSON_MEMORY).withPrimaryKey(record.getValue("id")));
+            assertEquals(AutomationStatus.RUNNING_INSERT_AUTOMATIONS.getId(), freshlyFetchedRecord.getValueInteger(TestUtils.standardQqqAutomationStatusField().getName()));
+
+            ///////////////////////////////////////////
+            // do whatever business logic we do here //
+            ///////////////////////////////////////////
             LocalDate birthDate = record.getValueLocalDate("birthDate");
             if(birthDate != null && birthDate.isAfter(limitDate))
             {
@@ -940,6 +956,29 @@ public class TestUtils
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public static class FailAutomationForSith extends RecordAutomationHandler
+   {
+
+      /*******************************************************************************
+       **
+       *******************************************************************************/
+      public void execute(RecordAutomationInput recordAutomationInput) throws QException
+      {
+         for(QRecord record : recordAutomationInput.getRecordList())
+         {
+            if("Darth".equals(record.getValue("firstName")))
+            {
+               throw new QException("Oops, you look like a Sith!");
+            }
+         }
+      }
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/