CE-847 - Add review screen to HealBadRecordAutomationStatusesProcess; update to query by createDate for pending-inserts

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/automation/HealBadRecordAutomationStatusesProcessStep.java

@@ -28,6 +28,7 @@ import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
+import java.util.function.Function;
 import com.kingsrook.qqq.backend.core.actions.automation.AutomationStatus;
 import com.kingsrook.qqq.backend.core.actions.processes.BackendStep;
 import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
@@ -101,20 +102,8 @@ public class HealBadRecordAutomationStatusesProcessStep implements BackendStep,
    @Override
    public QProcessMetaData produce(QInstance qInstance) throws QException
    {
-      QProcessMetaData processMetaData = new QProcessMetaData()
+      Function<String, QFrontendStepMetaData> makeReviewOrResultStep = (String name) -> new QFrontendStepMetaData()
-         .withName(NAME)
+         .withName(name)
-         .withStepList(List.of(
-            new QFrontendStepMetaData()
-               .withName("input")
-               .withComponent(new QFrontendComponentMetaData().withType(QComponentType.EDIT_FORM))
-               .withFormField(new QFieldMetaData("tableName", QFieldType.STRING).withPossibleValueSourceName(TablesPossibleValueSourceMetaDataProvider.NAME))
-               .withFormField(new QFieldMetaData("minutesOldLimit", QFieldType.INTEGER).withDefaultValue(60)),
-            new QBackendStepMetaData()
-               .withName("run")
-               .withCode(new QCodeReference(getClass())),
-            new QFrontendStepMetaData()
-               .withName("output")
-
          .withComponent(new NoCodeWidgetFrontendComponentMetaData()
             .withOutput(new WidgetHtmlLine()
                .withCondition(new QFilterCriteria("warningCount", QCriteriaOperator.GREATER_THAN, 0))
@@ -131,15 +120,29 @@ public class HealBadRecordAutomationStatusesProcessStep implements BackendStep,
                      #end
                   </ul>
                   """)))
-
          .withComponent(new QFrontendComponentMetaData().withType(QComponentType.VIEW_FORM))
-               .withViewField(new QFieldMetaData("totalRecordsUpdated", QFieldType.INTEGER) /* todo - didn't display commas... .withDisplayFormat(DisplayFormat.COMMAS) */)
+         .withViewField(new QFieldMetaData("review".equals(name) ? "totalRecordsToUpdate" : "totalRecordsUpdated", QFieldType.INTEGER) /* todo - didn't display commas... .withDisplayFormat(DisplayFormat.COMMAS) */)
-
          .withComponent(new QFrontendComponentMetaData().withType(QComponentType.RECORD_LIST))
          .withRecordListField(new QFieldMetaData("tableName", QFieldType.STRING))
          .withRecordListField(new QFieldMetaData("badStatus", QFieldType.STRING))
-               .withRecordListField(new QFieldMetaData("count", QFieldType.INTEGER).withDisplayFormat(DisplayFormat.COMMAS) /* todo - didn't display commas... */)
+         .withRecordListField(new QFieldMetaData("count", QFieldType.INTEGER).withDisplayFormat(DisplayFormat.COMMAS) /* todo - didn't display commas... */);
 
+      QProcessMetaData processMetaData = new QProcessMetaData()
+         .withName(NAME)
+         .withStepList(List.of(
+            new QFrontendStepMetaData()
+               .withName("input")
+               .withComponent(new QFrontendComponentMetaData().withType(QComponentType.EDIT_FORM))
+               .withFormField(new QFieldMetaData("tableName", QFieldType.STRING).withPossibleValueSourceName(TablesPossibleValueSourceMetaDataProvider.NAME))
+               .withFormField(new QFieldMetaData("minutesOldLimit", QFieldType.INTEGER).withDefaultValue(60)),
+            new QBackendStepMetaData()
+               .withName("preview")
+               .withCode(new QCodeReference(getClass())),
+            makeReviewOrResultStep.apply("review"),
+            new QBackendStepMetaData()
+               .withName("run")
+               .withCode(new QCodeReference(getClass())),
+            makeReviewOrResultStep.apply("result")
          ));
 
       return (processMetaData);
@@ -154,6 +157,7 @@ public class HealBadRecordAutomationStatusesProcessStep implements BackendStep,
    public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException
    {
       int recordsUpdated = 0;
+      boolean isReview = "preview".equals(runBackendStepInput.getStepName());
 
       ////////////////////////////////////////////////////////////////////////
       // if a table name is given, validate it, and run for just that table //
@@ -167,7 +171,7 @@ public class HealBadRecordAutomationStatusesProcessStep implements BackendStep,
             throw (new QException("Unrecognized table name: " + tableName));
          }
 
-         recordsUpdated += processTable(tableName, runBackendStepInput, runBackendStepOutput, warnings);
+         recordsUpdated += processTable(isReview, tableName, runBackendStepInput, runBackendStepOutput, warnings);
       }
       else
       {
@@ -176,11 +180,12 @@ public class HealBadRecordAutomationStatusesProcessStep implements BackendStep,
          //////////////////////////////////////////////////////////////////////////
          for(QTableMetaData table : QContext.getQInstance().getTables().values())
          {
-            recordsUpdated += processTable(table.getName(), runBackendStepInput, runBackendStepOutput, warnings);
+            recordsUpdated += processTable(isReview, table.getName(), runBackendStepInput, runBackendStepOutput, warnings);
          }
       }
 
       runBackendStepOutput.addValue("totalRecordsUpdated", recordsUpdated);
+      runBackendStepOutput.addValue("totalRecordsToUpdate", recordsUpdated);
       runBackendStepOutput.addValue("warnings", warnings);
       runBackendStepOutput.addValue("warningCount", warnings.size());
 
@@ -198,7 +203,7 @@ public class HealBadRecordAutomationStatusesProcessStep implements BackendStep,
    /*******************************************************************************
     **
     *******************************************************************************/
-   private int processTable(String tableName, RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput, List<String> warnings)
+   private int processTable(boolean isReview, String tableName, RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput, List<String> warnings)
    {
       try
       {
@@ -216,34 +221,42 @@ public class HealBadRecordAutomationStatusesProcessStep implements BackendStep,
             // find the modify-date field on the table //
             /////////////////////////////////////////////
             String modifyDateFieldName = null;
+            String createDateFieldName = null;
             for(QFieldMetaData field : table.getFields().values())
             {
                if(DynamicDefaultValueBehavior.MODIFY_DATE.equals(field.getBehaviorOnlyIfSet(DynamicDefaultValueBehavior.class)))
                {
                   modifyDateFieldName = field.getName();
-                  break;
                }
-            }
+               if(DynamicDefaultValueBehavior.CREATE_DATE.equals(field.getBehaviorOnlyIfSet(DynamicDefaultValueBehavior.class)))
-
-            if(modifyDateFieldName == null)
                {
-               warnings.add("Could not find a Modify Date field on table: " + tableName);
+                  createDateFieldName = field.getName();
-               LOG.info("Couldn't find a MODIFY_DATE field on table", logPair("tableName", tableName));
+               }
-               return 0;
+            }
+
+            //////////////////////////////////////////////////////////////////////////////////////////////////
+            // set up a filter to query for records either FAILED, or RUNNING w/ create/modify date too old //
+            //////////////////////////////////////////////////////////////////////////////////////////////////
+            QQueryFilter filter = new QQueryFilter().withBooleanOperator(QQueryFilter.BooleanOperator.OR);
+            filter.addSubFilter(new QQueryFilter().withCriteria(new QFilterCriteria(automationStatusFieldName, QCriteriaOperator.IN, AutomationStatus.FAILED_INSERT_AUTOMATIONS.getId(), AutomationStatus.FAILED_UPDATE_AUTOMATIONS.getId())));
+
+            if(modifyDateFieldName != null)
+            {
+               filter.addSubFilter(new QQueryFilter()
+                  .withCriteria(new QFilterCriteria(automationStatusFieldName, QCriteriaOperator.EQUALS, AutomationStatus.RUNNING_UPDATE_AUTOMATIONS.getId()))
+                  .withCriteria(new QFilterCriteria(modifyDateFieldName, QCriteriaOperator.LESS_THAN, NowWithOffset.minus(minutesOldLimit, ChronoUnit.MINUTES))));
+            }
+
+            if(createDateFieldName != null)
+            {
+               filter.addSubFilter(new QQueryFilter()
+                  .withCriteria(new QFilterCriteria(automationStatusFieldName, QCriteriaOperator.EQUALS, AutomationStatus.RUNNING_INSERT_AUTOMATIONS.getId()))
+                  .withCriteria(new QFilterCriteria(createDateFieldName, QCriteriaOperator.LESS_THAN, NowWithOffset.minus(minutesOldLimit, ChronoUnit.MINUTES))));
             }
 
-            ////////////////////////////////////////////////////////////////////////
-            // query for records either FAILED, or RUNNING w/ modify date too old //
-            ////////////////////////////////////////////////////////////////////////
             QueryInput queryInput = new QueryInput();
             queryInput.setTableName(tableName);
-            queryInput.setFilter(new QQueryFilter().withBooleanOperator(QQueryFilter.BooleanOperator.OR)
+            queryInput.setFilter(filter);
-               .withSubFilter(new QQueryFilter()
-                  .withCriteria(new QFilterCriteria(automationStatusFieldName, QCriteriaOperator.IN, AutomationStatus.FAILED_INSERT_AUTOMATIONS.getId(), AutomationStatus.FAILED_UPDATE_AUTOMATIONS.getId())))
-               .withSubFilter(new QQueryFilter()
-                  .withCriteria(new QFilterCriteria(automationStatusFieldName, QCriteriaOperator.IN, AutomationStatus.RUNNING_INSERT_AUTOMATIONS.getId(), AutomationStatus.RUNNING_UPDATE_AUTOMATIONS.getId()))
-                  .withCriteria(new QFilterCriteria(modifyDateFieldName, QCriteriaOperator.LESS_THAN, NowWithOffset.minus(minutesOldLimit, ChronoUnit.MINUTES))))
-            );
             QueryOutput queryOutput = new QueryAction().execute(queryInput);
 
             //////////////////////////////////////////////////////////////////////////////////////////////////////////////////
@@ -269,7 +282,10 @@ public class HealBadRecordAutomationStatusesProcessStep implements BackendStep,
                }
             }
 
-            if(!recordsToUpdate.isEmpty())
+            //////////////////////////////////////////////////////////////////////////////////////
+            // if there are record to update (and this isn't the review step), then update them //
+            //////////////////////////////////////////////////////////////////////////////////////
+            if(!recordsToUpdate.isEmpty() && !isReview)
             {
                LOG.info("Healing bad record automation statuses", logPair("tableName", tableName), logPair("count", recordsToUpdate.size()));
                new UpdateAction().execute(new UpdateInput(tableName).withRecords(recordsToUpdate).withOmitTriggeringAutomations(true));
@@ -278,7 +294,7 @@ public class HealBadRecordAutomationStatusesProcessStep implements BackendStep,
             for(Map.Entry<String, Integer> entry : countByStatus.entrySet())
             {
                runBackendStepOutput.addRecord(new QRecord()
-                  .withValue("tableName", tableName)
+                  .withValue("tableName", QContext.getQInstance().getTable(tableName).getLabel())
                   .withValue("badStatus", entry.getKey())
                   .withValue("count", entry.getValue()));
             }

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/automation/HealBadRecordAutomationStatusesProcessStepTest.java

@@ -103,7 +103,7 @@ class HealBadRecordAutomationStatusesProcessStepTest extends BaseTest
     **
     *******************************************************************************/
    @Test
-   void testOldRunning() throws QException
+   void testOldRunningUpdates() throws QException
    {
       /////////////////////////////////////////////////
       // temporarily remove the modify-date behavior //
@@ -160,6 +160,72 @@ class HealBadRecordAutomationStatusesProcessStepTest extends BaseTest
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testOldRunningInserts() throws QException
+   {
+      ///////////////////////////////////////////////////////////////
+      // temporarily remove the create-date & modify-date behavior //
+      ///////////////////////////////////////////////////////////////
+      QContext.getQInstance().getTable(tableName).getField("modifyDate").withBehavior(DynamicDefaultValueBehavior.NONE);
+      QContext.getQInstance().getTable(tableName).getField("createDate").withBehavior(DynamicDefaultValueBehavior.NONE);
+
+      //////////////////////////////////////////////////////////////////////////
+      // insert 2 records, one with an old createDate, one with 6 minutes ago //
+      // but set both with modifyDate very recent                             //
+      //////////////////////////////////////////////////////////////////////////
+      Instant old    = Instant.parse("2023-01-01T12:00:00Z");
+      Instant recent = Instant.now().minus(6, ChronoUnit.MINUTES);
+      new InsertAction().execute(new InsertInput(tableName).withRecords(List.of(
+         new QRecord().withValue("firstName", "Darin").withValue("createDate", old).withValue("modifyDate", recent),
+         new QRecord().withValue("firstName", "Tim").withValue("createDate", recent).withValue("modifyDate", recent)
+      )));
+      List<QRecord> records = queryAllRecords();
+
+      ///////////////////////////////////////////////////////
+      // put those records both in status: running-inserts //
+      ///////////////////////////////////////////////////////
+      RecordAutomationStatusUpdater.setAutomationStatusInRecordsAndUpdate(QContext.getQInstance().getTable(tableName), records, AutomationStatus.RUNNING_INSERT_AUTOMATIONS, null);
+
+      assertThat(queryAllRecords())
+         .allMatch(r -> AutomationStatus.RUNNING_INSERT_AUTOMATIONS.getId().equals(getAutomationStatus(r)));
+
+      //////////////////////////////////////////////////
+      // restore the createDate & modifyDate behavior //
+      //////////////////////////////////////////////////
+      QContext.getQInstance().getTable(tableName).getField("modifyDate").withBehavior(DynamicDefaultValueBehavior.MODIFY_DATE);
+      QContext.getQInstance().getTable(tableName).getField("createDate").withBehavior(DynamicDefaultValueBehavior.CREATE_DATE);
+
+      /////////////////////////
+      // run code under test //
+      /////////////////////////
+      RunBackendStepOutput output = runProcessStep();
+
+      /////////////////////////////////////////////////////////////////////////////////////////////
+      // assert we updated 1 (the old one) to pending-inserts, the other left as running-inserts //
+      /////////////////////////////////////////////////////////////////////////////////////////////
+      assertEquals(1, output.getValueInteger("totalRecordsUpdated"));
+      assertThat(queryAllRecords())
+         .anyMatch(r -> AutomationStatus.PENDING_INSERT_AUTOMATIONS.getId().equals(getAutomationStatus(r)))
+         .anyMatch(r -> AutomationStatus.RUNNING_INSERT_AUTOMATIONS.getId().equals(getAutomationStatus(r)));
+
+      /////////////////////////////////
+      // re-run, with 3-minute limit //
+      /////////////////////////////////
+      output = runProcessStep(new RunBackendStepInput().withValues(Map.of("minutesOldLimit", 3)));
+
+      /////////////////////////////////////////////////////////////////
+      // assert that one updated too, and all are now pending-insert //
+      /////////////////////////////////////////////////////////////////
+      assertEquals(1, output.getValueInteger("totalRecordsUpdated"));
+      assertThat(queryAllRecords())
+         .allMatch(r -> AutomationStatus.PENDING_INSERT_AUTOMATIONS.getId().equals(getAutomationStatus(r)));
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/