From cdf59e8f2b12d93784f3e6230369ac67d0128998 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Tue, 27 Feb 2024 10:09:00 -0600 Subject: [PATCH] CE-847 - Add review screen to HealBadRecordAutomationStatusesProcess; update to query by createDate for pending-inserts --- ...adRecordAutomationStatusesProcessStep.java | 112 ++++++++++-------- ...cordAutomationStatusesProcessStepTest.java | 68 ++++++++++- 2 files changed, 131 insertions(+), 49 deletions(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/automation/HealBadRecordAutomationStatusesProcessStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/automation/HealBadRecordAutomationStatusesProcessStep.java index 92d3b3a3..59a46c69 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/automation/HealBadRecordAutomationStatusesProcessStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/automation/HealBadRecordAutomationStatusesProcessStep.java @@ -28,6 +28,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.function.Function; import com.kingsrook.qqq.backend.core.actions.automation.AutomationStatus; import com.kingsrook.qqq.backend.core.actions.processes.BackendStep; import com.kingsrook.qqq.backend.core.actions.tables.QueryAction; @@ -101,6 +102,31 @@ public class HealBadRecordAutomationStatusesProcessStep implements BackendStep, @Override public QProcessMetaData produce(QInstance qInstance) throws QException { + Function makeReviewOrResultStep = (String name) -> new QFrontendStepMetaData() + .withName(name) + .withComponent(new NoCodeWidgetFrontendComponentMetaData() + .withOutput(new WidgetHtmlLine() + .withCondition(new QFilterCriteria("warningCount", QCriteriaOperator.GREATER_THAN, 0)) + .withWrapper(HtmlWrapper.divWithStyles(HtmlWrapper.STYLE_YELLOW)) + .withVelocityTemplate("Warning:")) + .withOutput(new WidgetHtmlLine() + .withCondition(new QFilterCriteria("warningCount", QCriteriaOperator.GREATER_THAN, 0)) + .withWrapper(HtmlWrapper.divWithStyles(HtmlWrapper.STYLE_INDENT_1)) + .withWrapper(HtmlWrapper.divWithStyles(HtmlWrapper.STYLE_YELLOW)) + .withVelocityTemplate(""" + + """))) + .withComponent(new QFrontendComponentMetaData().withType(QComponentType.VIEW_FORM)) + .withViewField(new QFieldMetaData("review".equals(name) ? "totalRecordsToUpdate" : "totalRecordsUpdated", QFieldType.INTEGER) /* todo - didn't display commas... .withDisplayFormat(DisplayFormat.COMMAS) */) + .withComponent(new QFrontendComponentMetaData().withType(QComponentType.RECORD_LIST)) + .withRecordListField(new QFieldMetaData("tableName", QFieldType.STRING)) + .withRecordListField(new QFieldMetaData("badStatus", QFieldType.STRING)) + .withRecordListField(new QFieldMetaData("count", QFieldType.INTEGER).withDisplayFormat(DisplayFormat.COMMAS) /* todo - didn't display commas... */); + QProcessMetaData processMetaData = new QProcessMetaData() .withName(NAME) .withStepList(List.of( @@ -109,37 +135,14 @@ public class HealBadRecordAutomationStatusesProcessStep implements BackendStep, .withComponent(new QFrontendComponentMetaData().withType(QComponentType.EDIT_FORM)) .withFormField(new QFieldMetaData("tableName", QFieldType.STRING).withPossibleValueSourceName(TablesPossibleValueSourceMetaDataProvider.NAME)) .withFormField(new QFieldMetaData("minutesOldLimit", QFieldType.INTEGER).withDefaultValue(60)), + new QBackendStepMetaData() + .withName("preview") + .withCode(new QCodeReference(getClass())), + makeReviewOrResultStep.apply("review"), new QBackendStepMetaData() .withName("run") .withCode(new QCodeReference(getClass())), - new QFrontendStepMetaData() - .withName("output") - - .withComponent(new NoCodeWidgetFrontendComponentMetaData() - .withOutput(new WidgetHtmlLine() - .withCondition(new QFilterCriteria("warningCount", QCriteriaOperator.GREATER_THAN, 0)) - .withWrapper(HtmlWrapper.divWithStyles(HtmlWrapper.STYLE_YELLOW)) - .withVelocityTemplate("Warning:")) - .withOutput(new WidgetHtmlLine() - .withCondition(new QFilterCriteria("warningCount", QCriteriaOperator.GREATER_THAN, 0)) - .withWrapper(HtmlWrapper.divWithStyles(HtmlWrapper.STYLE_INDENT_1)) - .withWrapper(HtmlWrapper.divWithStyles(HtmlWrapper.STYLE_YELLOW)) - .withVelocityTemplate(""" - - """))) - - .withComponent(new QFrontendComponentMetaData().withType(QComponentType.VIEW_FORM)) - .withViewField(new QFieldMetaData("totalRecordsUpdated", QFieldType.INTEGER) /* todo - didn't display commas... .withDisplayFormat(DisplayFormat.COMMAS) */) - - .withComponent(new QFrontendComponentMetaData().withType(QComponentType.RECORD_LIST)) - .withRecordListField(new QFieldMetaData("tableName", QFieldType.STRING)) - .withRecordListField(new QFieldMetaData("badStatus", QFieldType.STRING)) - .withRecordListField(new QFieldMetaData("count", QFieldType.INTEGER).withDisplayFormat(DisplayFormat.COMMAS) /* todo - didn't display commas... */) - + makeReviewOrResultStep.apply("result") )); return (processMetaData); @@ -154,6 +157,7 @@ public class HealBadRecordAutomationStatusesProcessStep implements BackendStep, public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException { int recordsUpdated = 0; + boolean isReview = "preview".equals(runBackendStepInput.getStepName()); //////////////////////////////////////////////////////////////////////// // if a table name is given, validate it, and run for just that table // @@ -167,7 +171,7 @@ public class HealBadRecordAutomationStatusesProcessStep implements BackendStep, throw (new QException("Unrecognized table name: " + tableName)); } - recordsUpdated += processTable(tableName, runBackendStepInput, runBackendStepOutput, warnings); + recordsUpdated += processTable(isReview, tableName, runBackendStepInput, runBackendStepOutput, warnings); } else { @@ -176,11 +180,12 @@ public class HealBadRecordAutomationStatusesProcessStep implements BackendStep, ////////////////////////////////////////////////////////////////////////// for(QTableMetaData table : QContext.getQInstance().getTables().values()) { - recordsUpdated += processTable(table.getName(), runBackendStepInput, runBackendStepOutput, warnings); + recordsUpdated += processTable(isReview, table.getName(), runBackendStepInput, runBackendStepOutput, warnings); } } runBackendStepOutput.addValue("totalRecordsUpdated", recordsUpdated); + runBackendStepOutput.addValue("totalRecordsToUpdate", recordsUpdated); runBackendStepOutput.addValue("warnings", warnings); runBackendStepOutput.addValue("warningCount", warnings.size()); @@ -198,7 +203,7 @@ public class HealBadRecordAutomationStatusesProcessStep implements BackendStep, /******************************************************************************* ** *******************************************************************************/ - private int processTable(String tableName, RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput, List warnings) + private int processTable(boolean isReview, String tableName, RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput, List warnings) { try { @@ -216,34 +221,42 @@ public class HealBadRecordAutomationStatusesProcessStep implements BackendStep, // find the modify-date field on the table // ///////////////////////////////////////////// String modifyDateFieldName = null; + String createDateFieldName = null; for(QFieldMetaData field : table.getFields().values()) { if(DynamicDefaultValueBehavior.MODIFY_DATE.equals(field.getBehaviorOnlyIfSet(DynamicDefaultValueBehavior.class))) { modifyDateFieldName = field.getName(); - break; + } + if(DynamicDefaultValueBehavior.CREATE_DATE.equals(field.getBehaviorOnlyIfSet(DynamicDefaultValueBehavior.class))) + { + createDateFieldName = field.getName(); } } - if(modifyDateFieldName == null) + ////////////////////////////////////////////////////////////////////////////////////////////////// + // set up a filter to query for records either FAILED, or RUNNING w/ create/modify date too old // + ////////////////////////////////////////////////////////////////////////////////////////////////// + QQueryFilter filter = new QQueryFilter().withBooleanOperator(QQueryFilter.BooleanOperator.OR); + filter.addSubFilter(new QQueryFilter().withCriteria(new QFilterCriteria(automationStatusFieldName, QCriteriaOperator.IN, AutomationStatus.FAILED_INSERT_AUTOMATIONS.getId(), AutomationStatus.FAILED_UPDATE_AUTOMATIONS.getId()))); + + if(modifyDateFieldName != null) { - warnings.add("Could not find a Modify Date field on table: " + tableName); - LOG.info("Couldn't find a MODIFY_DATE field on table", logPair("tableName", tableName)); - return 0; + filter.addSubFilter(new QQueryFilter() + .withCriteria(new QFilterCriteria(automationStatusFieldName, QCriteriaOperator.EQUALS, AutomationStatus.RUNNING_UPDATE_AUTOMATIONS.getId())) + .withCriteria(new QFilterCriteria(modifyDateFieldName, QCriteriaOperator.LESS_THAN, NowWithOffset.minus(minutesOldLimit, ChronoUnit.MINUTES)))); + } + + if(createDateFieldName != null) + { + filter.addSubFilter(new QQueryFilter() + .withCriteria(new QFilterCriteria(automationStatusFieldName, QCriteriaOperator.EQUALS, AutomationStatus.RUNNING_INSERT_AUTOMATIONS.getId())) + .withCriteria(new QFilterCriteria(createDateFieldName, QCriteriaOperator.LESS_THAN, NowWithOffset.minus(minutesOldLimit, ChronoUnit.MINUTES)))); } - //////////////////////////////////////////////////////////////////////// - // query for records either FAILED, or RUNNING w/ modify date too old // - //////////////////////////////////////////////////////////////////////// QueryInput queryInput = new QueryInput(); queryInput.setTableName(tableName); - queryInput.setFilter(new QQueryFilter().withBooleanOperator(QQueryFilter.BooleanOperator.OR) - .withSubFilter(new QQueryFilter() - .withCriteria(new QFilterCriteria(automationStatusFieldName, QCriteriaOperator.IN, AutomationStatus.FAILED_INSERT_AUTOMATIONS.getId(), AutomationStatus.FAILED_UPDATE_AUTOMATIONS.getId()))) - .withSubFilter(new QQueryFilter() - .withCriteria(new QFilterCriteria(automationStatusFieldName, QCriteriaOperator.IN, AutomationStatus.RUNNING_INSERT_AUTOMATIONS.getId(), AutomationStatus.RUNNING_UPDATE_AUTOMATIONS.getId())) - .withCriteria(new QFilterCriteria(modifyDateFieldName, QCriteriaOperator.LESS_THAN, NowWithOffset.minus(minutesOldLimit, ChronoUnit.MINUTES)))) - ); + queryInput.setFilter(filter); QueryOutput queryOutput = new QueryAction().execute(queryInput); ////////////////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -269,7 +282,10 @@ public class HealBadRecordAutomationStatusesProcessStep implements BackendStep, } } - if(!recordsToUpdate.isEmpty()) + ////////////////////////////////////////////////////////////////////////////////////// + // if there are record to update (and this isn't the review step), then update them // + ////////////////////////////////////////////////////////////////////////////////////// + if(!recordsToUpdate.isEmpty() && !isReview) { LOG.info("Healing bad record automation statuses", logPair("tableName", tableName), logPair("count", recordsToUpdate.size())); new UpdateAction().execute(new UpdateInput(tableName).withRecords(recordsToUpdate).withOmitTriggeringAutomations(true)); @@ -278,7 +294,7 @@ public class HealBadRecordAutomationStatusesProcessStep implements BackendStep, for(Map.Entry entry : countByStatus.entrySet()) { runBackendStepOutput.addRecord(new QRecord() - .withValue("tableName", tableName) + .withValue("tableName", QContext.getQInstance().getTable(tableName).getLabel()) .withValue("badStatus", entry.getKey()) .withValue("count", entry.getValue())); } diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/automation/HealBadRecordAutomationStatusesProcessStepTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/automation/HealBadRecordAutomationStatusesProcessStepTest.java index b6124e69..caf74762 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/automation/HealBadRecordAutomationStatusesProcessStepTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/automation/HealBadRecordAutomationStatusesProcessStepTest.java @@ -103,7 +103,7 @@ class HealBadRecordAutomationStatusesProcessStepTest extends BaseTest ** *******************************************************************************/ @Test - void testOldRunning() throws QException + void testOldRunningUpdates() throws QException { ///////////////////////////////////////////////// // temporarily remove the modify-date behavior // @@ -160,6 +160,72 @@ class HealBadRecordAutomationStatusesProcessStepTest extends BaseTest + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testOldRunningInserts() throws QException + { + /////////////////////////////////////////////////////////////// + // temporarily remove the create-date & modify-date behavior // + /////////////////////////////////////////////////////////////// + QContext.getQInstance().getTable(tableName).getField("modifyDate").withBehavior(DynamicDefaultValueBehavior.NONE); + QContext.getQInstance().getTable(tableName).getField("createDate").withBehavior(DynamicDefaultValueBehavior.NONE); + + ////////////////////////////////////////////////////////////////////////// + // insert 2 records, one with an old createDate, one with 6 minutes ago // + // but set both with modifyDate very recent // + ////////////////////////////////////////////////////////////////////////// + Instant old = Instant.parse("2023-01-01T12:00:00Z"); + Instant recent = Instant.now().minus(6, ChronoUnit.MINUTES); + new InsertAction().execute(new InsertInput(tableName).withRecords(List.of( + new QRecord().withValue("firstName", "Darin").withValue("createDate", old).withValue("modifyDate", recent), + new QRecord().withValue("firstName", "Tim").withValue("createDate", recent).withValue("modifyDate", recent) + ))); + List records = queryAllRecords(); + + /////////////////////////////////////////////////////// + // put those records both in status: running-inserts // + /////////////////////////////////////////////////////// + RecordAutomationStatusUpdater.setAutomationStatusInRecordsAndUpdate(QContext.getQInstance().getTable(tableName), records, AutomationStatus.RUNNING_INSERT_AUTOMATIONS, null); + + assertThat(queryAllRecords()) + .allMatch(r -> AutomationStatus.RUNNING_INSERT_AUTOMATIONS.getId().equals(getAutomationStatus(r))); + + ////////////////////////////////////////////////// + // restore the createDate & modifyDate behavior // + ////////////////////////////////////////////////// + QContext.getQInstance().getTable(tableName).getField("modifyDate").withBehavior(DynamicDefaultValueBehavior.MODIFY_DATE); + QContext.getQInstance().getTable(tableName).getField("createDate").withBehavior(DynamicDefaultValueBehavior.CREATE_DATE); + + ///////////////////////// + // run code under test // + ///////////////////////// + RunBackendStepOutput output = runProcessStep(); + + ///////////////////////////////////////////////////////////////////////////////////////////// + // assert we updated 1 (the old one) to pending-inserts, the other left as running-inserts // + ///////////////////////////////////////////////////////////////////////////////////////////// + assertEquals(1, output.getValueInteger("totalRecordsUpdated")); + assertThat(queryAllRecords()) + .anyMatch(r -> AutomationStatus.PENDING_INSERT_AUTOMATIONS.getId().equals(getAutomationStatus(r))) + .anyMatch(r -> AutomationStatus.RUNNING_INSERT_AUTOMATIONS.getId().equals(getAutomationStatus(r))); + + ///////////////////////////////// + // re-run, with 3-minute limit // + ///////////////////////////////// + output = runProcessStep(new RunBackendStepInput().withValues(Map.of("minutesOldLimit", 3))); + + ///////////////////////////////////////////////////////////////// + // assert that one updated too, and all are now pending-insert // + ///////////////////////////////////////////////////////////////// + assertEquals(1, output.getValueInteger("totalRecordsUpdated")); + assertThat(queryAllRecords()) + .allMatch(r -> AutomationStatus.PENDING_INSERT_AUTOMATIONS.getId().equals(getAutomationStatus(r))); + } + + + /******************************************************************************* ** *******************************************************************************/