QRunIO/qqq
2
0
Fork 0
mirror of https://github.com/Kingsrook/qqq.git synced 2025-07-18 13:10:44 +00:00
Code Issues Packages Projects Releases Wiki Activity

CE-847 - Add review screen to HealBadRecordAutomationStatusesProcess; update to query by createDate for pending-inserts

Browse Source
This commit is contained in:
darin.kelkhoff
2024-02-27 10:09:00 -06:00
parent edf942a01b
commit cdf59e8f2b
2 changed files with 131 additions and 49 deletions
Download Patch File Download Diff File Expand all files Collapse all files

112
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/automation/HealBadRecordAutomationStatusesProcessStep.java
View File

@ -28,6 +28,7 @@ import java.util.HashMap;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.Objects; import java.util.Objects;
import java.util.function.Function;
import com.kingsrook.qqq.backend.core.actions.automation.AutomationStatus; import com.kingsrook.qqq.backend.core.actions.automation.AutomationStatus;
import com.kingsrook.qqq.backend.core.actions.processes.BackendStep; import com.kingsrook.qqq.backend.core.actions.processes.BackendStep;
import com.kingsrook.qqq.backend.core.actions.tables.QueryAction; import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
@ -101,6 +102,31 @@ public class HealBadRecordAutomationStatusesProcessStep implements BackendStep,
@Override @Override
public QProcessMetaData produce(QInstance qInstance) throws QException public QProcessMetaData produce(QInstance qInstance) throws QException
{ {
Function<String, QFrontendStepMetaData> makeReviewOrResultStep = (String name) -> new QFrontendStepMetaData()
.withName(name)
.withComponent(new NoCodeWidgetFrontendComponentMetaData()
.withOutput(new WidgetHtmlLine()
.withCondition(new QFilterCriteria("warningCount", QCriteriaOperator.GREATER_THAN, 0))
.withWrapper(HtmlWrapper.divWithStyles(HtmlWrapper.STYLE_YELLOW))
.withVelocityTemplate("<b>Warning:</b>"))
.withOutput(new WidgetHtmlLine()
.withCondition(new QFilterCriteria("warningCount", QCriteriaOperator.GREATER_THAN, 0))
.withWrapper(HtmlWrapper.divWithStyles(HtmlWrapper.STYLE_INDENT_1))
.withWrapper(HtmlWrapper.divWithStyles(HtmlWrapper.STYLE_YELLOW))
.withVelocityTemplate("""
<ul>
#foreach($string in $warnings)
<li>$string</li>
#end
</ul>
""")))
.withComponent(new QFrontendComponentMetaData().withType(QComponentType.VIEW_FORM))
.withViewField(new QFieldMetaData("review".equals(name) ? "totalRecordsToUpdate" : "totalRecordsUpdated", QFieldType.INTEGER) /* todo - didn't display commas... .withDisplayFormat(DisplayFormat.COMMAS) */)
.withComponent(new QFrontendComponentMetaData().withType(QComponentType.RECORD_LIST))
.withRecordListField(new QFieldMetaData("tableName", QFieldType.STRING))
.withRecordListField(new QFieldMetaData("badStatus", QFieldType.STRING))
.withRecordListField(new QFieldMetaData("count", QFieldType.INTEGER).withDisplayFormat(DisplayFormat.COMMAS) /* todo - didn't display commas... */);
QProcessMetaData processMetaData = new QProcessMetaData() QProcessMetaData processMetaData = new QProcessMetaData()
.withName(NAME) .withName(NAME)
.withStepList(List.of( .withStepList(List.of(
@ -109,37 +135,14 @@ public class HealBadRecordAutomationStatusesProcessStep implements BackendStep,
.withComponent(new QFrontendComponentMetaData().withType(QComponentType.EDIT_FORM)) .withComponent(new QFrontendComponentMetaData().withType(QComponentType.EDIT_FORM))
.withFormField(new QFieldMetaData("tableName", QFieldType.STRING).withPossibleValueSourceName(TablesPossibleValueSourceMetaDataProvider.NAME)) .withFormField(new QFieldMetaData("tableName", QFieldType.STRING).withPossibleValueSourceName(TablesPossibleValueSourceMetaDataProvider.NAME))
.withFormField(new QFieldMetaData("minutesOldLimit", QFieldType.INTEGER).withDefaultValue(60)), .withFormField(new QFieldMetaData("minutesOldLimit", QFieldType.INTEGER).withDefaultValue(60)),
new QBackendStepMetaData()
.withName("preview")
.withCode(new QCodeReference(getClass())),
makeReviewOrResultStep.apply("review"),
new QBackendStepMetaData() new QBackendStepMetaData()
.withName("run") .withName("run")
.withCode(new QCodeReference(getClass())), .withCode(new QCodeReference(getClass())),
new QFrontendStepMetaData() makeReviewOrResultStep.apply("result")
.withName("output")
.withComponent(new NoCodeWidgetFrontendComponentMetaData()
.withOutput(new WidgetHtmlLine()
.withCondition(new QFilterCriteria("warningCount", QCriteriaOperator.GREATER_THAN, 0))
.withWrapper(HtmlWrapper.divWithStyles(HtmlWrapper.STYLE_YELLOW))
.withVelocityTemplate("<b>Warning:</b>"))
.withOutput(new WidgetHtmlLine()
.withCondition(new QFilterCriteria("warningCount", QCriteriaOperator.GREATER_THAN, 0))
.withWrapper(HtmlWrapper.divWithStyles(HtmlWrapper.STYLE_INDENT_1))
.withWrapper(HtmlWrapper.divWithStyles(HtmlWrapper.STYLE_YELLOW))
.withVelocityTemplate("""
<ul>
#foreach($string in $warnings)
<li>$string</li>
#end
</ul>
""")))
.withComponent(new QFrontendComponentMetaData().withType(QComponentType.VIEW_FORM))
.withViewField(new QFieldMetaData("totalRecordsUpdated", QFieldType.INTEGER) /* todo - didn't display commas... .withDisplayFormat(DisplayFormat.COMMAS) */)
.withComponent(new QFrontendComponentMetaData().withType(QComponentType.RECORD_LIST))
.withRecordListField(new QFieldMetaData("tableName", QFieldType.STRING))
.withRecordListField(new QFieldMetaData("badStatus", QFieldType.STRING))
.withRecordListField(new QFieldMetaData("count", QFieldType.INTEGER).withDisplayFormat(DisplayFormat.COMMAS) /* todo - didn't display commas... */)
)); ));
return (processMetaData); return (processMetaData);
@ -154,6 +157,7 @@ public class HealBadRecordAutomationStatusesProcessStep implements BackendStep,
public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException
{ {
int recordsUpdated = 0; int recordsUpdated = 0;
boolean isReview = "preview".equals(runBackendStepInput.getStepName());
//////////////////////////////////////////////////////////////////////// ////////////////////////////////////////////////////////////////////////
// if a table name is given, validate it, and run for just that table // // if a table name is given, validate it, and run for just that table //
@ -167,7 +171,7 @@ public class HealBadRecordAutomationStatusesProcessStep implements BackendStep,
throw (new QException("Unrecognized table name: " + tableName)); throw (new QException("Unrecognized table name: " + tableName));
} }
recordsUpdated += processTable(tableName, runBackendStepInput, runBackendStepOutput, warnings); recordsUpdated += processTable(isReview, tableName, runBackendStepInput, runBackendStepOutput, warnings);
} }
else else
{ {
@ -176,11 +180,12 @@ public class HealBadRecordAutomationStatusesProcessStep implements BackendStep,
////////////////////////////////////////////////////////////////////////// //////////////////////////////////////////////////////////////////////////
for(QTableMetaData table : QContext.getQInstance().getTables().values()) for(QTableMetaData table : QContext.getQInstance().getTables().values())
{ {
recordsUpdated += processTable(table.getName(), runBackendStepInput, runBackendStepOutput, warnings); recordsUpdated += processTable(isReview, table.getName(), runBackendStepInput, runBackendStepOutput, warnings);
} }
} }
runBackendStepOutput.addValue("totalRecordsUpdated", recordsUpdated); runBackendStepOutput.addValue("totalRecordsUpdated", recordsUpdated);
runBackendStepOutput.addValue("totalRecordsToUpdate", recordsUpdated);
runBackendStepOutput.addValue("warnings", warnings); runBackendStepOutput.addValue("warnings", warnings);
runBackendStepOutput.addValue("warningCount", warnings.size()); runBackendStepOutput.addValue("warningCount", warnings.size());
@ -198,7 +203,7 @@ public class HealBadRecordAutomationStatusesProcessStep implements BackendStep,
/******************************************************************************* /*******************************************************************************
** **
*******************************************************************************/ *******************************************************************************/
private int processTable(String tableName, RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput, List<String> warnings) private int processTable(boolean isReview, String tableName, RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput, List<String> warnings)
{ {
try try
{ {
@ -216,34 +221,42 @@ public class HealBadRecordAutomationStatusesProcessStep implements BackendStep,
// find the modify-date field on the table // // find the modify-date field on the table //
///////////////////////////////////////////// /////////////////////////////////////////////
String modifyDateFieldName = null; String modifyDateFieldName = null;
String createDateFieldName = null;
for(QFieldMetaData field : table.getFields().values()) for(QFieldMetaData field : table.getFields().values())
{ {
if(DynamicDefaultValueBehavior.MODIFY_DATE.equals(field.getBehaviorOnlyIfSet(DynamicDefaultValueBehavior.class))) if(DynamicDefaultValueBehavior.MODIFY_DATE.equals(field.getBehaviorOnlyIfSet(DynamicDefaultValueBehavior.class)))
{ {
modifyDateFieldName = field.getName(); modifyDateFieldName = field.getName();
break; }
if(DynamicDefaultValueBehavior.CREATE_DATE.equals(field.getBehaviorOnlyIfSet(DynamicDefaultValueBehavior.class)))
{
createDateFieldName = field.getName();
} }
} }
if(modifyDateFieldName == null) //////////////////////////////////////////////////////////////////////////////////////////////////
// set up a filter to query for records either FAILED, or RUNNING w/ create/modify date too old //
//////////////////////////////////////////////////////////////////////////////////////////////////
QQueryFilter filter = new QQueryFilter().withBooleanOperator(QQueryFilter.BooleanOperator.OR);
filter.addSubFilter(new QQueryFilter().withCriteria(new QFilterCriteria(automationStatusFieldName, QCriteriaOperator.IN, AutomationStatus.FAILED_INSERT_AUTOMATIONS.getId(), AutomationStatus.FAILED_UPDATE_AUTOMATIONS.getId())));
if(modifyDateFieldName != null)
{ {
warnings.add("Could not find a Modify Date field on table: " + tableName); filter.addSubFilter(new QQueryFilter()
LOG.info("Couldn't find a MODIFY_DATE field on table", logPair("tableName", tableName)); .withCriteria(new QFilterCriteria(automationStatusFieldName, QCriteriaOperator.EQUALS, AutomationStatus.RUNNING_UPDATE_AUTOMATIONS.getId()))
return 0; .withCriteria(new QFilterCriteria(modifyDateFieldName, QCriteriaOperator.LESS_THAN, NowWithOffset.minus(minutesOldLimit, ChronoUnit.MINUTES))));
}
if(createDateFieldName != null)
{
filter.addSubFilter(new QQueryFilter()
.withCriteria(new QFilterCriteria(automationStatusFieldName, QCriteriaOperator.EQUALS, AutomationStatus.RUNNING_INSERT_AUTOMATIONS.getId()))
.withCriteria(new QFilterCriteria(createDateFieldName, QCriteriaOperator.LESS_THAN, NowWithOffset.minus(minutesOldLimit, ChronoUnit.MINUTES))));
} }
////////////////////////////////////////////////////////////////////////
// query for records either FAILED, or RUNNING w/ modify date too old //
////////////////////////////////////////////////////////////////////////
QueryInput queryInput = new QueryInput(); QueryInput queryInput = new QueryInput();
queryInput.setTableName(tableName); queryInput.setTableName(tableName);
queryInput.setFilter(new QQueryFilter().withBooleanOperator(QQueryFilter.BooleanOperator.OR) queryInput.setFilter(filter);
.withSubFilter(new QQueryFilter()
.withCriteria(new QFilterCriteria(automationStatusFieldName, QCriteriaOperator.IN, AutomationStatus.FAILED_INSERT_AUTOMATIONS.getId(), AutomationStatus.FAILED_UPDATE_AUTOMATIONS.getId())))
.withSubFilter(new QQueryFilter()
.withCriteria(new QFilterCriteria(automationStatusFieldName, QCriteriaOperator.IN, AutomationStatus.RUNNING_INSERT_AUTOMATIONS.getId(), AutomationStatus.RUNNING_UPDATE_AUTOMATIONS.getId()))
.withCriteria(new QFilterCriteria(modifyDateFieldName, QCriteriaOperator.LESS_THAN, NowWithOffset.minus(minutesOldLimit, ChronoUnit.MINUTES))))
);
QueryOutput queryOutput = new QueryAction().execute(queryInput); QueryOutput queryOutput = new QueryAction().execute(queryInput);
////////////////////////////////////////////////////////////////////////////////////////////////////////////////// //////////////////////////////////////////////////////////////////////////////////////////////////////////////////
@ -269,7 +282,10 @@ public class HealBadRecordAutomationStatusesProcessStep implements BackendStep,
} }
} }
if(!recordsToUpdate.isEmpty()) //////////////////////////////////////////////////////////////////////////////////////
// if there are record to update (and this isn't the review step), then update them //
//////////////////////////////////////////////////////////////////////////////////////
if(!recordsToUpdate.isEmpty() && !isReview)
{ {
LOG.info("Healing bad record automation statuses", logPair("tableName", tableName), logPair("count", recordsToUpdate.size())); LOG.info("Healing bad record automation statuses", logPair("tableName", tableName), logPair("count", recordsToUpdate.size()));
new UpdateAction().execute(new UpdateInput(tableName).withRecords(recordsToUpdate).withOmitTriggeringAutomations(true)); new UpdateAction().execute(new UpdateInput(tableName).withRecords(recordsToUpdate).withOmitTriggeringAutomations(true));
@ -278,7 +294,7 @@ public class HealBadRecordAutomationStatusesProcessStep implements BackendStep,
for(Map.Entry<String, Integer> entry : countByStatus.entrySet()) for(Map.Entry<String, Integer> entry : countByStatus.entrySet())
{ {
runBackendStepOutput.addRecord(new QRecord() runBackendStepOutput.addRecord(new QRecord()
.withValue("tableName", tableName) .withValue("tableName", QContext.getQInstance().getTable(tableName).getLabel())
.withValue("badStatus", entry.getKey()) .withValue("badStatus", entry.getKey())
.withValue("count", entry.getValue())); .withValue("count", entry.getValue()));
} }

68
qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/automation/HealBadRecordAutomationStatusesProcessStepTest.java
View File

@ -103,7 +103,7 @@ class HealBadRecordAutomationStatusesProcessStepTest extends BaseTest
** **
*******************************************************************************/ *******************************************************************************/
@Test @Test
void testOldRunning() throws QException void testOldRunningUpdates() throws QException
{ {
///////////////////////////////////////////////// /////////////////////////////////////////////////
// temporarily remove the modify-date behavior // // temporarily remove the modify-date behavior //
@ -160,6 +160,72 @@ class HealBadRecordAutomationStatusesProcessStepTest extends BaseTest
/*******************************************************************************
**
*******************************************************************************/
@Test
void testOldRunningInserts() throws QException
{
///////////////////////////////////////////////////////////////
// temporarily remove the create-date & modify-date behavior //
///////////////////////////////////////////////////////////////
QContext.getQInstance().getTable(tableName).getField("modifyDate").withBehavior(DynamicDefaultValueBehavior.NONE);
QContext.getQInstance().getTable(tableName).getField("createDate").withBehavior(DynamicDefaultValueBehavior.NONE);
//////////////////////////////////////////////////////////////////////////
// insert 2 records, one with an old createDate, one with 6 minutes ago //
// but set both with modifyDate very recent //
//////////////////////////////////////////////////////////////////////////
Instant old = Instant.parse("2023-01-01T12:00:00Z");
Instant recent = Instant.now().minus(6, ChronoUnit.MINUTES);
new InsertAction().execute(new InsertInput(tableName).withRecords(List.of(
new QRecord().withValue("firstName", "Darin").withValue("createDate", old).withValue("modifyDate", recent),
new QRecord().withValue("firstName", "Tim").withValue("createDate", recent).withValue("modifyDate", recent)
)));
List<QRecord> records = queryAllRecords();
///////////////////////////////////////////////////////
// put those records both in status: running-inserts //
///////////////////////////////////////////////////////
RecordAutomationStatusUpdater.setAutomationStatusInRecordsAndUpdate(QContext.getQInstance().getTable(tableName), records, AutomationStatus.RUNNING_INSERT_AUTOMATIONS, null);
assertThat(queryAllRecords())
.allMatch(r -> AutomationStatus.RUNNING_INSERT_AUTOMATIONS.getId().equals(getAutomationStatus(r)));
//////////////////////////////////////////////////
// restore the createDate & modifyDate behavior //
//////////////////////////////////////////////////
QContext.getQInstance().getTable(tableName).getField("modifyDate").withBehavior(DynamicDefaultValueBehavior.MODIFY_DATE);
QContext.getQInstance().getTable(tableName).getField("createDate").withBehavior(DynamicDefaultValueBehavior.CREATE_DATE);
/////////////////////////
// run code under test //
/////////////////////////
RunBackendStepOutput output = runProcessStep();
/////////////////////////////////////////////////////////////////////////////////////////////
// assert we updated 1 (the old one) to pending-inserts, the other left as running-inserts //
/////////////////////////////////////////////////////////////////////////////////////////////
assertEquals(1, output.getValueInteger("totalRecordsUpdated"));
assertThat(queryAllRecords())
.anyMatch(r -> AutomationStatus.PENDING_INSERT_AUTOMATIONS.getId().equals(getAutomationStatus(r)))
.anyMatch(r -> AutomationStatus.RUNNING_INSERT_AUTOMATIONS.getId().equals(getAutomationStatus(r)));
/////////////////////////////////
// re-run, with 3-minute limit //
/////////////////////////////////
output = runProcessStep(new RunBackendStepInput().withValues(Map.of("minutesOldLimit", 3)));
/////////////////////////////////////////////////////////////////
// assert that one updated too, and all are now pending-insert //
/////////////////////////////////////////////////////////////////
assertEquals(1, output.getValueInteger("totalRecordsUpdated"));
assertThat(queryAllRecords())
.allMatch(r -> AutomationStatus.PENDING_INSERT_AUTOMATIONS.getId().equals(getAutomationStatus(r)));
}
/******************************************************************************* /*******************************************************************************
** **
*******************************************************************************/ *******************************************************************************/