diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/audits/AuditAction.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/audits/AuditAction.java index b8b3d4e0..540d0611 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/audits/AuditAction.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/audits/AuditAction.java @@ -29,6 +29,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.Optional; import com.kingsrook.qqq.backend.core.actions.AbstractQActionFunction; import com.kingsrook.qqq.backend.core.actions.tables.GetAction; import com.kingsrook.qqq.backend.core.actions.tables.InsertAction; @@ -49,6 +50,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.model.session.QUser; import com.kingsrook.qqq.backend.core.utils.CollectionUtils; import com.kingsrook.qqq.backend.core.utils.Pair; +import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; /******************************************************************************* @@ -108,12 +110,26 @@ public class AuditAction extends AbstractQActionFunction securityKeyValues, String message) + public static void appendToInput(AuditInput auditInput, String tableName, Integer recordId, Map securityKeyValues, String message) { - return (appendToInput(auditInput, tableName, recordId, securityKeyValues, message, null)); + appendToInput(auditInput, tableName, recordId, securityKeyValues, message, null); } @@ -139,6 +155,44 @@ public class AuditAction extends AbstractQActionFunction getRecordSecurityKeyValues(QTableMetaData table, QRecord record, Optional oldRecord) + { + Map securityKeyValues = new HashMap<>(); + for(RecordSecurityLock recordSecurityLock : RecordSecurityLockFilters.filterForReadLocks(CollectionUtils.nonNullList(table.getRecordSecurityLocks()))) + { + Serializable keyValue = record == null ? null : record.getValue(recordSecurityLock.getFieldName()); + + if(keyValue == null && oldRecord.isPresent()) + { + LOG.debug("Table with a securityLock, but value not found in field", logPair("table", table.getName()), logPair("field", recordSecurityLock.getFieldName())); + keyValue = oldRecord.get().getValue(recordSecurityLock.getFieldName()); + } + + if(keyValue == null) + { + LOG.debug("Table with a securityLock, but value not found in field", logPair("table", table.getName()), logPair("field", recordSecurityLock.getFieldName()), logPair("oldRecordIsPresent", oldRecord.isPresent())); + } + + securityKeyValues.put(recordSecurityLock.getSecurityKeyType(), keyValue); + } + return securityKeyValues; + } + + + /******************************************************************************* ** *******************************************************************************/ diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/audits/DMLAuditAction.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/audits/DMLAuditAction.java index f1ce3c83..18484fcd 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/audits/DMLAuditAction.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/audits/DMLAuditAction.java @@ -23,8 +23,10 @@ package com.kingsrook.qqq.backend.core.actions.audits; import java.io.Serializable; +import java.math.BigDecimal; import java.time.Instant; import java.time.ZoneId; +import java.time.temporal.ChronoUnit; import java.util.ArrayList; import java.util.Comparator; import java.util.HashMap; @@ -52,13 +54,12 @@ import com.kingsrook.qqq.backend.core.model.metadata.audits.AuditLevel; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; -import com.kingsrook.qqq.backend.core.model.metadata.security.RecordSecurityLock; -import com.kingsrook.qqq.backend.core.model.metadata.security.RecordSecurityLockFilters; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.model.session.QSession; import com.kingsrook.qqq.backend.core.utils.CollectionUtils; import com.kingsrook.qqq.backend.core.utils.StringUtils; import com.kingsrook.qqq.backend.core.utils.ValueUtils; +import static com.kingsrook.qqq.backend.core.actions.audits.AuditAction.getRecordSecurityKeyValues; import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; @@ -70,6 +71,8 @@ public class DMLAuditAction extends AbstractQActionFunction actionInput = QContext.getFirstActionInStack(); - if(actionInput.isPresent() && actionInput.get() instanceof RunProcessInput runProcessInput) - { - String processName = runProcessInput.getProcessName(); - QProcessMetaData process = QContext.getQInstance().getProcess(processName); - if(process != null) - { - contextSuffix = " during process: " + process.getLabel(); - } - } - - QSession qSession = QContext.getQSession(); - String apiVersion = qSession.getValue("apiVersion"); - if(apiVersion != null) - { - String apiLabel = qSession.getValue("apiLabel"); - if(!StringUtils.hasContent(apiLabel)) - { - apiLabel = "API"; - } - contextSuffix += (" via " + apiLabel + " Version: " + apiVersion); - } + String contextSuffix = getContentSuffix(input); AuditInput auditInput = new AuditInput(); if(auditLevel.equals(AuditLevel.RECORD) || (auditLevel.equals(AuditLevel.FIELD) && !dmlType.supportsFields)) @@ -137,7 +113,7 @@ public class DMLAuditAction extends AbstractQActionFunction details = new ArrayList<>(); for(String fieldName : sortedFieldNames) { - if(!record.getValues().containsKey(fieldName)) - { - //////////////////////////////////////////////////////////////////////////////////////////////// - // if the stored record doesn't have this field name, then don't audit anything about it // - // this is to deal with our Patch style updates not looking like every field was cleared out. // - //////////////////////////////////////////////////////////////////////////////////////////////// - continue; - } - - if(fieldName.equals("modifyDate") || fieldName.equals("createDate") || fieldName.equals("automationStatus")) - { - continue; - } - - QFieldMetaData field = table.getField(fieldName); - Serializable value = ValueUtils.getValueAsFieldType(field.getType(), record.getValue(fieldName)); - Serializable oldValue = oldRecord == null ? null : ValueUtils.getValueAsFieldType(field.getType(), oldRecord.getValue(fieldName)); - QRecord detailRecord = null; - - if(oldRecord == null) - { - if(DMLType.INSERT.equals(dmlType) && value == null) - { - continue; - } - - if(field.getType().equals(QFieldType.BLOB) || field.getType().needsMasked()) - { - detailRecord = new QRecord().withValue("message", "Set " + field.getLabel()); - } - else - { - String formattedValue = getFormattedValueForAuditDetail(record, fieldName, field, value); - detailRecord = new QRecord().withValue("message", "Set " + field.getLabel() + " to " + formattedValue); - detailRecord.withValue("newValue", formattedValue); - } - } - else - { - if(!Objects.equals(oldValue, value)) - { - if(field.getType().equals(QFieldType.BLOB) || field.getType().needsMasked()) - { - if(oldValue == null) - { - detailRecord = new QRecord().withValue("message", "Set " + field.getLabel()); - } - else if(value == null) - { - detailRecord = new QRecord().withValue("message", "Removed " + field.getLabel()); - } - else - { - detailRecord = new QRecord().withValue("message", "Changed " + field.getLabel()); - } - } - else - { - String formattedValue = getFormattedValueForAuditDetail(record, fieldName, field, value); - String formattedOldValue = getFormattedValueForAuditDetail(oldRecord, fieldName, field, oldValue); - - if(oldValue == null) - { - detailRecord = new QRecord().withValue("message", "Set " + field.getLabel() + " to " + formatFormattedValueForDetailMessage(field, formattedValue)); - detailRecord.withValue("newValue", formattedValue); - } - else if(value == null) - { - detailRecord = new QRecord().withValue("message", "Removed " + formatFormattedValueForDetailMessage(field, formattedOldValue) + " from " + field.getLabel()); - detailRecord.withValue("oldValue", formattedOldValue); - } - else - { - detailRecord = new QRecord().withValue("message", "Changed " + field.getLabel() + " from " + formatFormattedValueForDetailMessage(field, formattedOldValue) + " to " + formatFormattedValueForDetailMessage(field, formattedValue)); - detailRecord.withValue("oldValue", formattedOldValue); - detailRecord.withValue("newValue", formattedValue); - } - } - } - } - - if(detailRecord != null) - { - detailRecord.withValue("fieldName", fieldName); - details.add(detailRecord); - } + makeAuditDetailRecordForField(fieldName, table, dmlType, record, oldRecord) + .ifPresent(details::add); } if(details.isEmpty() && DMLType.UPDATE.equals(dmlType)) @@ -264,7 +156,7 @@ public class DMLAuditAction extends AbstractQActionFunction actionInput = QContext.getFirstActionInStack(); + if(actionInput.isPresent() && actionInput.get() instanceof RunProcessInput runProcessInput) + { + String processAuditContext = ValueUtils.getValueAsString(runProcessInput.getValue(AUDIT_CONTEXT_FIELD_NAME)); + if(StringUtils.hasContent(processAuditContext)) + { + contextSuffix.append(" ").append(processAuditContext); + } + + String processName = runProcessInput.getProcessName(); + QProcessMetaData process = QContext.getQInstance().getProcess(processName); + if(process != null) + { + contextSuffix.append(" during process: ").append(process.getLabel()); + } + } + + /////////////////////////////////////////////////// + // use api label & version if present in session // + /////////////////////////////////////////////////// + QSession qSession = QContext.getQSession(); + String apiVersion = qSession.getValue("apiVersion"); + if(apiVersion != null) + { + String apiLabel = qSession.getValue("apiLabel"); + if(!StringUtils.hasContent(apiLabel)) + { + apiLabel = "API"; + } + contextSuffix.append(" via ").append(apiLabel).append(" Version: ").append(apiVersion); + } + return (contextSuffix.toString()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + static Optional makeAuditDetailRecordForField(String fieldName, QTableMetaData table, DMLType dmlType, QRecord record, QRecord oldRecord) + { + if(!record.getValues().containsKey(fieldName)) + { + //////////////////////////////////////////////////////////////////////////////////////////////// + // if the stored record doesn't have this field name, then don't audit anything about it // + // this is to deal with our Patch style updates not looking like every field was cleared out. // + //////////////////////////////////////////////////////////////////////////////////////////////// + return (Optional.empty()); + } + + if(fieldName.equals("modifyDate") || fieldName.equals("createDate") || fieldName.equals("automationStatus")) + { + return (Optional.empty()); + } + + QFieldMetaData field = table.getField(fieldName); + Serializable value = ValueUtils.getValueAsFieldType(field.getType(), record.getValue(fieldName)); + Serializable oldValue = oldRecord == null ? null : ValueUtils.getValueAsFieldType(field.getType(), oldRecord.getValue(fieldName)); + QRecord detailRecord = null; + + if(oldRecord == null) + { + if(DMLType.INSERT.equals(dmlType) && value == null) + { + return (Optional.empty()); + } + + if(field.getType().equals(QFieldType.BLOB) || field.getType().needsMasked()) + { + detailRecord = new QRecord().withValue("message", "Set " + field.getLabel()); + } + else + { + String formattedValue = getFormattedValueForAuditDetail(record, fieldName, field, value); + detailRecord = new QRecord().withValue("message", "Set " + field.getLabel() + " to " + formattedValue); + detailRecord.withValue("newValue", formattedValue); + } + } + else + { + if(areValuesDifferentForAudit(field, value, oldValue)) + { + if(field.getType().equals(QFieldType.BLOB) || field.getType().needsMasked()) + { + if(oldValue == null) + { + detailRecord = new QRecord().withValue("message", "Set " + field.getLabel()); + } + else if(value == null) + { + detailRecord = new QRecord().withValue("message", "Removed " + field.getLabel()); + } + else + { + detailRecord = new QRecord().withValue("message", "Changed " + field.getLabel()); + } + } + else + { + String formattedValue = getFormattedValueForAuditDetail(record, fieldName, field, value); + String formattedOldValue = getFormattedValueForAuditDetail(oldRecord, fieldName, field, oldValue); + + if(oldValue == null) + { + detailRecord = new QRecord().withValue("message", "Set " + field.getLabel() + " to " + formatFormattedValueForDetailMessage(field, formattedValue)); + detailRecord.withValue("newValue", formattedValue); + } + else if(value == null) + { + detailRecord = new QRecord().withValue("message", "Removed " + formatFormattedValueForDetailMessage(field, formattedOldValue) + " from " + field.getLabel()); + detailRecord.withValue("oldValue", formattedOldValue); + } + else + { + detailRecord = new QRecord().withValue("message", "Changed " + field.getLabel() + " from " + formatFormattedValueForDetailMessage(field, formattedOldValue) + " to " + formatFormattedValueForDetailMessage(field, formattedValue)); + detailRecord.withValue("oldValue", formattedOldValue); + detailRecord.withValue("newValue", formattedValue); + } + } + } + } + + if(detailRecord != null) + { + LOG.debug("Returning with message: " + detailRecord.getValueString("message")); + + detailRecord.withValue("fieldName", fieldName); + return (Optional.of(detailRecord)); + } + + return (Optional.empty()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + static boolean areValuesDifferentForAudit(QFieldMetaData field, Serializable value, Serializable oldValue) + { + try + { + /////////////////// + // decimal rules // + /////////////////// + if(field.getType().equals(QFieldType.DECIMAL)) + { + BigDecimal newBD = ValueUtils.getValueAsBigDecimal(value); + BigDecimal oldBD = ValueUtils.getValueAsBigDecimal(oldValue); + + if(newBD == null && oldBD == null) + { + return (false); + } + + if(newBD == null || oldBD == null) + { + return (true); + } + + return (newBD.compareTo(oldBD) != 0); + } + + //////////////////// + // dateTime rules // + //////////////////// + if(field.getType().equals(QFieldType.DATE_TIME)) + { + Instant newI = ValueUtils.getValueAsInstant(value); + Instant oldI = ValueUtils.getValueAsInstant(oldValue); + + if(newI == null && oldI == null) + { + return (false); + } + + if(newI == null || oldI == null) + { + return (true); + } + + //////////////////////////////// + // just compare to the second // + //////////////////////////////// + return (newI.truncatedTo(ChronoUnit.SECONDS).compareTo(oldI.truncatedTo(ChronoUnit.SECONDS)) != 0); + } + + ////////////////// + // string rules // + ////////////////// + if(field.getType().isStringLike()) + { + String newString = ValueUtils.getValueAsString(value); + String oldString = ValueUtils.getValueAsString(oldValue); + + boolean newIsNullOrEmpty = !StringUtils.hasContent(newString); + boolean oldIsNullOrEmpty = !StringUtils.hasContent(oldString); + + if(newIsNullOrEmpty && oldIsNullOrEmpty) + { + return (false); + } + + if(newIsNullOrEmpty || oldIsNullOrEmpty) + { + return (true); + } + + return (newString.compareTo(oldString) != 0); + } + + ///////////////////////////////////// + // default just use Objects.equals // + ///////////////////////////////////// + return !Objects.equals(oldValue, value); + } + catch(Exception e) + { + LOG.debug("Error checking areValuesDifferentForAudit", e, logPair("fieldName", field.getName()), logPair("value", value), logPair("oldValue", oldValue)); + } + + //////////////////////////////////// + // default to something simple... // + //////////////////////////////////// + return !Objects.equals(oldValue, value); + } + + + /******************************************************************************* ** *******************************************************************************/ @@ -373,21 +513,6 @@ public class DMLAuditAction extends AbstractQActionFunction getRecordSecurityKeyValues(QTableMetaData table, QRecord record) - { - Map securityKeyValues = new HashMap<>(); - for(RecordSecurityLock recordSecurityLock : RecordSecurityLockFilters.filterForReadLocks(CollectionUtils.nonNullList(table.getRecordSecurityLocks()))) - { - securityKeyValues.put(recordSecurityLock.getSecurityKeyType(), record == null ? null : record.getValue(recordSecurityLock.getFieldName())); - } - return securityKeyValues; - } - - - /******************************************************************************* ** *******************************************************************************/ @@ -407,7 +532,7 @@ public class DMLAuditAction extends AbstractQActionFunction tableActions = Objects.requireNonNullElse(table.getAutomationDetails().getActions(), new ArrayList<>()); + List tableActions = Collections.emptyList(); + if(table.getAutomationDetails() != null && table.getAutomationDetails().getActions() != null) + { + tableActions = table.getAutomationDetails().getActions(); + } if(automationStatus.equals(AutomationStatus.PENDING_INSERT_AUTOMATIONS)) { @@ -135,6 +142,12 @@ public class RecordAutomationStatusUpdater { return (false); } + + //////////////////////////////////////////////////////////////////////////////////////// + // if we're going to pending-insert, and there are no insert automations or triggers, // + // then we may skip pending and go to okay. // + //////////////////////////////////////////////////////////////////////////////////////// + return (true); } else if(automationStatus.equals(AutomationStatus.PENDING_UPDATE_AUTOMATIONS)) { @@ -146,9 +159,21 @@ public class RecordAutomationStatusUpdater { return (false); } - } - return (true); + //////////////////////////////////////////////////////////////////////////////////////// + // if we're going to pending-update, and there are no insert automations or triggers, // + // then we may skip pending and go to okay. // + //////////////////////////////////////////////////////////////////////////////////////// + return (true); + } + else + { + /////////////////////////////////////////////////////////////////////////////////////////////////////// + // if we're going to any other automation status - then we may never "skip pending" and go to okay - // + // because we weren't asked to go to pending! // + /////////////////////////////////////////////////////////////////////////////////////////////////////// + return (false); + } } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/automation/polling/PollingAutomationPerTableRunner.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/automation/polling/PollingAutomationPerTableRunner.java index 52cc7646..8d37403e 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/automation/polling/PollingAutomationPerTableRunner.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/automation/polling/PollingAutomationPerTableRunner.java @@ -342,22 +342,9 @@ public class PollingAutomationPerTableRunner implements Runnable boolean anyActionsFailed = false; for(TableAutomationAction action : actions) { - try + boolean hadError = applyActionToRecords(table, records, action); + if(hadError) { - ////////////////////////////////////////////////////////////////////////////////////////////////////////////// - // note - this method - will re-query the objects, so we should have confidence that their data is fresh... // - ////////////////////////////////////////////////////////////////////////////////////////////////////////////// - List matchingQRecords = getRecordsMatchingActionFilter(table, records, action); - LOG.debug("Of the {} records that were pending automations, {} of them match the filter on the action {}", records.size(), matchingQRecords.size(), action); - if(CollectionUtils.nullSafeHasContents(matchingQRecords)) - { - LOG.debug(" Processing " + matchingQRecords.size() + " records in " + table + " for action " + action); - applyActionToMatchingRecords(table, matchingQRecords, action); - } - } - catch(Exception e) - { - LOG.warn("Caught exception processing records on " + table + " for action " + action, e); anyActionsFailed = true; } } @@ -377,6 +364,37 @@ public class PollingAutomationPerTableRunner implements Runnable + /******************************************************************************* + ** Run one action over a list of records (if they match the action's filter). + ** + ** @return hadError - true if an exception was caught; false if all OK. + *******************************************************************************/ + protected boolean applyActionToRecords(QTableMetaData table, List records, TableAutomationAction action) + { + try + { + ////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // note - this method - will re-query the objects, so we should have confidence that their data is fresh... // + ////////////////////////////////////////////////////////////////////////////////////////////////////////////// + List matchingQRecords = getRecordsMatchingActionFilter(table, records, action); + LOG.debug("Of the {} records that were pending automations, {} of them match the filter on the action {}", records.size(), matchingQRecords.size(), action); + if(CollectionUtils.nullSafeHasContents(matchingQRecords)) + { + LOG.debug(" Processing " + matchingQRecords.size() + " records in " + table + " for action " + action); + applyActionToMatchingRecords(table, matchingQRecords, action); + } + + return (false); + } + catch(Exception e) + { + LOG.warn("Caught exception processing records on " + table + " for action " + action, e); + return (true); + } + } + + + /******************************************************************************* ** For a given action, and a list of records - return a new list, of the ones ** which match the action's filter (if there is one - if not, then all match). diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/scripts/ScriptsMetaDataProvider.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/scripts/ScriptsMetaDataProvider.java index bd7f3231..b7282874 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/scripts/ScriptsMetaDataProvider.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/scripts/ScriptsMetaDataProvider.java @@ -383,7 +383,7 @@ public class ScriptsMetaDataProvider /******************************************************************************* ** *******************************************************************************/ - private QTableMetaData defineTableTriggerTable(String backendName) throws QException + public QTableMetaData defineTableTriggerTable(String backendName) throws QException { QTableMetaData tableMetaData = defineStandardTable(backendName, TableTrigger.TABLE_NAME, TableTrigger.class) .withRecordLabelFields("id") diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/scripts/RunRecordScriptLoadStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/scripts/RunRecordScriptLoadStep.java index 8a036f52..78c015a0 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/scripts/RunRecordScriptLoadStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/scripts/RunRecordScriptLoadStep.java @@ -25,11 +25,15 @@ package com.kingsrook.qqq.backend.core.processes.implementations.scripts; import java.io.Serializable; import java.util.ArrayList; import java.util.List; +import com.kingsrook.qqq.backend.core.actions.audits.AuditAction; +import com.kingsrook.qqq.backend.core.actions.audits.DMLAuditAction; import com.kingsrook.qqq.backend.core.actions.scripts.RunAdHocRecordScriptAction; import com.kingsrook.qqq.backend.core.actions.scripts.logging.StoreScriptLogAndScriptLogLineExecutionLogger; import com.kingsrook.qqq.backend.core.actions.tables.GetAction; +import com.kingsrook.qqq.backend.core.context.QContext; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.logging.QLogger; +import com.kingsrook.qqq.backend.core.model.actions.audits.AuditInput; import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryFilterLink; import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLine; import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLineInterface; @@ -42,7 +46,9 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetInput; import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetOutput; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; +import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.metadata.code.AdHocScriptCodeReference; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.model.scripts.Script; import com.kingsrook.qqq.backend.core.model.scripts.ScriptLog; import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.AbstractLoadStep; @@ -121,16 +127,25 @@ public class RunRecordScriptLoadStep extends AbstractLoadStep implements Process getInput.setTableName(Script.TABLE_NAME); getInput.setPrimaryKey(scriptId); GetOutput getOutput = new GetAction().execute(getInput); - if(getOutput.getRecord() == null) + QRecord script = getOutput.getRecord(); + if(script == null) { throw (new QException("Could not find script by id: " + scriptId)); } + ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // set an "audit context" - so any DML executed during the script will include the note of what script was running. // + ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + runBackendStepInput.addValue(DMLAuditAction.AUDIT_CONTEXT_FIELD_NAME, "via Script \"" + script.getValue("name") + "\""); + + String tableName = script.getValueString("tableName"); + RunAdHocRecordScriptInput input = new RunAdHocRecordScriptInput(); input.setRecordList(runBackendStepInput.getRecords()); input.setCodeReference(new AdHocScriptCodeReference().withScriptId(scriptId)); - input.setTableName(getOutput.getRecord().getValueString("tableName")); + input.setTableName(tableName); input.setLogger(scriptLogger); + RunAdHocRecordScriptOutput output = new RunAdHocRecordScriptOutput(); Exception caughtException = null; try @@ -147,11 +162,17 @@ public class RunRecordScriptLoadStep extends AbstractLoadStep implements Process caughtException = e; } + String auditMessage = "Script \"" + script.getValueString("name") + "\" (id: " + scriptId + ") was executed against this record"; + + ////////////////////////////////////////////////////////// + // add the record to the appropriate processSummaryLine // + ////////////////////////////////////////////////////////// if(scriptLogger.getScriptLog() != null) { Integer id = scriptLogger.getScriptLog().getValueInteger("id"); if(id != null) { + auditMessage += ", creating script log: " + id; boolean hadError = BooleanUtils.isTrue(scriptLogger.getScriptLog().getValueBoolean("hadError")); (hadError ? errorScriptLogIds : okScriptLogIds).add(id); } @@ -160,6 +181,34 @@ public class RunRecordScriptLoadStep extends AbstractLoadStep implements Process { unloggedExceptionLine.incrementCount(runBackendStepInput.getRecords().size()); } + + //////////////////////////////////////////////////////////// + // audit that the script was executed against the records // + //////////////////////////////////////////////////////////// + audit(runBackendStepInput, tableName, auditMessage); + } + + + + /******************************************************************************* + ** for each input record, add an audit stating that the script was executed. + *******************************************************************************/ + private static void audit(RunBackendStepInput runBackendStepInput, String tableName, String auditMessage) + { + try + { + QTableMetaData table = QContext.getQInstance().getTable(tableName); + AuditInput auditInput = new AuditInput(); + for(QRecord record : runBackendStepInput.getRecords()) + { + AuditAction.appendToInput(auditInput, table, record, auditMessage); + } + new AuditAction().execute(auditInput); + } + catch(Exception e) + { + LOG.warn("Error recording audits after running record script", e, logPair("tableName", tableName), logPair("auditMessage", auditMessage)); + } } diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/BaseTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/BaseTest.java index 1cae9018..27513ccb 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/BaseTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/BaseTest.java @@ -48,6 +48,8 @@ public class BaseTest @BeforeEach void baseBeforeEach() { + System.setProperty("qqq.logger.logSessionId.disabled", "true"); + QContext.init(TestUtils.defineInstance(), new QSession() .withUser(new QUser() .withIdReference("001") diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/audits/AuditActionTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/audits/AuditActionTest.java index 40b05461..40b60d9d 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/audits/AuditActionTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/audits/AuditActionTest.java @@ -31,6 +31,7 @@ import com.kingsrook.qqq.backend.core.BaseTest; import com.kingsrook.qqq.backend.core.context.QContext; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.actions.audits.AuditInput; +import com.kingsrook.qqq.backend.core.model.actions.audits.AuditSingleInput; import com.kingsrook.qqq.backend.core.model.audits.AuditsMetaDataProvider; import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; @@ -38,6 +39,7 @@ import com.kingsrook.qqq.backend.core.model.session.QSession; import com.kingsrook.qqq.backend.core.model.session.QUser; import com.kingsrook.qqq.backend.core.processes.utils.GeneralProcessUtils; import com.kingsrook.qqq.backend.core.utils.TestUtils; +import com.kingsrook.qqq.backend.core.utils.collections.MapBuilder; import org.junit.jupiter.api.Test; import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -197,4 +199,31 @@ class AuditActionTest extends BaseTest assertThat(auditDetails).anyMatch(r -> r.getValueString("message").equals("Detail3")); } + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testAppendToInputThatTakesRecordNotIdAndSecurityKeyValues() + { + AuditInput auditInput = new AuditInput(); + + ////////////////////////////////////////////////////////////// + // make sure the recordId & securityKey got build correctly // + ////////////////////////////////////////////////////////////// + AuditAction.appendToInput(auditInput, QContext.getQInstance().getTable(TestUtils.TABLE_NAME_ORDER), new QRecord().withValue("id", 47).withValue("storeId", 42), "Test"); + AuditSingleInput auditSingleInput = auditInput.getAuditSingleInputList().get(0); + assertEquals(47, auditSingleInput.getRecordId()); + assertEquals(MapBuilder.of(TestUtils.SECURITY_KEY_TYPE_STORE, 42), auditSingleInput.getSecurityKeyValues()); + + /////////////////////////////////////////////////////////////////////////////////////////// + // acknowledge that we might get back a null key value if the record doesn't have it set // + /////////////////////////////////////////////////////////////////////////////////////////// + AuditAction.appendToInput(auditInput, QContext.getQInstance().getTable(TestUtils.TABLE_NAME_ORDER), new QRecord().withValue("id", 47), "Test"); + auditSingleInput = auditInput.getAuditSingleInputList().get(1); + assertEquals(47, auditSingleInput.getRecordId()); + assertEquals(MapBuilder.of(TestUtils.SECURITY_KEY_TYPE_STORE, null), auditSingleInput.getSecurityKeyValues()); + } + } diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/audits/DMLAuditActionTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/audits/DMLAuditActionTest.java index 9d8f8b47..58d1f6d4 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/audits/DMLAuditActionTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/audits/DMLAuditActionTest.java @@ -22,12 +22,14 @@ package com.kingsrook.qqq.backend.core.actions.audits; +import java.math.BigDecimal; import java.time.Instant; import java.util.List; import com.kingsrook.qqq.backend.core.BaseTest; import com.kingsrook.qqq.backend.core.context.QContext; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.actions.audits.DMLAuditInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput; import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput; import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput; @@ -36,10 +38,18 @@ import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.audits.AuditLevel; import com.kingsrook.qqq.backend.core.model.metadata.audits.QAuditRules; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.model.session.QSession; import com.kingsrook.qqq.backend.core.model.statusmessages.BadInputStatusMessage; import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryRecordStore; import com.kingsrook.qqq.backend.core.utils.TestUtils; +import com.kingsrook.qqq.backend.core.utils.ValueUtils; import org.junit.jupiter.api.Test; +import static com.kingsrook.qqq.backend.core.actions.audits.DMLAuditAction.DMLType.INSERT; +import static com.kingsrook.qqq.backend.core.actions.audits.DMLAuditAction.DMLType.UPDATE; +import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNull; @@ -200,4 +210,194 @@ class DMLAuditActionTest extends BaseTest } } -} \ No newline at end of file + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testMakeAuditDetailRecordForField() + { + QTableMetaData table = new QTableMetaData() + .withField(new QFieldMetaData("createDate", QFieldType.DATE_TIME).withLabel("Create Date")) + .withField(new QFieldMetaData("modifyDate", QFieldType.DATE_TIME).withLabel("Modify Date")) + .withField(new QFieldMetaData("someTimestamp", QFieldType.DATE_TIME).withLabel("Some Timestamp")) + .withField(new QFieldMetaData("name", QFieldType.STRING).withLabel("Name")) + .withField(new QFieldMetaData("seqNo", QFieldType.INTEGER).withLabel("Sequence No.")) + .withField(new QFieldMetaData("price", QFieldType.DECIMAL).withLabel("Price")); + + /////////////////////////////// + // create date - never audit // + /////////////////////////////// + assertThat(DMLAuditAction.makeAuditDetailRecordForField("createDate", table, INSERT, + new QRecord().withValue("createDate", Instant.now()), + new QRecord().withValue("createDate", Instant.now().minusSeconds(100)))) + .isEmpty(); + + /////////////////////////////// + // modify date - never audit // + /////////////////////////////// + assertThat(DMLAuditAction.makeAuditDetailRecordForField("modifyDate", table, UPDATE, + new QRecord().withValue("modifyDate", Instant.now()), + new QRecord().withValue("modifyDate", Instant.now().minusSeconds(100)))) + .isEmpty(); + + //////////////////////////////////////////////////////// + // datetime different only in precision - don't audit // + //////////////////////////////////////////////////////// + assertThat(DMLAuditAction.makeAuditDetailRecordForField("someTimestamp", table, UPDATE, + new QRecord().withValue("someTimestamp", ValueUtils.getValueAsInstant("2023-04-17T14:33:08.777")), + new QRecord().withValue("someTimestamp", Instant.parse("2023-04-17T14:33:08Z")))) + .isEmpty(); + + ///////////////////////////////////////////// + // datetime actually different - audit it. // + ///////////////////////////////////////////// + assertThat(DMLAuditAction.makeAuditDetailRecordForField("someTimestamp", table, UPDATE, + new QRecord().withValue("someTimestamp", Instant.parse("2023-04-17T14:33:09Z")), + new QRecord().withValue("someTimestamp", Instant.parse("2023-04-17T14:33:08Z")))) + .isPresent() + .get().extracting(r -> r.getValueString("message")) + .matches(s -> s.matches("Changed Some Timestamp from 2023.* to 2023.*")); + + //////////////////////////////////////////////// + // datetime changing null to not null - audit // + //////////////////////////////////////////////// + assertThat(DMLAuditAction.makeAuditDetailRecordForField("someTimestamp", table, UPDATE, + new QRecord().withValue("someTimestamp", ValueUtils.getValueAsInstant("2023-04-17T14:33:08.777")), + new QRecord().withValue("someTimestamp", null))) + .isPresent() + .get().extracting(r -> r.getValueString("message")) + .matches(s -> s.matches("Set Some Timestamp to 2023.*")); + + //////////////////////////////////////////////// + // datetime changing not null to null - audit // + //////////////////////////////////////////////// + assertThat(DMLAuditAction.makeAuditDetailRecordForField("someTimestamp", table, UPDATE, + new QRecord().withValue("someTimestamp", null), + new QRecord().withValue("someTimestamp", Instant.parse("2023-04-17T14:33:08Z")))) + .isPresent() + .get().extracting(r -> r.getValueString("message")) + .matches(s -> s.matches("Removed 2023.*from Some Timestamp")); + + //////////////////////////////////////// + // string that is the same - no audit // + //////////////////////////////////////// + assertThat(DMLAuditAction.makeAuditDetailRecordForField("name", table, UPDATE, + new QRecord().withValue("name", "Homer"), + new QRecord().withValue("name", "Homer"))) + .isEmpty(); + + ////////////////////////////////////////// + // string from null to empty - no audit // + ////////////////////////////////////////// + assertThat(DMLAuditAction.makeAuditDetailRecordForField("name", table, UPDATE, + new QRecord().withValue("name", null), + new QRecord().withValue("name", ""))) + .isEmpty(); + + ////////////////////////////////////////// + // string from empty to null - no audit // + ////////////////////////////////////////// + assertThat(DMLAuditAction.makeAuditDetailRecordForField("name", table, UPDATE, + new QRecord().withValue("name", ""), + new QRecord().withValue("name", null))) + .isEmpty(); + + ////////////////////////////////////////////////////////// + // decimal that only changes in precision - don't audit // + ////////////////////////////////////////////////////////// + assertThat(DMLAuditAction.makeAuditDetailRecordForField("price", table, UPDATE, + new QRecord().withValue("price", "10"), + new QRecord().withValue("price", new BigDecimal("10.00")))) + .isEmpty(); + + ////////////////////////////////////////////////// + // decimal that's actually different - do audit // + ////////////////////////////////////////////////// + assertThat(DMLAuditAction.makeAuditDetailRecordForField("price", table, UPDATE, + new QRecord().withValue("price", "10.01"), + new QRecord().withValue("price", new BigDecimal("10.00")))) + .isPresent() + .get().extracting(r -> r.getValueString("message")) + .matches(s -> s.matches("Changed Price from 10.00 to 10.01")); + + /////////////////////////////////////// + // decimal null, input "" - no audit // + /////////////////////////////////////// + assertThat(DMLAuditAction.makeAuditDetailRecordForField("price", table, UPDATE, + new QRecord().withValue("price", ""), + new QRecord().withValue("price", null))) + .isEmpty(); + + ///////////////////////////////////////// + // decimal not-null to null - do audit // + ///////////////////////////////////////// + assertThat(DMLAuditAction.makeAuditDetailRecordForField("price", table, UPDATE, + new QRecord().withValue("price", BigDecimal.ONE), + new QRecord().withValue("price", null))) + .isPresent() + .get().extracting(r -> r.getValueString("message")) + .matches(s -> s.matches("Set Price to 1")); + + ///////////////////////////////////////// + // decimal null to not-null - do audit // + ///////////////////////////////////////// + assertThat(DMLAuditAction.makeAuditDetailRecordForField("price", table, UPDATE, + new QRecord().withValue("price", null), + new QRecord().withValue("price", BigDecimal.ONE))) + .isPresent() + .get().extracting(r -> r.getValueString("message")) + .matches(s -> s.matches("Removed 1 from Price")); + + /////////////////////////////////////// + // integer null, input "" - no audit // + /////////////////////////////////////// + assertThat(DMLAuditAction.makeAuditDetailRecordForField("seqNo", table, UPDATE, + new QRecord().withValue("seqNo", ""), + new QRecord().withValue("seqNo", null))) + .isEmpty(); + + //////////////////////////////// + // integer changed - do audit // + //////////////////////////////// + assertThat(DMLAuditAction.makeAuditDetailRecordForField("seqNo", table, UPDATE, + new QRecord().withValue("seqNo", 2), + new QRecord().withValue("seqNo", 1))) + .isPresent() + .get().extracting(r -> r.getValueString("message")) + .matches(s -> s.matches("Changed Sequence No. from 1 to 2")); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testGetContextSuffix() + { + assertEquals("", DMLAuditAction.getContentSuffix(new DMLAuditInput())); + assertEquals(" while shipping an order", DMLAuditAction.getContentSuffix(new DMLAuditInput().withAuditContext("while shipping an order"))); + + QContext.pushAction(new RunProcessInput().withValue(DMLAuditAction.AUDIT_CONTEXT_FIELD_NAME, "via Script \"My Script\"")); + assertEquals(" via Script \"My Script\"", DMLAuditAction.getContentSuffix(new DMLAuditInput())); + QContext.popAction(); + + QContext.pushAction(new RunProcessInput().withProcessName(TestUtils.PROCESS_NAME_GREET_PEOPLE)); + assertEquals(" during process: Greet", DMLAuditAction.getContentSuffix(new DMLAuditInput())); + QContext.popAction(); + + QContext.setQSession(new QSession().withValue("apiVersion", "1.0")); + assertEquals(" via API Version: 1.0", DMLAuditAction.getContentSuffix(new DMLAuditInput())); + + QContext.setQSession(new QSession().withValue("apiVersion", "20230921").withValue("apiLabel", "Our Public API")); + assertEquals(" via Our Public API Version: 20230921", DMLAuditAction.getContentSuffix(new DMLAuditInput())); + + QContext.pushAction(new RunProcessInput().withProcessName(TestUtils.PROCESS_NAME_GREET_PEOPLE).withValue(DMLAuditAction.AUDIT_CONTEXT_FIELD_NAME, "via Script \"My Script\"")); + QContext.setQSession(new QSession().withValue("apiVersion", "20230921").withValue("apiLabel", "Our Public API")); + assertEquals(" while shipping an order via Script \"My Script\" during process: Greet via Our Public API Version: 20230921", DMLAuditAction.getContentSuffix(new DMLAuditInput().withAuditContext("while shipping an order"))); + QContext.popAction(); + } + +} diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/automation/RecordAutomationStatusUpdaterTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/automation/RecordAutomationStatusUpdaterTest.java new file mode 100644 index 00000000..81a1a34f --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/automation/RecordAutomationStatusUpdaterTest.java @@ -0,0 +1,128 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2023. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.actions.automation; + + +import java.util.List; +import com.kingsrook.qqq.backend.core.BaseTest; +import com.kingsrook.qqq.backend.core.actions.tables.InsertAction; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; +import com.kingsrook.qqq.backend.core.model.automation.TableTrigger; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.QTableAutomationDetails; +import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.TableAutomationAction; +import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.TriggerEvent; +import com.kingsrook.qqq.backend.core.model.scripts.ScriptsMetaDataProvider; +import com.kingsrook.qqq.backend.core.utils.TestUtils; +import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + + +/******************************************************************************* + ** Unit test for RecordAutomationStatusUpdater + *******************************************************************************/ +class RecordAutomationStatusUpdaterTest extends BaseTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testCanWeSkipPendingAndGoToOkay() throws QException + { + QContext.getQInstance() + .addTable(new ScriptsMetaDataProvider().defineTableTriggerTable(TestUtils.MEMORY_BACKEND_NAME)); + + //////////////////////////////////////////////////////////// + // define tables with various automations and/or triggers // + //////////////////////////////////////////////////////////// + QTableMetaData tableWithNoAutomations = new QTableMetaData() + .withName("tableWithNoAutomations"); + + QTableMetaData tableWithInsertAutomation = new QTableMetaData() + .withName("tableWithInsertAutomation") + .withAutomationDetails(new QTableAutomationDetails() + .withAction(new TableAutomationAction().withTriggerEvent(TriggerEvent.POST_INSERT))); + + QTableMetaData tableWithUpdateAutomation = new QTableMetaData() + .withName("tableWithUpdateAutomation") + .withAutomationDetails(new QTableAutomationDetails() + .withAction(new TableAutomationAction() + .withTriggerEvent(TriggerEvent.POST_UPDATE))); + + QTableMetaData tableWithInsertAndUpdateAutomations = new QTableMetaData() + .withName("tableWithInsertAndUpdateAutomations ") + .withAutomationDetails(new QTableAutomationDetails() + .withAction(new TableAutomationAction().withTriggerEvent(TriggerEvent.POST_INSERT)) + .withAction(new TableAutomationAction().withTriggerEvent(TriggerEvent.POST_UPDATE))); + + QTableMetaData tableWithInsertTrigger = new QTableMetaData() + .withName("tableWithInsertTrigger"); + new InsertAction().execute(new InsertInput(TableTrigger.TABLE_NAME) + .withRecordEntity(new TableTrigger().withTableName(tableWithInsertTrigger.getName()).withPostInsert(true).withPostUpdate(false))); + + QTableMetaData tableWithUpdateTrigger = new QTableMetaData() + .withName("tableWithUpdateTrigger"); + new InsertAction().execute(new InsertInput(TableTrigger.TABLE_NAME) + .withRecordEntity(new TableTrigger().withTableName(tableWithUpdateTrigger.getName()).withPostInsert(false).withPostUpdate(true))); + + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // tests for going to PENDING_INSERT. // + // we should be allowed to skip and go to OK (return true) if the table does not have insert automations or triggers // + // we should NOT be allowed to skip and go to OK (return false) if the table does NOT have insert automations or triggers // + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + assertTrue(RecordAutomationStatusUpdater.canWeSkipPendingAndGoToOkay(tableWithNoAutomations, AutomationStatus.PENDING_INSERT_AUTOMATIONS)); + assertFalse(RecordAutomationStatusUpdater.canWeSkipPendingAndGoToOkay(tableWithInsertAutomation, AutomationStatus.PENDING_INSERT_AUTOMATIONS)); + assertTrue(RecordAutomationStatusUpdater.canWeSkipPendingAndGoToOkay(tableWithUpdateAutomation, AutomationStatus.PENDING_INSERT_AUTOMATIONS)); + assertFalse(RecordAutomationStatusUpdater.canWeSkipPendingAndGoToOkay(tableWithInsertAndUpdateAutomations, AutomationStatus.PENDING_INSERT_AUTOMATIONS)); + assertFalse(RecordAutomationStatusUpdater.canWeSkipPendingAndGoToOkay(tableWithInsertTrigger, AutomationStatus.PENDING_INSERT_AUTOMATIONS)); + assertTrue(RecordAutomationStatusUpdater.canWeSkipPendingAndGoToOkay(tableWithUpdateTrigger, AutomationStatus.PENDING_INSERT_AUTOMATIONS)); + + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // tests for going to PENDING_UPDATE. // + // we should be allowed to skip and go to OK (return true) if the table does not have update automations or triggers // + // we should NOT be allowed to skip and go to OK (return false) if the table does NOT have insert automations or triggers // + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + assertTrue(RecordAutomationStatusUpdater.canWeSkipPendingAndGoToOkay(tableWithNoAutomations, AutomationStatus.PENDING_UPDATE_AUTOMATIONS)); + assertTrue(RecordAutomationStatusUpdater.canWeSkipPendingAndGoToOkay(tableWithInsertAutomation, AutomationStatus.PENDING_UPDATE_AUTOMATIONS)); + assertFalse(RecordAutomationStatusUpdater.canWeSkipPendingAndGoToOkay(tableWithUpdateAutomation, AutomationStatus.PENDING_UPDATE_AUTOMATIONS)); + assertFalse(RecordAutomationStatusUpdater.canWeSkipPendingAndGoToOkay(tableWithInsertAndUpdateAutomations, AutomationStatus.PENDING_UPDATE_AUTOMATIONS)); + assertTrue(RecordAutomationStatusUpdater.canWeSkipPendingAndGoToOkay(tableWithInsertTrigger, AutomationStatus.PENDING_UPDATE_AUTOMATIONS)); + assertFalse(RecordAutomationStatusUpdater.canWeSkipPendingAndGoToOkay(tableWithUpdateTrigger, AutomationStatus.PENDING_UPDATE_AUTOMATIONS)); + + ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // tests for going to non-PENDING states // + // this function should NEVER return true for skipping pending if the target state (2nd arg) isn't a pending state. // + ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + for(AutomationStatus automationStatus : List.of(AutomationStatus.RUNNING_INSERT_AUTOMATIONS, AutomationStatus.RUNNING_UPDATE_AUTOMATIONS, AutomationStatus.FAILED_INSERT_AUTOMATIONS, AutomationStatus.FAILED_UPDATE_AUTOMATIONS, AutomationStatus.OK)) + { + for(QTableMetaData table : List.of(tableWithNoAutomations, tableWithInsertAutomation, tableWithUpdateAutomation, tableWithInsertAndUpdateAutomations, tableWithInsertTrigger, tableWithUpdateTrigger)) + { + assertFalse(RecordAutomationStatusUpdater.canWeSkipPendingAndGoToOkay(table, automationStatus), "Should never be okay to skip pending and go to OK (because we weren't going to pending). table=[" + table.getName() + "], status=[" + automationStatus + "]"); + } + } + } + +} \ No newline at end of file diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/automation/polling/PollingAutomationPerTableRunnerTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/automation/polling/PollingAutomationPerTableRunnerTest.java index a0cb010d..d4dcb1d5 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/automation/polling/PollingAutomationPerTableRunnerTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/automation/polling/PollingAutomationPerTableRunnerTest.java @@ -22,21 +22,27 @@ package com.kingsrook.qqq.backend.core.actions.automation.polling; +import java.io.Serializable; import java.time.LocalDate; import java.time.Month; import java.util.ArrayList; import java.util.List; import java.util.Optional; +import java.util.function.Supplier; import com.kingsrook.qqq.backend.core.BaseTest; import com.kingsrook.qqq.backend.core.actions.automation.AutomationStatus; import com.kingsrook.qqq.backend.core.actions.tables.InsertAction; +import com.kingsrook.qqq.backend.core.actions.tables.QueryAction; import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction; import com.kingsrook.qqq.backend.core.context.QContext; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertOutput; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput; import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput; import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; @@ -60,7 +66,9 @@ import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; /******************************************************************************* @@ -155,6 +163,40 @@ class PollingAutomationPerTableRunnerTest extends BaseTest + /******************************************************************************* + ** Test that if an automation has an error that we get error status + *******************************************************************************/ + @Test + void testAutomationWithError() throws QException + { + QInstance qInstance = QContext.getQInstance(); + + ///////////////////////////////////////////////////////////////////////////////////////////////////////////// + // insert 2 person records, both updated by the insert action, and 1 logged by logger-on-update automation // + ///////////////////////////////////////////////////////////////////////////////////////////////////////////// + InsertInput insertInput = new InsertInput(); + insertInput.setTableName(TestUtils.TABLE_NAME_PERSON_MEMORY); + insertInput.setRecords(List.of( + new QRecord().withValue("id", 1).withValue("firstName", "Han").withValue("lastName", "Solo").withValue("birthDate", LocalDate.parse("1977-05-25")), + new QRecord().withValue("id", 2).withValue("firstName", "Luke").withValue("lastName", "Skywalker").withValue("birthDate", LocalDate.parse("1977-05-25")), + new QRecord().withValue("id", 3).withValue("firstName", "Darth").withValue("lastName", "Vader").withValue("birthDate", LocalDate.parse("1977-05-25")) + )); + new InsertAction().execute(insertInput); + assertAllRecordsAutomationStatus(AutomationStatus.PENDING_INSERT_AUTOMATIONS); + + ///////////////////////// + // run the automations // + ///////////////////////// + runAllTableActions(qInstance); + + /////////////////////////////////////////////////////////////////////////////////////////////////////////// + // make sure all records are in status ERROR (even though only 1 threw, it breaks the page that it's in) // + /////////////////////////////////////////////////////////////////////////////////////////////////////////// + assertAllRecordsAutomationStatus(AutomationStatus.FAILED_INSERT_AUTOMATIONS); + } + + + /******************************************************************************* ** *******************************************************************************/ @@ -169,7 +211,6 @@ class PollingAutomationPerTableRunnerTest extends BaseTest // note - don't call run - it is meant to be called async - e.g., it sets & clears thread context. // ///////////////////////////////////////////////////////////////////////////////////////////////////// pollingAutomationPerTableRunner.processTableInsertOrUpdate(qInstance.getTable(tableAction.tableName()), QContext.getQSession(), tableAction.status()); - } } @@ -228,6 +269,77 @@ class PollingAutomationPerTableRunnerTest extends BaseTest + /******************************************************************************* + ** Test a large-ish number - to demonstrate paging working - and how it deals + ** with intermittent errors + ** + *******************************************************************************/ + @Test + void testMultiPagesWithSomeFailures() throws QException + { + QInstance qInstance = QContext.getQInstance(); + + ////////////////////////////////////////////////////////////////////////////////////////////////////// + // adjust table's automations batch size - as any exceptions thrown put the whole batch into error. // + // so we'll make batches (pages) of 100 - run for 500 records, and make just a couple bad records // + // that'll cause errors - so we should get a few failed pages, and the rest ok. // + ////////////////////////////////////////////////////////////////////////////////////////////////////// + int pageSize = 100; + qInstance.getTable(TestUtils.TABLE_NAME_PERSON_MEMORY) + .getAutomationDetails() + .setOverrideBatchSize(pageSize); + + ////////////////////////////////////////////////////////////////////////////////// + // insert many people - half who should be updated by the AgeChecker automation // + ////////////////////////////////////////////////////////////////////////////////// + InsertInput insertInput = new InsertInput(); + insertInput.setTableName(TestUtils.TABLE_NAME_PERSON_MEMORY); + + insertInput.setRecords(new ArrayList<>()); + int SIZE = 500; + for(int i = 0; i < SIZE; i++) + { + insertInput.getRecords().add(new QRecord().withValue("firstName", "Qui Gon").withValue("lastName", "Jinn " + i).withValue("birthDate", LocalDate.now())); + insertInput.getRecords().add(new QRecord().withValue("firstName", "Obi Wan").withValue("lastName", "Kenobi " + i)); + + ///////////////////////////////// + // throw 2 Darths into the mix // + ///////////////////////////////// + if(i == 101 || i == 301) + { + insertInput.getRecords().add(new QRecord().withValue("firstName", "Darth").withValue("lastName", "Maul " + i)); + } + } + + InsertOutput insertOutput = new InsertAction().execute(insertInput); + List insertedIds = insertOutput.getRecords().stream().map(r -> r.getValue("id")).toList(); + + assertAllRecordsAutomationStatus(AutomationStatus.PENDING_INSERT_AUTOMATIONS); + + ///////////////////////// + // run the automations // + ///////////////////////// + runAllTableActions(qInstance); + + //////////////////////////////////////////////////////////////////// + // make sure that some records became ok, but others became error // + //////////////////////////////////////////////////////////////////// + QueryOutput queryOutput = new QueryAction().execute(new QueryInput(TestUtils.TABLE_NAME_PERSON_MEMORY).withFilter(new QQueryFilter(new QFilterCriteria("id", QCriteriaOperator.IN, insertedIds)))); + List okRecords = queryOutput.getRecords().stream().filter(r -> AutomationStatus.OK.getId().equals(r.getValueInteger(TestUtils.standardQqqAutomationStatusField().getName()))).toList(); + List failedRecords = queryOutput.getRecords().stream().filter(r -> AutomationStatus.FAILED_INSERT_AUTOMATIONS.getId().equals(r.getValueInteger(TestUtils.standardQqqAutomationStatusField().getName()))).toList(); + + assertFalse(okRecords.isEmpty(), "Some inserted records should be automation status OK"); + assertFalse(failedRecords.isEmpty(), "Some inserted records should be automation status Failed"); + assertEquals(insertedIds.size(), okRecords.size() + failedRecords.size(), "All inserted records should be OK or Failed"); + + /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // make sure that only 2 pages failed - meaning our number of failedRecords is < pageSize * 2 (as any page may be smaller than the pageSize) // + /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + assertThat(failedRecords.size()).isLessThanOrEqualTo(pageSize * 2).describedAs("No more than 2 pages should be in failed status."); + } + + + /******************************************************************************* ** Test running a process for automation, instead of a code ref. *******************************************************************************/ @@ -367,6 +479,61 @@ class PollingAutomationPerTableRunnerTest extends BaseTest + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testServerShutdownMidRunLeavesRecordsInRunningStatus() throws QException + { + QInstance qInstance = QContext.getQInstance(); + + ///////////////////////////////////////////////////////////////////////////// + // insert 2 person records that should have insert action ran against them // + ///////////////////////////////////////////////////////////////////////////// + InsertInput insertInput = new InsertInput(); + insertInput.setTableName(TestUtils.TABLE_NAME_PERSON_MEMORY); + insertInput.setRecords(List.of( + new QRecord().withValue("id", 1).withValue("firstName", "Tim").withValue("birthDate", LocalDate.now()), + new QRecord().withValue("id", 2).withValue("firstName", "Darin").withValue("birthDate", LocalDate.now()) + )); + new InsertAction().execute(insertInput); + assertAllRecordsAutomationStatus(AutomationStatus.PENDING_INSERT_AUTOMATIONS); + + ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // duplicate the runAllTableActions method - but using the subclass of PollingAutomationPerTableRunner that will throw. // + ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + assertThatThrownBy(() -> + { + List tableActions = PollingAutomationPerTableRunner.getTableActions(qInstance, TestUtils.POLLING_AUTOMATION); + for(PollingAutomationPerTableRunner.TableActions tableAction : tableActions) + { + PollingAutomationPerTableRunner pollingAutomationPerTableRunner = new PollingAutomationPerTableRunnerThatShouldSimulateServerShutdownMidRun(qInstance, TestUtils.POLLING_AUTOMATION, QSession::new, tableAction); + + ///////////////////////////////////////////////////////////////////////////////////////////////////// + // note - don't call run - it is meant to be called async - e.g., it sets & clears thread context. // + ///////////////////////////////////////////////////////////////////////////////////////////////////// + pollingAutomationPerTableRunner.processTableInsertOrUpdate(qInstance.getTable(tableAction.tableName()), QContext.getQSession(), tableAction.status()); + } + }).hasMessage(PollingAutomationPerTableRunnerThatShouldSimulateServerShutdownMidRun.EXCEPTION_MESSAGE); + + ////////////////////////////////////////////////// + // records should be "leaked" in running status // + ////////////////////////////////////////////////// + assertAllRecordsAutomationStatus(AutomationStatus.RUNNING_INSERT_AUTOMATIONS); + + ///////////////////////////////////// + // simulate another run of the job // + ///////////////////////////////////// + runAllTableActions(qInstance); + + //////////////////////////////////////////////////////////////////////////////// + // it should NOT have updated those records - they're officially "leaked" now // + //////////////////////////////////////////////////////////////////////////////// + assertAllRecordsAutomationStatus(AutomationStatus.RUNNING_INSERT_AUTOMATIONS); + } + + + /******************************************************************************* ** *******************************************************************************/ @@ -376,4 +543,42 @@ class PollingAutomationPerTableRunnerTest extends BaseTest .isNotEmpty() .allMatch(r -> pendingInsertAutomations.getId().equals(r.getValue(TestUtils.standardQqqAutomationStatusField().getName()))); } + + + + /******************************************************************************* + ** this subclass of the class under test allows us to simulate: + ** + ** what happens if, after records have been marked as running-updates, if, + ** for example, a server shuts down? + ** + ** It does this by overriding a method that runs between those points in time, + ** and throwing a runtime exception. + *******************************************************************************/ + public static class PollingAutomationPerTableRunnerThatShouldSimulateServerShutdownMidRun extends PollingAutomationPerTableRunner + { + private static String EXCEPTION_MESSAGE = "Throwing outside of catch here, to simulate a server shutdown mid-run"; + + + + /******************************************************************************* + ** + *******************************************************************************/ + public PollingAutomationPerTableRunnerThatShouldSimulateServerShutdownMidRun(QInstance instance, String providerName, Supplier sessionSupplier, TableActions tableActions) + { + super(instance, providerName, sessionSupplier, tableActions); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + protected boolean applyActionToRecords(QTableMetaData table, List records, TableAutomationAction action) + { + throw (new RuntimeException(EXCEPTION_MESSAGE)); + } + } + } \ No newline at end of file diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java index c019aae5..c9c89e2f 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java @@ -34,6 +34,7 @@ import com.kingsrook.qqq.backend.core.actions.processes.BackendStep; import com.kingsrook.qqq.backend.core.actions.processes.person.addtopeoplesage.AddAge; import com.kingsrook.qqq.backend.core.actions.processes.person.addtopeoplesage.GetAgeStatistics; import com.kingsrook.qqq.backend.core.actions.tables.CountAction; +import com.kingsrook.qqq.backend.core.actions.tables.GetAction; import com.kingsrook.qqq.backend.core.actions.tables.InsertAction; import com.kingsrook.qqq.backend.core.actions.tables.QueryAction; import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction; @@ -45,6 +46,7 @@ import com.kingsrook.qqq.backend.core.logging.QLogger; import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetInput; import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; @@ -813,6 +815,11 @@ public class TestUtils .withFilter(new QQueryFilter().withCriteria(new QFilterCriteria("birthDate", QCriteriaOperator.GREATER_THAN, List.of(youngPersonLimitDate)))) .withCodeReference(new QCodeReference(CheckAge.class)) ) + .withAction(new TableAutomationAction() + .withName("failAutomationForSith") + .withTriggerEvent(TriggerEvent.POST_INSERT) + .withCodeReference(new QCodeReference(FailAutomationForSith.class)) + ) .withAction(new TableAutomationAction() .withName("increaseBirthdate") .withTriggerEvent(TriggerEvent.POST_INSERT) @@ -918,6 +925,15 @@ public class TestUtils List recordsToUpdate = new ArrayList<>(); for(QRecord record : recordAutomationInput.getRecordList()) { + //////////////////////////////////////////////////////////////////////// + // get the record - its automation status should currently be RUNNING // + //////////////////////////////////////////////////////////////////////// + QRecord freshlyFetchedRecord = new GetAction().executeForRecord(new GetInput(TABLE_NAME_PERSON_MEMORY).withPrimaryKey(record.getValue("id"))); + assertEquals(AutomationStatus.RUNNING_INSERT_AUTOMATIONS.getId(), freshlyFetchedRecord.getValueInteger(TestUtils.standardQqqAutomationStatusField().getName())); + + /////////////////////////////////////////// + // do whatever business logic we do here // + /////////////////////////////////////////// LocalDate birthDate = record.getValueLocalDate("birthDate"); if(birthDate != null && birthDate.isAfter(limitDate)) { @@ -940,6 +956,29 @@ public class TestUtils + /******************************************************************************* + ** + *******************************************************************************/ + public static class FailAutomationForSith extends RecordAutomationHandler + { + + /******************************************************************************* + ** + *******************************************************************************/ + public void execute(RecordAutomationInput recordAutomationInput) throws QException + { + for(QRecord record : recordAutomationInput.getRecordList()) + { + if("Darth".equals(record.getValue("firstName"))) + { + throw new QException("Oops, you look like a Sith!"); + } + } + } + } + + + /******************************************************************************* ** *******************************************************************************/