QRunIO/qqq
2
0
Fork 0
mirror of https://github.com/Kingsrook/qqq.git synced 2025-07-17 20:50:44 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #41 from Kingsrook/integration/20230921

Browse Source 
automationStatus → OK fixes; script + audit updates;
This commit is contained in:
darin.kelkhoff
2023-09-25 13:24:44 -05:00
committed by GitHub
parent a95e9d06a2 070dec1266
commit 164087beb0
12 changed files with 1037 additions and 163 deletions
Download Patch File Download Diff File Expand all files Collapse all files

58
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/audits/AuditAction.java
View File

@ -29,6 +29,7 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import com.kingsrook.qqq.backend.core.actions.AbstractQActionFunction;
import com.kingsrook.qqq.backend.core.actions.tables.GetAction;
import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
@ -49,6 +50,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.model.session.QUser;
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
import com.kingsrook.qqq.backend.core.utils.Pair;
import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
/*******************************************************************************
@ -108,12 +110,26 @@ public class AuditAction extends AbstractQActionFunction<AuditInput, AuditOutput
/*******************************************************************************
** Simple overload that internally figures out primary key and security key values
**
** Be aware - if the record doesn't have its security key values set (say it's a
** partial record as part of an update), then those values won't be in the
** security key map... This should probably be considered a bug.
*******************************************************************************/
public static void appendToInput(AuditInput auditInput, QTableMetaData table, QRecord record, String auditMessage)
{
appendToInput(auditInput, table.getName(), record.getValueInteger(table.getPrimaryKeyField()), getRecordSecurityKeyValues(table, record, Optional.empty()), auditMessage);
}
/*******************************************************************************
** Add 1 auditSingleInput to an AuditInput object - with no details (child records).
*******************************************************************************/
public static AuditInput appendToInput(AuditInput auditInput, String tableName, Integer recordId, Map<String, Serializable> securityKeyValues, String message)
public static void appendToInput(AuditInput auditInput, String tableName, Integer recordId, Map<String, Serializable> securityKeyValues, String message)
{
return (appendToInput(auditInput, tableName, recordId, securityKeyValues, message, null));
appendToInput(auditInput, tableName, recordId, securityKeyValues, message, null);
}
@ -139,6 +155,44 @@ public class AuditAction extends AbstractQActionFunction<AuditInput, AuditOutput
/*******************************************************************************
** For a given record, from a given table, build a map of the record's security
** key values.
**
** If, in case, the record has null value(s), and the oldRecord is given (e.g.,
** for the case of an update, where the record may not have all fields set, and
** oldRecord should be known for doing field-diffs), then try to get the value(s)
** from oldRecord.
**
** Currently, will leave values null if they aren't found after that.
**
** An alternative could be to re-fetch the record from its source if needed...
*******************************************************************************/
public static Map<String, Serializable> getRecordSecurityKeyValues(QTableMetaData table, QRecord record, Optional<QRecord> oldRecord)
{
Map<String, Serializable> securityKeyValues = new HashMap<>();
for(RecordSecurityLock recordSecurityLock : RecordSecurityLockFilters.filterForReadLocks(CollectionUtils.nonNullList(table.getRecordSecurityLocks())))
{
Serializable keyValue = record == null ? null : record.getValue(recordSecurityLock.getFieldName());
if(keyValue == null && oldRecord.isPresent())
{
LOG.debug("Table with a securityLock, but value not found in field", logPair("table", table.getName()), logPair("field", recordSecurityLock.getFieldName()));
keyValue = oldRecord.get().getValue(recordSecurityLock.getFieldName());
}
if(keyValue == null)
{
LOG.debug("Table with a securityLock, but value not found in field", logPair("table", table.getName()), logPair("field", recordSecurityLock.getFieldName()), logPair("oldRecordIsPresent", oldRecord.isPresent()));
}
securityKeyValues.put(recordSecurityLock.getSecurityKeyType(), keyValue);
}
return securityKeyValues;
}
/*******************************************************************************
**
*******************************************************************************/

395
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/audits/DMLAuditAction.java
View File

@ -23,8 +23,10 @@ package com.kingsrook.qqq.backend.core.actions.audits;
import java.io.Serializable;
import java.math.BigDecimal;
import java.time.Instant;
import java.time.ZoneId;
import java.time.temporal.ChronoUnit;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.HashMap;
@ -52,13 +54,12 @@ import com.kingsrook.qqq.backend.core.model.metadata.audits.AuditLevel;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.security.RecordSecurityLock;
import com.kingsrook.qqq.backend.core.model.metadata.security.RecordSecurityLockFilters;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.model.session.QSession;
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
import com.kingsrook.qqq.backend.core.utils.StringUtils;
import com.kingsrook.qqq.backend.core.utils.ValueUtils;
import static com.kingsrook.qqq.backend.core.actions.audits.AuditAction.getRecordSecurityKeyValues;
import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
@ -70,6 +71,8 @@ public class DMLAuditAction extends AbstractQActionFunction<DMLAuditInput, DMLAu
{
private static final QLogger LOG = QLogger.getLogger(DMLAuditAction.class);
public static final String AUDIT_CONTEXT_FIELD_NAME = "auditContext";
/*******************************************************************************
@ -99,34 +102,7 @@ public class DMLAuditAction extends AbstractQActionFunction<DMLAuditInput, DMLAu
return (output);
}
String contextSuffix = "";
if(StringUtils.hasContent(input.getAuditContext()))
{
contextSuffix = " " + input.getAuditContext();
}
Optional<AbstractActionInput> actionInput = QContext.getFirstActionInStack();
if(actionInput.isPresent() && actionInput.get() instanceof RunProcessInput runProcessInput)
{
String processName = runProcessInput.getProcessName();
QProcessMetaData process = QContext.getQInstance().getProcess(processName);
if(process != null)
{
contextSuffix = " during process: " + process.getLabel();
}
}
QSession qSession = QContext.getQSession();
String apiVersion = qSession.getValue("apiVersion");
if(apiVersion != null)
{
String apiLabel = qSession.getValue("apiLabel");
if(!StringUtils.hasContent(apiLabel))
{
apiLabel = "API";
}
contextSuffix += (" via " + apiLabel + " Version: " + apiVersion);
}
String contextSuffix = getContentSuffix(input);
AuditInput auditInput = new AuditInput();
if(auditLevel.equals(AuditLevel.RECORD) || (auditLevel.equals(AuditLevel.FIELD) && !dmlType.supportsFields))
@ -137,7 +113,7 @@ public class DMLAuditAction extends AbstractQActionFunction<DMLAuditInput, DMLAu
////////////////////////////////////////////////////////////////////////////////////////////////////////////
for(QRecord record : recordList)
{
AuditAction.appendToInput(auditInput, table.getName(), record.getValueInteger(table.getPrimaryKeyField()), getRecordSecurityKeyValues(table, record), "Record was " + dmlType.pastTenseVerb + contextSuffix);
AuditAction.appendToInput(auditInput, table.getName(), record.getValueInteger(table.getPrimaryKeyField()), getRecordSecurityKeyValues(table, record, Optional.empty()), "Record was " + dmlType.pastTenseVerb + contextSuffix);
}
}
else if(auditLevel.equals(AuditLevel.FIELD))
@ -147,7 +123,7 @@ public class DMLAuditAction extends AbstractQActionFunction<DMLAuditInput, DMLAu
///////////////////////////////////////////////////////////////////
// do many audits, all with field level details, for FIELD level //
///////////////////////////////////////////////////////////////////
QPossibleValueTranslator qPossibleValueTranslator = new QPossibleValueTranslator(QContext.getQInstance(), qSession);
QPossibleValueTranslator qPossibleValueTranslator = new QPossibleValueTranslator(QContext.getQInstance(), QContext.getQSession());
qPossibleValueTranslator.translatePossibleValuesInRecords(table, CollectionUtils.mergeLists(recordList, oldRecordList));
//////////////////////////////////////////
@ -169,92 +145,8 @@ public class DMLAuditAction extends AbstractQActionFunction<DMLAuditInput, DMLAu
List<QRecord> details = new ArrayList<>();
for(String fieldName : sortedFieldNames)
{
if(!record.getValues().containsKey(fieldName))
{
////////////////////////////////////////////////////////////////////////////////////////////////
// if the stored record doesn't have this field name, then don't audit anything about it //
// this is to deal with our Patch style updates not looking like every field was cleared out. //
////////////////////////////////////////////////////////////////////////////////////////////////
continue;
}
if(fieldName.equals("modifyDate") || fieldName.equals("createDate") || fieldName.equals("automationStatus"))
{
continue;
}
QFieldMetaData field = table.getField(fieldName);
Serializable value = ValueUtils.getValueAsFieldType(field.getType(), record.getValue(fieldName));
Serializable oldValue = oldRecord == null ? null : ValueUtils.getValueAsFieldType(field.getType(), oldRecord.getValue(fieldName));
QRecord detailRecord = null;
if(oldRecord == null)
{
if(DMLType.INSERT.equals(dmlType) && value == null)
{
continue;
}
if(field.getType().equals(QFieldType.BLOB) || field.getType().needsMasked())
{
detailRecord = new QRecord().withValue("message", "Set " + field.getLabel());
}
else
{
String formattedValue = getFormattedValueForAuditDetail(record, fieldName, field, value);
detailRecord = new QRecord().withValue("message", "Set " + field.getLabel() + " to " + formattedValue);
detailRecord.withValue("newValue", formattedValue);
}
}
else
{
if(!Objects.equals(oldValue, value))
{
if(field.getType().equals(QFieldType.BLOB) || field.getType().needsMasked())
{
if(oldValue == null)
{
detailRecord = new QRecord().withValue("message", "Set " + field.getLabel());
}
else if(value == null)
{
detailRecord = new QRecord().withValue("message", "Removed " + field.getLabel());
}
else
{
detailRecord = new QRecord().withValue("message", "Changed " + field.getLabel());
}
}
else
{
String formattedValue = getFormattedValueForAuditDetail(record, fieldName, field, value);
String formattedOldValue = getFormattedValueForAuditDetail(oldRecord, fieldName, field, oldValue);
if(oldValue == null)
{
detailRecord = new QRecord().withValue("message", "Set " + field.getLabel() + " to " + formatFormattedValueForDetailMessage(field, formattedValue));
detailRecord.withValue("newValue", formattedValue);
}
else if(value == null)
{
detailRecord = new QRecord().withValue("message", "Removed " + formatFormattedValueForDetailMessage(field, formattedOldValue) + " from " + field.getLabel());
detailRecord.withValue("oldValue", formattedOldValue);
}
else
{
detailRecord = new QRecord().withValue("message", "Changed " + field.getLabel() + " from " + formatFormattedValueForDetailMessage(field, formattedOldValue) + " to " + formatFormattedValueForDetailMessage(field, formattedValue));
detailRecord.withValue("oldValue", formattedOldValue);
detailRecord.withValue("newValue", formattedValue);
}
}
}
}
if(detailRecord != null)
{
detailRecord.withValue("fieldName", fieldName);
details.add(detailRecord);
}
makeAuditDetailRecordForField(fieldName, table, dmlType, record, oldRecord)
.ifPresent(details::add);
}
if(details.isEmpty() && DMLType.UPDATE.equals(dmlType))
@ -264,7 +156,7 @@ public class DMLAuditAction extends AbstractQActionFunction<DMLAuditInput, DMLAu
}
else
{
AuditAction.appendToInput(auditInput, table.getName(), record.getValueInteger(table.getPrimaryKeyField()), getRecordSecurityKeyValues(table, record), "Record was " + dmlType.pastTenseVerb + contextSuffix, details);
AuditAction.appendToInput(auditInput, table.getName(), record.getValueInteger(table.getPrimaryKeyField()), getRecordSecurityKeyValues(table, record, Optional.ofNullable(oldRecord)), "Record was " + dmlType.pastTenseVerb + contextSuffix, details);
}
}
}
@ -284,6 +176,254 @@ public class DMLAuditAction extends AbstractQActionFunction<DMLAuditInput, DMLAu
/*******************************************************************************
**
*******************************************************************************/
static String getContentSuffix(DMLAuditInput input)
{
StringBuilder contextSuffix = new StringBuilder();
/////////////////////////////////////////////////////////////////////////////
// start with context from the input wrapper //
// note, these contexts get propagated down from Input/Update/Delete Input //
/////////////////////////////////////////////////////////////////////////////
if(StringUtils.hasContent(input.getAuditContext()))
{
contextSuffix.append(" ").append(input.getAuditContext());
}
/////////////////////////////////////////////////////////////////////////////////////
// note process label (and a possible context from the process's state) if present //
/////////////////////////////////////////////////////////////////////////////////////
Optional<AbstractActionInput> actionInput = QContext.getFirstActionInStack();
if(actionInput.isPresent() && actionInput.get() instanceof RunProcessInput runProcessInput)
{
String processAuditContext = ValueUtils.getValueAsString(runProcessInput.getValue(AUDIT_CONTEXT_FIELD_NAME));
if(StringUtils.hasContent(processAuditContext))
{
contextSuffix.append(" ").append(processAuditContext);
}
String processName = runProcessInput.getProcessName();
QProcessMetaData process = QContext.getQInstance().getProcess(processName);
if(process != null)
{
contextSuffix.append(" during process: ").append(process.getLabel());
}
}
///////////////////////////////////////////////////
// use api label & version if present in session //
///////////////////////////////////////////////////
QSession qSession = QContext.getQSession();
String apiVersion = qSession.getValue("apiVersion");
if(apiVersion != null)
{
String apiLabel = qSession.getValue("apiLabel");
if(!StringUtils.hasContent(apiLabel))
{
apiLabel = "API";
}
contextSuffix.append(" via ").append(apiLabel).append(" Version: ").append(apiVersion);
}
return (contextSuffix.toString());
}
/*******************************************************************************
**
*******************************************************************************/
static Optional<QRecord> makeAuditDetailRecordForField(String fieldName, QTableMetaData table, DMLType dmlType, QRecord record, QRecord oldRecord)
{
if(!record.getValues().containsKey(fieldName))
{
////////////////////////////////////////////////////////////////////////////////////////////////
// if the stored record doesn't have this field name, then don't audit anything about it //
// this is to deal with our Patch style updates not looking like every field was cleared out. //
////////////////////////////////////////////////////////////////////////////////////////////////
return (Optional.empty());
}
if(fieldName.equals("modifyDate") || fieldName.equals("createDate") || fieldName.equals("automationStatus"))
{
return (Optional.empty());
}
QFieldMetaData field = table.getField(fieldName);
Serializable value = ValueUtils.getValueAsFieldType(field.getType(), record.getValue(fieldName));
Serializable oldValue = oldRecord == null ? null : ValueUtils.getValueAsFieldType(field.getType(), oldRecord.getValue(fieldName));
QRecord detailRecord = null;
if(oldRecord == null)
{
if(DMLType.INSERT.equals(dmlType) && value == null)
{
return (Optional.empty());
}
if(field.getType().equals(QFieldType.BLOB) || field.getType().needsMasked())
{
detailRecord = new QRecord().withValue("message", "Set " + field.getLabel());
}
else
{
String formattedValue = getFormattedValueForAuditDetail(record, fieldName, field, value);
detailRecord = new QRecord().withValue("message", "Set " + field.getLabel() + " to " + formattedValue);
detailRecord.withValue("newValue", formattedValue);
}
}
else
{
if(areValuesDifferentForAudit(field, value, oldValue))
{
if(field.getType().equals(QFieldType.BLOB) || field.getType().needsMasked())
{
if(oldValue == null)
{
detailRecord = new QRecord().withValue("message", "Set " + field.getLabel());
}
else if(value == null)
{
detailRecord = new QRecord().withValue("message", "Removed " + field.getLabel());
}
else
{
detailRecord = new QRecord().withValue("message", "Changed " + field.getLabel());
}
}
else
{
String formattedValue = getFormattedValueForAuditDetail(record, fieldName, field, value);
String formattedOldValue = getFormattedValueForAuditDetail(oldRecord, fieldName, field, oldValue);
if(oldValue == null)
{
detailRecord = new QRecord().withValue("message", "Set " + field.getLabel() + " to " + formatFormattedValueForDetailMessage(field, formattedValue));
detailRecord.withValue("newValue", formattedValue);
}
else if(value == null)
{
detailRecord = new QRecord().withValue("message", "Removed " + formatFormattedValueForDetailMessage(field, formattedOldValue) + " from " + field.getLabel());
detailRecord.withValue("oldValue", formattedOldValue);
}
else
{
detailRecord = new QRecord().withValue("message", "Changed " + field.getLabel() + " from " + formatFormattedValueForDetailMessage(field, formattedOldValue) + " to " + formatFormattedValueForDetailMessage(field, formattedValue));
detailRecord.withValue("oldValue", formattedOldValue);
detailRecord.withValue("newValue", formattedValue);
}
}
}
}
if(detailRecord != null)
{
LOG.debug("Returning with message: " + detailRecord.getValueString("message"));
detailRecord.withValue("fieldName", fieldName);
return (Optional.of(detailRecord));
}
return (Optional.empty());
}
/*******************************************************************************
**
*******************************************************************************/
static boolean areValuesDifferentForAudit(QFieldMetaData field, Serializable value, Serializable oldValue)
{
try
{
///////////////////
// decimal rules //
///////////////////
if(field.getType().equals(QFieldType.DECIMAL))
{
BigDecimal newBD = ValueUtils.getValueAsBigDecimal(value);
BigDecimal oldBD = ValueUtils.getValueAsBigDecimal(oldValue);
if(newBD == null && oldBD == null)
{
return (false);
}
if(newBD == null || oldBD == null)
{
return (true);
}
return (newBD.compareTo(oldBD) != 0);
}
////////////////////
// dateTime rules //
////////////////////
if(field.getType().equals(QFieldType.DATE_TIME))
{
Instant newI = ValueUtils.getValueAsInstant(value);
Instant oldI = ValueUtils.getValueAsInstant(oldValue);
if(newI == null && oldI == null)
{
return (false);
}
if(newI == null || oldI == null)
{
return (true);
}
////////////////////////////////
// just compare to the second //
////////////////////////////////
return (newI.truncatedTo(ChronoUnit.SECONDS).compareTo(oldI.truncatedTo(ChronoUnit.SECONDS)) != 0);
}
//////////////////
// string rules //
//////////////////
if(field.getType().isStringLike())
{
String newString = ValueUtils.getValueAsString(value);
String oldString = ValueUtils.getValueAsString(oldValue);
boolean newIsNullOrEmpty = !StringUtils.hasContent(newString);
boolean oldIsNullOrEmpty = !StringUtils.hasContent(oldString);
if(newIsNullOrEmpty && oldIsNullOrEmpty)
{
return (false);
}
if(newIsNullOrEmpty || oldIsNullOrEmpty)
{
return (true);
}
return (newString.compareTo(oldString) != 0);
}
/////////////////////////////////////
// default just use Objects.equals //
/////////////////////////////////////
return !Objects.equals(oldValue, value);
}
catch(Exception e)
{
LOG.debug("Error checking areValuesDifferentForAudit", e, logPair("fieldName", field.getName()), logPair("value", value), logPair("oldValue", oldValue));
}
////////////////////////////////////
// default to something simple... //
////////////////////////////////////
return !Objects.equals(oldValue, value);
}
/*******************************************************************************
**
*******************************************************************************/
@ -373,21 +513,6 @@ public class DMLAuditAction extends AbstractQActionFunction<DMLAuditInput, DMLAu
/*******************************************************************************
**
*******************************************************************************/
private static Map<String, Serializable> getRecordSecurityKeyValues(QTableMetaData table, QRecord record)
{
Map<String, Serializable> securityKeyValues = new HashMap<>();
for(RecordSecurityLock recordSecurityLock : RecordSecurityLockFilters.filterForReadLocks(CollectionUtils.nonNullList(table.getRecordSecurityLocks())))
{
securityKeyValues.put(recordSecurityLock.getSecurityKeyType(), record == null ? null : record.getValue(recordSecurityLock.getFieldName()));
}
return securityKeyValues;
}
/*******************************************************************************
**
*******************************************************************************/
@ -407,7 +532,7 @@ public class DMLAuditAction extends AbstractQActionFunction<DMLAuditInput, DMLAu
/*******************************************************************************
**
*******************************************************************************/
private enum DMLType
enum DMLType
{
INSERT("Inserted", true),
UPDATE("Edited", true),

37
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/automation/RecordAutomationStatusUpdater.java
View File

@ -22,9 +22,8 @@
package com.kingsrook.qqq.backend.core.actions.automation;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Objects;
import com.kingsrook.qqq.backend.core.actions.tables.CountAction;
import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction;
import com.kingsrook.qqq.backend.core.context.QContext;
@ -90,6 +89,10 @@ public class RecordAutomationStatusUpdater
}
}
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// Avoid setting records to PENDING_INSERT or PENDING_UPDATE even if they don't have any insert or update automations or triggers //
// such records should go straight to OK status. //
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
if(canWeSkipPendingAndGoToOkay(table, automationStatus))
{
automationStatus = AutomationStatus.OK;
@ -121,9 +124,13 @@ public class RecordAutomationStatusUpdater
** being asked to set status to PENDING_INSERT (or PENDING_UPDATE), then just
** move the status straight to OK.
*******************************************************************************/
private static boolean canWeSkipPendingAndGoToOkay(QTableMetaData table, AutomationStatus automationStatus)
static boolean canWeSkipPendingAndGoToOkay(QTableMetaData table, AutomationStatus automationStatus)
{
List<TableAutomationAction> tableActions = Objects.requireNonNullElse(table.getAutomationDetails().getActions(), new ArrayList<>());
List<TableAutomationAction> tableActions = Collections.emptyList();
if(table.getAutomationDetails() != null && table.getAutomationDetails().getActions() != null)
{
tableActions = table.getAutomationDetails().getActions();
}
if(automationStatus.equals(AutomationStatus.PENDING_INSERT_AUTOMATIONS))
{
@ -135,6 +142,12 @@ public class RecordAutomationStatusUpdater
{
return (false);
}
////////////////////////////////////////////////////////////////////////////////////////
// if we're going to pending-insert, and there are no insert automations or triggers, //
// then we may skip pending and go to okay. //
////////////////////////////////////////////////////////////////////////////////////////
return (true);
}
else if(automationStatus.equals(AutomationStatus.PENDING_UPDATE_AUTOMATIONS))
{
@ -146,9 +159,21 @@ public class RecordAutomationStatusUpdater
{
return (false);
}
}
return (true);
////////////////////////////////////////////////////////////////////////////////////////
// if we're going to pending-update, and there are no insert automations or triggers, //
// then we may skip pending and go to okay. //
////////////////////////////////////////////////////////////////////////////////////////
return (true);
}
else
{
///////////////////////////////////////////////////////////////////////////////////////////////////////
// if we're going to any other automation status - then we may never "skip pending" and go to okay - //
// because we weren't asked to go to pending! //
///////////////////////////////////////////////////////////////////////////////////////////////////////
return (false);
}
}

48
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/automation/polling/PollingAutomationPerTableRunner.java
View File

@ -342,22 +342,9 @@ public class PollingAutomationPerTableRunner implements Runnable
boolean anyActionsFailed = false;
for(TableAutomationAction action : actions)
{
try
boolean hadError = applyActionToRecords(table, records, action);
if(hadError)
{
//////////////////////////////////////////////////////////////////////////////////////////////////////////////
// note - this method - will re-query the objects, so we should have confidence that their data is fresh... //
//////////////////////////////////////////////////////////////////////////////////////////////////////////////
List<QRecord> matchingQRecords = getRecordsMatchingActionFilter(table, records, action);
LOG.debug("Of the {} records that were pending automations, {} of them match the filter on the action {}", records.size(), matchingQRecords.size(), action);
if(CollectionUtils.nullSafeHasContents(matchingQRecords))
{
LOG.debug(" Processing " + matchingQRecords.size() + " records in " + table + " for action " + action);
applyActionToMatchingRecords(table, matchingQRecords, action);
}
}
catch(Exception e)
{
LOG.warn("Caught exception processing records on " + table + " for action " + action, e);
anyActionsFailed = true;
}
}
@ -377,6 +364,37 @@ public class PollingAutomationPerTableRunner implements Runnable
/*******************************************************************************
** Run one action over a list of records (if they match the action's filter).
**
** @return hadError - true if an exception was caught; false if all OK.
*******************************************************************************/
protected boolean applyActionToRecords(QTableMetaData table, List<QRecord> records, TableAutomationAction action)
{
try
{
//////////////////////////////////////////////////////////////////////////////////////////////////////////////
// note - this method - will re-query the objects, so we should have confidence that their data is fresh... //
//////////////////////////////////////////////////////////////////////////////////////////////////////////////
List<QRecord> matchingQRecords = getRecordsMatchingActionFilter(table, records, action);
LOG.debug("Of the {} records that were pending automations, {} of them match the filter on the action {}", records.size(), matchingQRecords.size(), action);
if(CollectionUtils.nullSafeHasContents(matchingQRecords))
{
LOG.debug(" Processing " + matchingQRecords.size() + " records in " + table + " for action " + action);
applyActionToMatchingRecords(table, matchingQRecords, action);
}
return (false);
}
catch(Exception e)
{
LOG.warn("Caught exception processing records on " + table + " for action " + action, e);
return (true);
}
}
/*******************************************************************************
** For a given action, and a list of records - return a new list, of the ones
** which match the action's filter (if there is one - if not, then all match).

2
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/scripts/ScriptsMetaDataProvider.java
View File

@ -383,7 +383,7 @@ public class ScriptsMetaDataProvider
/*******************************************************************************
**
*******************************************************************************/
private QTableMetaData defineTableTriggerTable(String backendName) throws QException
public QTableMetaData defineTableTriggerTable(String backendName) throws QException
{
QTableMetaData tableMetaData = defineStandardTable(backendName, TableTrigger.TABLE_NAME, TableTrigger.class)
.withRecordLabelFields("id")

53
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/scripts/RunRecordScriptLoadStep.java
View File

@ -25,11 +25,15 @@ package com.kingsrook.qqq.backend.core.processes.implementations.scripts;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.List;
import com.kingsrook.qqq.backend.core.actions.audits.AuditAction;
import com.kingsrook.qqq.backend.core.actions.audits.DMLAuditAction;
import com.kingsrook.qqq.backend.core.actions.scripts.RunAdHocRecordScriptAction;
import com.kingsrook.qqq.backend.core.actions.scripts.logging.StoreScriptLogAndScriptLogLineExecutionLogger;
import com.kingsrook.qqq.backend.core.actions.tables.GetAction;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.logging.QLogger;
import com.kingsrook.qqq.backend.core.model.actions.audits.AuditInput;
import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryFilterLink;
import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLine;
import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLineInterface;
@ -42,7 +46,9 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetOutput;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.code.AdHocScriptCodeReference;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.model.scripts.Script;
import com.kingsrook.qqq.backend.core.model.scripts.ScriptLog;
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.AbstractLoadStep;
@ -121,16 +127,25 @@ public class RunRecordScriptLoadStep extends AbstractLoadStep implements Process
getInput.setTableName(Script.TABLE_NAME);
getInput.setPrimaryKey(scriptId);
GetOutput getOutput = new GetAction().execute(getInput);
if(getOutput.getRecord() == null)
QRecord script = getOutput.getRecord();
if(script == null)
{
throw (new QException("Could not find script by id: " + scriptId));
}
//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// set an "audit context" - so any DML executed during the script will include the note of what script was running. //
//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
runBackendStepInput.addValue(DMLAuditAction.AUDIT_CONTEXT_FIELD_NAME, "via Script \"" + script.getValue("name") + "\"");
String tableName = script.getValueString("tableName");
RunAdHocRecordScriptInput input = new RunAdHocRecordScriptInput();
input.setRecordList(runBackendStepInput.getRecords());
input.setCodeReference(new AdHocScriptCodeReference().withScriptId(scriptId));
input.setTableName(getOutput.getRecord().getValueString("tableName"));
input.setTableName(tableName);
input.setLogger(scriptLogger);
RunAdHocRecordScriptOutput output = new RunAdHocRecordScriptOutput();
Exception caughtException = null;
try
@ -147,11 +162,17 @@ public class RunRecordScriptLoadStep extends AbstractLoadStep implements Process
caughtException = e;
}
String auditMessage = "Script \"" + script.getValueString("name") + "\" (id: " + scriptId + ") was executed against this record";
//////////////////////////////////////////////////////////
// add the record to the appropriate processSummaryLine //
//////////////////////////////////////////////////////////
if(scriptLogger.getScriptLog() != null)
{
Integer id = scriptLogger.getScriptLog().getValueInteger("id");
if(id != null)
{
auditMessage += ", creating script log: " + id;
boolean hadError = BooleanUtils.isTrue(scriptLogger.getScriptLog().getValueBoolean("hadError"));
(hadError ? errorScriptLogIds : okScriptLogIds).add(id);
}
@ -160,6 +181,34 @@ public class RunRecordScriptLoadStep extends AbstractLoadStep implements Process
{
unloggedExceptionLine.incrementCount(runBackendStepInput.getRecords().size());
}
////////////////////////////////////////////////////////////
// audit that the script was executed against the records //
////////////////////////////////////////////////////////////
audit(runBackendStepInput, tableName, auditMessage);
}
/*******************************************************************************
** for each input record, add an audit stating that the script was executed.
*******************************************************************************/
private static void audit(RunBackendStepInput runBackendStepInput, String tableName, String auditMessage)
{
try
{
QTableMetaData table = QContext.getQInstance().getTable(tableName);
AuditInput auditInput = new AuditInput();
for(QRecord record : runBackendStepInput.getRecords())
{
AuditAction.appendToInput(auditInput, table, record, auditMessage);
}
new AuditAction().execute(auditInput);
}
catch(Exception e)
{
LOG.warn("Error recording audits after running record script", e, logPair("tableName", tableName), logPair("auditMessage", auditMessage));
}
}

2
qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/BaseTest.java
View File

@ -48,6 +48,8 @@ public class BaseTest
@BeforeEach
void baseBeforeEach()
{
System.setProperty("qqq.logger.logSessionId.disabled", "true");
QContext.init(TestUtils.defineInstance(), new QSession()
.withUser(new QUser()
.withIdReference("001")

29
qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/audits/AuditActionTest.java
View File

@ -31,6 +31,7 @@ import com.kingsrook.qqq.backend.core.BaseTest;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.audits.AuditInput;
import com.kingsrook.qqq.backend.core.model.actions.audits.AuditSingleInput;
import com.kingsrook.qqq.backend.core.model.audits.AuditsMetaDataProvider;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
@ -38,6 +39,7 @@ import com.kingsrook.qqq.backend.core.model.session.QSession;
import com.kingsrook.qqq.backend.core.model.session.QUser;
import com.kingsrook.qqq.backend.core.processes.utils.GeneralProcessUtils;
import com.kingsrook.qqq.backend.core.utils.TestUtils;
import com.kingsrook.qqq.backend.core.utils.collections.MapBuilder;
import org.junit.jupiter.api.Test;
import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.jupiter.api.Assertions.assertEquals;
@ -197,4 +199,31 @@ class AuditActionTest extends BaseTest
assertThat(auditDetails).anyMatch(r -> r.getValueString("message").equals("Detail3"));
}
/*******************************************************************************
**
*******************************************************************************/
@Test
void testAppendToInputThatTakesRecordNotIdAndSecurityKeyValues()
{
AuditInput auditInput = new AuditInput();
//////////////////////////////////////////////////////////////
// make sure the recordId & securityKey got build correctly //
//////////////////////////////////////////////////////////////
AuditAction.appendToInput(auditInput, QContext.getQInstance().getTable(TestUtils.TABLE_NAME_ORDER), new QRecord().withValue("id", 47).withValue("storeId", 42), "Test");
AuditSingleInput auditSingleInput = auditInput.getAuditSingleInputList().get(0);
assertEquals(47, auditSingleInput.getRecordId());
assertEquals(MapBuilder.of(TestUtils.SECURITY_KEY_TYPE_STORE, 42), auditSingleInput.getSecurityKeyValues());
///////////////////////////////////////////////////////////////////////////////////////////
// acknowledge that we might get back a null key value if the record doesn't have it set //
///////////////////////////////////////////////////////////////////////////////////////////
AuditAction.appendToInput(auditInput, QContext.getQInstance().getTable(TestUtils.TABLE_NAME_ORDER), new QRecord().withValue("id", 47), "Test");
auditSingleInput = auditInput.getAuditSingleInputList().get(1);
assertEquals(47, auditSingleInput.getRecordId());
assertEquals(MapBuilder.of(TestUtils.SECURITY_KEY_TYPE_STORE, null), auditSingleInput.getSecurityKeyValues());
}
}

202
qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/audits/DMLAuditActionTest.java
View File

@ -22,12 +22,14 @@
package com.kingsrook.qqq.backend.core.actions.audits;
import java.math.BigDecimal;
import java.time.Instant;
import java.util.List;
import com.kingsrook.qqq.backend.core.BaseTest;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.audits.DMLAuditInput;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
@ -36,10 +38,18 @@ import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.audits.AuditLevel;
import com.kingsrook.qqq.backend.core.model.metadata.audits.QAuditRules;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.model.session.QSession;
import com.kingsrook.qqq.backend.core.model.statusmessages.BadInputStatusMessage;
import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryRecordStore;
import com.kingsrook.qqq.backend.core.utils.TestUtils;
import com.kingsrook.qqq.backend.core.utils.ValueUtils;
import org.junit.jupiter.api.Test;
import static com.kingsrook.qqq.backend.core.actions.audits.DMLAuditAction.DMLType.INSERT;
import static com.kingsrook.qqq.backend.core.actions.audits.DMLAuditAction.DMLType.UPDATE;
import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNull;
@ -200,4 +210,194 @@ class DMLAuditActionTest extends BaseTest
}
}
}
/*******************************************************************************
**
*******************************************************************************/
@Test
void testMakeAuditDetailRecordForField()
{
QTableMetaData table = new QTableMetaData()
.withField(new QFieldMetaData("createDate", QFieldType.DATE_TIME).withLabel("Create Date"))
.withField(new QFieldMetaData("modifyDate", QFieldType.DATE_TIME).withLabel("Modify Date"))
.withField(new QFieldMetaData("someTimestamp", QFieldType.DATE_TIME).withLabel("Some Timestamp"))
.withField(new QFieldMetaData("name", QFieldType.STRING).withLabel("Name"))
.withField(new QFieldMetaData("seqNo", QFieldType.INTEGER).withLabel("Sequence No."))
.withField(new QFieldMetaData("price", QFieldType.DECIMAL).withLabel("Price"));
///////////////////////////////
// create date - never audit //
///////////////////////////////
assertThat(DMLAuditAction.makeAuditDetailRecordForField("createDate", table, INSERT,
new QRecord().withValue("createDate", Instant.now()),
new QRecord().withValue("createDate", Instant.now().minusSeconds(100))))
.isEmpty();
///////////////////////////////
// modify date - never audit //
///////////////////////////////
assertThat(DMLAuditAction.makeAuditDetailRecordForField("modifyDate", table, UPDATE,
new QRecord().withValue("modifyDate", Instant.now()),
new QRecord().withValue("modifyDate", Instant.now().minusSeconds(100))))
.isEmpty();
////////////////////////////////////////////////////////
// datetime different only in precision - don't audit //
////////////////////////////////////////////////////////
assertThat(DMLAuditAction.makeAuditDetailRecordForField("someTimestamp", table, UPDATE,
new QRecord().withValue("someTimestamp", ValueUtils.getValueAsInstant("2023-04-17T14:33:08.777")),
new QRecord().withValue("someTimestamp", Instant.parse("2023-04-17T14:33:08Z"))))
.isEmpty();
/////////////////////////////////////////////
// datetime actually different - audit it. //
/////////////////////////////////////////////
assertThat(DMLAuditAction.makeAuditDetailRecordForField("someTimestamp", table, UPDATE,
new QRecord().withValue("someTimestamp", Instant.parse("2023-04-17T14:33:09Z")),
new QRecord().withValue("someTimestamp", Instant.parse("2023-04-17T14:33:08Z"))))
.isPresent()
.get().extracting(r -> r.getValueString("message"))
.matches(s -> s.matches("Changed Some Timestamp from 2023.* to 2023.*"));
////////////////////////////////////////////////
// datetime changing null to not null - audit //
////////////////////////////////////////////////
assertThat(DMLAuditAction.makeAuditDetailRecordForField("someTimestamp", table, UPDATE,
new QRecord().withValue("someTimestamp", ValueUtils.getValueAsInstant("2023-04-17T14:33:08.777")),
new QRecord().withValue("someTimestamp", null)))
.isPresent()
.get().extracting(r -> r.getValueString("message"))
.matches(s -> s.matches("Set Some Timestamp to 2023.*"));
////////////////////////////////////////////////
// datetime changing not null to null - audit //
////////////////////////////////////////////////
assertThat(DMLAuditAction.makeAuditDetailRecordForField("someTimestamp", table, UPDATE,
new QRecord().withValue("someTimestamp", null),
new QRecord().withValue("someTimestamp", Instant.parse("2023-04-17T14:33:08Z"))))
.isPresent()
.get().extracting(r -> r.getValueString("message"))
.matches(s -> s.matches("Removed 2023.*from Some Timestamp"));
////////////////////////////////////////
// string that is the same - no audit //
////////////////////////////////////////
assertThat(DMLAuditAction.makeAuditDetailRecordForField("name", table, UPDATE,
new QRecord().withValue("name", "Homer"),
new QRecord().withValue("name", "Homer")))
.isEmpty();
//////////////////////////////////////////
// string from null to empty - no audit //
//////////////////////////////////////////
assertThat(DMLAuditAction.makeAuditDetailRecordForField("name", table, UPDATE,
new QRecord().withValue("name", null),
new QRecord().withValue("name", "")))
.isEmpty();
//////////////////////////////////////////
// string from empty to null - no audit //
//////////////////////////////////////////
assertThat(DMLAuditAction.makeAuditDetailRecordForField("name", table, UPDATE,
new QRecord().withValue("name", ""),
new QRecord().withValue("name", null)))
.isEmpty();
//////////////////////////////////////////////////////////
// decimal that only changes in precision - don't audit //
//////////////////////////////////////////////////////////
assertThat(DMLAuditAction.makeAuditDetailRecordForField("price", table, UPDATE,
new QRecord().withValue("price", "10"),
new QRecord().withValue("price", new BigDecimal("10.00"))))
.isEmpty();
//////////////////////////////////////////////////
// decimal that's actually different - do audit //
//////////////////////////////////////////////////
assertThat(DMLAuditAction.makeAuditDetailRecordForField("price", table, UPDATE,
new QRecord().withValue("price", "10.01"),
new QRecord().withValue("price", new BigDecimal("10.00"))))
.isPresent()
.get().extracting(r -> r.getValueString("message"))
.matches(s -> s.matches("Changed Price from 10.00 to 10.01"));
///////////////////////////////////////
// decimal null, input "" - no audit //
///////////////////////////////////////
assertThat(DMLAuditAction.makeAuditDetailRecordForField("price", table, UPDATE,
new QRecord().withValue("price", ""),
new QRecord().withValue("price", null)))
.isEmpty();
/////////////////////////////////////////
// decimal not-null to null - do audit //
/////////////////////////////////////////
assertThat(DMLAuditAction.makeAuditDetailRecordForField("price", table, UPDATE,
new QRecord().withValue("price", BigDecimal.ONE),
new QRecord().withValue("price", null)))
.isPresent()
.get().extracting(r -> r.getValueString("message"))
.matches(s -> s.matches("Set Price to 1"));
/////////////////////////////////////////
// decimal null to not-null - do audit //
/////////////////////////////////////////
assertThat(DMLAuditAction.makeAuditDetailRecordForField("price", table, UPDATE,
new QRecord().withValue("price", null),
new QRecord().withValue("price", BigDecimal.ONE)))
.isPresent()
.get().extracting(r -> r.getValueString("message"))
.matches(s -> s.matches("Removed 1 from Price"));
///////////////////////////////////////
// integer null, input "" - no audit //
///////////////////////////////////////
assertThat(DMLAuditAction.makeAuditDetailRecordForField("seqNo", table, UPDATE,
new QRecord().withValue("seqNo", ""),
new QRecord().withValue("seqNo", null)))
.isEmpty();
////////////////////////////////
// integer changed - do audit //
////////////////////////////////
assertThat(DMLAuditAction.makeAuditDetailRecordForField("seqNo", table, UPDATE,
new QRecord().withValue("seqNo", 2),
new QRecord().withValue("seqNo", 1)))
.isPresent()
.get().extracting(r -> r.getValueString("message"))
.matches(s -> s.matches("Changed Sequence No. from 1 to 2"));
}
/*******************************************************************************
**
*******************************************************************************/
@Test
void testGetContextSuffix()
{
assertEquals("", DMLAuditAction.getContentSuffix(new DMLAuditInput()));
assertEquals(" while shipping an order", DMLAuditAction.getContentSuffix(new DMLAuditInput().withAuditContext("while shipping an order")));
QContext.pushAction(new RunProcessInput().withValue(DMLAuditAction.AUDIT_CONTEXT_FIELD_NAME, "via Script \"My Script\""));
assertEquals(" via Script \"My Script\"", DMLAuditAction.getContentSuffix(new DMLAuditInput()));
QContext.popAction();
QContext.pushAction(new RunProcessInput().withProcessName(TestUtils.PROCESS_NAME_GREET_PEOPLE));
assertEquals(" during process: Greet", DMLAuditAction.getContentSuffix(new DMLAuditInput()));
QContext.popAction();
QContext.setQSession(new QSession().withValue("apiVersion", "1.0"));
assertEquals(" via API Version: 1.0", DMLAuditAction.getContentSuffix(new DMLAuditInput()));
QContext.setQSession(new QSession().withValue("apiVersion", "20230921").withValue("apiLabel", "Our Public API"));
assertEquals(" via Our Public API Version: 20230921", DMLAuditAction.getContentSuffix(new DMLAuditInput()));
QContext.pushAction(new RunProcessInput().withProcessName(TestUtils.PROCESS_NAME_GREET_PEOPLE).withValue(DMLAuditAction.AUDIT_CONTEXT_FIELD_NAME, "via Script \"My Script\""));
QContext.setQSession(new QSession().withValue("apiVersion", "20230921").withValue("apiLabel", "Our Public API"));
assertEquals(" while shipping an order via Script \"My Script\" during process: Greet via Our Public API Version: 20230921", DMLAuditAction.getContentSuffix(new DMLAuditInput().withAuditContext("while shipping an order")));
QContext.popAction();
}
}

128
qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/automation/RecordAutomationStatusUpdaterTest.java Normal file
View File

@ -0,0 +1,128 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2023. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.actions.automation;
import java.util.List;
import com.kingsrook.qqq.backend.core.BaseTest;
import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
import com.kingsrook.qqq.backend.core.model.automation.TableTrigger;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.QTableAutomationDetails;
import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.TableAutomationAction;
import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.TriggerEvent;
import com.kingsrook.qqq.backend.core.model.scripts.ScriptsMetaDataProvider;
import com.kingsrook.qqq.backend.core.utils.TestUtils;
import org.junit.jupiter.api.Test;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;
/*******************************************************************************
** Unit test for RecordAutomationStatusUpdater
*******************************************************************************/
class RecordAutomationStatusUpdaterTest extends BaseTest
{
/*******************************************************************************
**
*******************************************************************************/
@Test
void testCanWeSkipPendingAndGoToOkay() throws QException
{
QContext.getQInstance()
.addTable(new ScriptsMetaDataProvider().defineTableTriggerTable(TestUtils.MEMORY_BACKEND_NAME));
////////////////////////////////////////////////////////////
// define tables with various automations and/or triggers //
////////////////////////////////////////////////////////////
QTableMetaData tableWithNoAutomations = new QTableMetaData()
.withName("tableWithNoAutomations");
QTableMetaData tableWithInsertAutomation = new QTableMetaData()
.withName("tableWithInsertAutomation")
.withAutomationDetails(new QTableAutomationDetails()
.withAction(new TableAutomationAction().withTriggerEvent(TriggerEvent.POST_INSERT)));
QTableMetaData tableWithUpdateAutomation = new QTableMetaData()
.withName("tableWithUpdateAutomation")
.withAutomationDetails(new QTableAutomationDetails()
.withAction(new TableAutomationAction()
.withTriggerEvent(TriggerEvent.POST_UPDATE)));
QTableMetaData tableWithInsertAndUpdateAutomations = new QTableMetaData()
.withName("tableWithInsertAndUpdateAutomations ")
.withAutomationDetails(new QTableAutomationDetails()
.withAction(new TableAutomationAction().withTriggerEvent(TriggerEvent.POST_INSERT))
.withAction(new TableAutomationAction().withTriggerEvent(TriggerEvent.POST_UPDATE)));
QTableMetaData tableWithInsertTrigger = new QTableMetaData()
.withName("tableWithInsertTrigger");
new InsertAction().execute(new InsertInput(TableTrigger.TABLE_NAME)
.withRecordEntity(new TableTrigger().withTableName(tableWithInsertTrigger.getName()).withPostInsert(true).withPostUpdate(false)));
QTableMetaData tableWithUpdateTrigger = new QTableMetaData()
.withName("tableWithUpdateTrigger");
new InsertAction().execute(new InsertInput(TableTrigger.TABLE_NAME)
.withRecordEntity(new TableTrigger().withTableName(tableWithUpdateTrigger.getName()).withPostInsert(false).withPostUpdate(true)));
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// tests for going to PENDING_INSERT. //
// we should be allowed to skip and go to OK (return true) if the table does not have insert automations or triggers //
// we should NOT be allowed to skip and go to OK (return false) if the table does NOT have insert automations or triggers //
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
assertTrue(RecordAutomationStatusUpdater.canWeSkipPendingAndGoToOkay(tableWithNoAutomations, AutomationStatus.PENDING_INSERT_AUTOMATIONS));
assertFalse(RecordAutomationStatusUpdater.canWeSkipPendingAndGoToOkay(tableWithInsertAutomation, AutomationStatus.PENDING_INSERT_AUTOMATIONS));
assertTrue(RecordAutomationStatusUpdater.canWeSkipPendingAndGoToOkay(tableWithUpdateAutomation, AutomationStatus.PENDING_INSERT_AUTOMATIONS));
assertFalse(RecordAutomationStatusUpdater.canWeSkipPendingAndGoToOkay(tableWithInsertAndUpdateAutomations, AutomationStatus.PENDING_INSERT_AUTOMATIONS));
assertFalse(RecordAutomationStatusUpdater.canWeSkipPendingAndGoToOkay(tableWithInsertTrigger, AutomationStatus.PENDING_INSERT_AUTOMATIONS));
assertTrue(RecordAutomationStatusUpdater.canWeSkipPendingAndGoToOkay(tableWithUpdateTrigger, AutomationStatus.PENDING_INSERT_AUTOMATIONS));
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// tests for going to PENDING_UPDATE. //
// we should be allowed to skip and go to OK (return true) if the table does not have update automations or triggers //
// we should NOT be allowed to skip and go to OK (return false) if the table does NOT have insert automations or triggers //
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
assertTrue(RecordAutomationStatusUpdater.canWeSkipPendingAndGoToOkay(tableWithNoAutomations, AutomationStatus.PENDING_UPDATE_AUTOMATIONS));
assertTrue(RecordAutomationStatusUpdater.canWeSkipPendingAndGoToOkay(tableWithInsertAutomation, AutomationStatus.PENDING_UPDATE_AUTOMATIONS));
assertFalse(RecordAutomationStatusUpdater.canWeSkipPendingAndGoToOkay(tableWithUpdateAutomation, AutomationStatus.PENDING_UPDATE_AUTOMATIONS));
assertFalse(RecordAutomationStatusUpdater.canWeSkipPendingAndGoToOkay(tableWithInsertAndUpdateAutomations, AutomationStatus.PENDING_UPDATE_AUTOMATIONS));
assertTrue(RecordAutomationStatusUpdater.canWeSkipPendingAndGoToOkay(tableWithInsertTrigger, AutomationStatus.PENDING_UPDATE_AUTOMATIONS));
assertFalse(RecordAutomationStatusUpdater.canWeSkipPendingAndGoToOkay(tableWithUpdateTrigger, AutomationStatus.PENDING_UPDATE_AUTOMATIONS));
//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// tests for going to non-PENDING states //
// this function should NEVER return true for skipping pending if the target state (2nd arg) isn't a pending state. //
//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
for(AutomationStatus automationStatus : List.of(AutomationStatus.RUNNING_INSERT_AUTOMATIONS, AutomationStatus.RUNNING_UPDATE_AUTOMATIONS, AutomationStatus.FAILED_INSERT_AUTOMATIONS, AutomationStatus.FAILED_UPDATE_AUTOMATIONS, AutomationStatus.OK))
{
for(QTableMetaData table : List.of(tableWithNoAutomations, tableWithInsertAutomation, tableWithUpdateAutomation, tableWithInsertAndUpdateAutomations, tableWithInsertTrigger, tableWithUpdateTrigger))
{
assertFalse(RecordAutomationStatusUpdater.canWeSkipPendingAndGoToOkay(table, automationStatus), "Should never be okay to skip pending and go to OK (because we weren't going to pending). table=[" + table.getName() + "], status=[" + automationStatus + "]");
}
}
}
}

207
qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/automation/polling/PollingAutomationPerTableRunnerTest.java
View File

@ -22,21 +22,27 @@
package com.kingsrook.qqq.backend.core.actions.automation.polling;
import java.io.Serializable;
import java.time.LocalDate;
import java.time.Month;
import java.util.ArrayList;
import java.util.List;
import java.util.Optional;
import java.util.function.Supplier;
import com.kingsrook.qqq.backend.core.BaseTest;
import com.kingsrook.qqq.backend.core.actions.automation.AutomationStatus;
import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertOutput;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput;
import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
@ -60,7 +66,9 @@ import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
/*******************************************************************************
@ -155,6 +163,40 @@ class PollingAutomationPerTableRunnerTest extends BaseTest
/*******************************************************************************
** Test that if an automation has an error that we get error status
*******************************************************************************/
@Test
void testAutomationWithError() throws QException
{
QInstance qInstance = QContext.getQInstance();
/////////////////////////////////////////////////////////////////////////////////////////////////////////////
// insert 2 person records, both updated by the insert action, and 1 logged by logger-on-update automation //
/////////////////////////////////////////////////////////////////////////////////////////////////////////////
InsertInput insertInput = new InsertInput();
insertInput.setTableName(TestUtils.TABLE_NAME_PERSON_MEMORY);
insertInput.setRecords(List.of(
new QRecord().withValue("id", 1).withValue("firstName", "Han").withValue("lastName", "Solo").withValue("birthDate", LocalDate.parse("1977-05-25")),
new QRecord().withValue("id", 2).withValue("firstName", "Luke").withValue("lastName", "Skywalker").withValue("birthDate", LocalDate.parse("1977-05-25")),
new QRecord().withValue("id", 3).withValue("firstName", "Darth").withValue("lastName", "Vader").withValue("birthDate", LocalDate.parse("1977-05-25"))
));
new InsertAction().execute(insertInput);
assertAllRecordsAutomationStatus(AutomationStatus.PENDING_INSERT_AUTOMATIONS);
/////////////////////////
// run the automations //
/////////////////////////
runAllTableActions(qInstance);
///////////////////////////////////////////////////////////////////////////////////////////////////////////
// make sure all records are in status ERROR (even though only 1 threw, it breaks the page that it's in) //
///////////////////////////////////////////////////////////////////////////////////////////////////////////
assertAllRecordsAutomationStatus(AutomationStatus.FAILED_INSERT_AUTOMATIONS);
}
/*******************************************************************************
**
*******************************************************************************/
@ -169,7 +211,6 @@ class PollingAutomationPerTableRunnerTest extends BaseTest
// note - don't call run - it is meant to be called async - e.g., it sets & clears thread context. //
/////////////////////////////////////////////////////////////////////////////////////////////////////
pollingAutomationPerTableRunner.processTableInsertOrUpdate(qInstance.getTable(tableAction.tableName()), QContext.getQSession(), tableAction.status());
}
}
@ -228,6 +269,77 @@ class PollingAutomationPerTableRunnerTest extends BaseTest
/*******************************************************************************
** Test a large-ish number - to demonstrate paging working - and how it deals
** with intermittent errors
**
*******************************************************************************/
@Test
void testMultiPagesWithSomeFailures() throws QException
{
QInstance qInstance = QContext.getQInstance();
//////////////////////////////////////////////////////////////////////////////////////////////////////
// adjust table's automations batch size - as any exceptions thrown put the whole batch into error. //
// so we'll make batches (pages) of 100 - run for 500 records, and make just a couple bad records //
// that'll cause errors - so we should get a few failed pages, and the rest ok. //
//////////////////////////////////////////////////////////////////////////////////////////////////////
int pageSize = 100;
qInstance.getTable(TestUtils.TABLE_NAME_PERSON_MEMORY)
.getAutomationDetails()
.setOverrideBatchSize(pageSize);
//////////////////////////////////////////////////////////////////////////////////
// insert many people - half who should be updated by the AgeChecker automation //
//////////////////////////////////////////////////////////////////////////////////
InsertInput insertInput = new InsertInput();
insertInput.setTableName(TestUtils.TABLE_NAME_PERSON_MEMORY);
insertInput.setRecords(new ArrayList<>());
int SIZE = 500;
for(int i = 0; i < SIZE; i++)
{
insertInput.getRecords().add(new QRecord().withValue("firstName", "Qui Gon").withValue("lastName", "Jinn " + i).withValue("birthDate", LocalDate.now()));
insertInput.getRecords().add(new QRecord().withValue("firstName", "Obi Wan").withValue("lastName", "Kenobi " + i));
/////////////////////////////////
// throw 2 Darths into the mix //
/////////////////////////////////
if(i == 101 || i == 301)
{
insertInput.getRecords().add(new QRecord().withValue("firstName", "Darth").withValue("lastName", "Maul " + i));
}
}
InsertOutput insertOutput = new InsertAction().execute(insertInput);
List<Serializable> insertedIds = insertOutput.getRecords().stream().map(r -> r.getValue("id")).toList();
assertAllRecordsAutomationStatus(AutomationStatus.PENDING_INSERT_AUTOMATIONS);
/////////////////////////
// run the automations //
/////////////////////////
runAllTableActions(qInstance);
////////////////////////////////////////////////////////////////////
// make sure that some records became ok, but others became error //
////////////////////////////////////////////////////////////////////
QueryOutput queryOutput = new QueryAction().execute(new QueryInput(TestUtils.TABLE_NAME_PERSON_MEMORY).withFilter(new QQueryFilter(new QFilterCriteria("id", QCriteriaOperator.IN, insertedIds))));
List<QRecord> okRecords = queryOutput.getRecords().stream().filter(r -> AutomationStatus.OK.getId().equals(r.getValueInteger(TestUtils.standardQqqAutomationStatusField().getName()))).toList();
List<QRecord> failedRecords = queryOutput.getRecords().stream().filter(r -> AutomationStatus.FAILED_INSERT_AUTOMATIONS.getId().equals(r.getValueInteger(TestUtils.standardQqqAutomationStatusField().getName()))).toList();
assertFalse(okRecords.isEmpty(), "Some inserted records should be automation status OK");
assertFalse(failedRecords.isEmpty(), "Some inserted records should be automation status Failed");
assertEquals(insertedIds.size(), okRecords.size() + failedRecords.size(), "All inserted records should be OK or Failed");
///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// make sure that only 2 pages failed - meaning our number of failedRecords is < pageSize * 2 (as any page may be smaller than the pageSize) //
///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
assertThat(failedRecords.size()).isLessThanOrEqualTo(pageSize * 2).describedAs("No more than 2 pages should be in failed status.");
}
/*******************************************************************************
** Test running a process for automation, instead of a code ref.
*******************************************************************************/
@ -367,6 +479,61 @@ class PollingAutomationPerTableRunnerTest extends BaseTest
/*******************************************************************************
**
*******************************************************************************/
@Test
void testServerShutdownMidRunLeavesRecordsInRunningStatus() throws QException
{
QInstance qInstance = QContext.getQInstance();
/////////////////////////////////////////////////////////////////////////////
// insert 2 person records that should have insert action ran against them //
/////////////////////////////////////////////////////////////////////////////
InsertInput insertInput = new InsertInput();
insertInput.setTableName(TestUtils.TABLE_NAME_PERSON_MEMORY);
insertInput.setRecords(List.of(
new QRecord().withValue("id", 1).withValue("firstName", "Tim").withValue("birthDate", LocalDate.now()),
new QRecord().withValue("id", 2).withValue("firstName", "Darin").withValue("birthDate", LocalDate.now())
));
new InsertAction().execute(insertInput);
assertAllRecordsAutomationStatus(AutomationStatus.PENDING_INSERT_AUTOMATIONS);
//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// duplicate the runAllTableActions method - but using the subclass of PollingAutomationPerTableRunner that will throw. //
//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
assertThatThrownBy(() ->
{
List<PollingAutomationPerTableRunner.TableActions> tableActions = PollingAutomationPerTableRunner.getTableActions(qInstance, TestUtils.POLLING_AUTOMATION);
for(PollingAutomationPerTableRunner.TableActions tableAction : tableActions)
{
PollingAutomationPerTableRunner pollingAutomationPerTableRunner = new PollingAutomationPerTableRunnerThatShouldSimulateServerShutdownMidRun(qInstance, TestUtils.POLLING_AUTOMATION, QSession::new, tableAction);
/////////////////////////////////////////////////////////////////////////////////////////////////////
// note - don't call run - it is meant to be called async - e.g., it sets & clears thread context. //
/////////////////////////////////////////////////////////////////////////////////////////////////////
pollingAutomationPerTableRunner.processTableInsertOrUpdate(qInstance.getTable(tableAction.tableName()), QContext.getQSession(), tableAction.status());
}
}).hasMessage(PollingAutomationPerTableRunnerThatShouldSimulateServerShutdownMidRun.EXCEPTION_MESSAGE);
//////////////////////////////////////////////////
// records should be "leaked" in running status //
//////////////////////////////////////////////////
assertAllRecordsAutomationStatus(AutomationStatus.RUNNING_INSERT_AUTOMATIONS);
/////////////////////////////////////
// simulate another run of the job //
/////////////////////////////////////
runAllTableActions(qInstance);
////////////////////////////////////////////////////////////////////////////////
// it should NOT have updated those records - they're officially "leaked" now //
////////////////////////////////////////////////////////////////////////////////
assertAllRecordsAutomationStatus(AutomationStatus.RUNNING_INSERT_AUTOMATIONS);
}
/*******************************************************************************
**
*******************************************************************************/
@ -376,4 +543,42 @@ class PollingAutomationPerTableRunnerTest extends BaseTest
.isNotEmpty()
.allMatch(r -> pendingInsertAutomations.getId().equals(r.getValue(TestUtils.standardQqqAutomationStatusField().getName())));
}
/*******************************************************************************
** this subclass of the class under test allows us to simulate:
**
** what happens if, after records have been marked as running-updates, if,
** for example, a server shuts down?
**
** It does this by overriding a method that runs between those points in time,
** and throwing a runtime exception.
*******************************************************************************/
public static class PollingAutomationPerTableRunnerThatShouldSimulateServerShutdownMidRun extends PollingAutomationPerTableRunner
{
private static String EXCEPTION_MESSAGE = "Throwing outside of catch here, to simulate a server shutdown mid-run";
/*******************************************************************************
**
*******************************************************************************/
public PollingAutomationPerTableRunnerThatShouldSimulateServerShutdownMidRun(QInstance instance, String providerName, Supplier<QSession> sessionSupplier, TableActions tableActions)
{
super(instance, providerName, sessionSupplier, tableActions);
}
/*******************************************************************************
**
*******************************************************************************/
@Override
protected boolean applyActionToRecords(QTableMetaData table, List<QRecord> records, TableAutomationAction action)
{
throw (new RuntimeException(EXCEPTION_MESSAGE));
}
}
}

39
qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java
View File

@ -34,6 +34,7 @@ import com.kingsrook.qqq.backend.core.actions.processes.BackendStep;
import com.kingsrook.qqq.backend.core.actions.processes.person.addtopeoplesage.AddAge;
import com.kingsrook.qqq.backend.core.actions.processes.person.addtopeoplesage.GetAgeStatistics;
import com.kingsrook.qqq.backend.core.actions.tables.CountAction;
import com.kingsrook.qqq.backend.core.actions.tables.GetAction;
import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction;
@ -45,6 +46,7 @@ import com.kingsrook.qqq.backend.core.logging.QLogger;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
@ -813,6 +815,11 @@ public class TestUtils
.withFilter(new QQueryFilter().withCriteria(new QFilterCriteria("birthDate", QCriteriaOperator.GREATER_THAN, List.of(youngPersonLimitDate))))
.withCodeReference(new QCodeReference(CheckAge.class))
)
.withAction(new TableAutomationAction()
.withName("failAutomationForSith")
.withTriggerEvent(TriggerEvent.POST_INSERT)
.withCodeReference(new QCodeReference(FailAutomationForSith.class))
)
.withAction(new TableAutomationAction()
.withName("increaseBirthdate")
.withTriggerEvent(TriggerEvent.POST_INSERT)
@ -918,6 +925,15 @@ public class TestUtils
List<QRecord> recordsToUpdate = new ArrayList<>();
for(QRecord record : recordAutomationInput.getRecordList())
{
////////////////////////////////////////////////////////////////////////
// get the record - its automation status should currently be RUNNING //
////////////////////////////////////////////////////////////////////////
QRecord freshlyFetchedRecord = new GetAction().executeForRecord(new GetInput(TABLE_NAME_PERSON_MEMORY).withPrimaryKey(record.getValue("id")));
assertEquals(AutomationStatus.RUNNING_INSERT_AUTOMATIONS.getId(), freshlyFetchedRecord.getValueInteger(TestUtils.standardQqqAutomationStatusField().getName()));
///////////////////////////////////////////
// do whatever business logic we do here //
///////////////////////////////////////////
LocalDate birthDate = record.getValueLocalDate("birthDate");
if(birthDate != null && birthDate.isAfter(limitDate))
{
@ -940,6 +956,29 @@ public class TestUtils
/*******************************************************************************
**
*******************************************************************************/
public static class FailAutomationForSith extends RecordAutomationHandler
{
/*******************************************************************************
**
*******************************************************************************/
public void execute(RecordAutomationInput recordAutomationInput) throws QException
{
for(QRecord record : recordAutomationInput.getRecordList())
{
if("Darth".equals(record.getValue("firstName")))
{
throw new QException("Oops, you look like a Sith!");
}
}
}
}
/*******************************************************************************
**
*******************************************************************************/