QRunIO/qqq
2
0
Fork 0
mirror of https://github.com/Kingsrook/qqq.git synced 2025-07-18 13:10:44 +00:00
Code Issues Packages Projects Releases Wiki Activity

Build audits when running Record Scripts; add script name to audit context; clean up some bogus 'changed x to x' messages.

Browse Source
This commit is contained in:
darin.kelkhoff
2023-09-21 14:42:01 -05:00
parent 3e8afde744
commit 1c99ea2c6f
6 changed files with 599 additions and 140 deletions
Download Patch File Download Diff File Expand all files Collapse all files

58
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/audits/AuditAction.java
View File

@ -29,6 +29,7 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import com.kingsrook.qqq.backend.core.actions.AbstractQActionFunction;
import com.kingsrook.qqq.backend.core.actions.tables.GetAction;
import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
@ -49,6 +50,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.model.session.QUser;
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
import com.kingsrook.qqq.backend.core.utils.Pair;
import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
/*******************************************************************************
@ -108,12 +110,26 @@ public class AuditAction extends AbstractQActionFunction<AuditInput, AuditOutput
/*******************************************************************************
** Simple overload that internally figures out primary key and security key values
**
** Be aware - if the record doesn't have its security key values set (say it's a
** partial record as part of an update), then those values won't be in the
** security key map... This should probably be considered a bug.
*******************************************************************************/
public static void appendToInput(AuditInput auditInput, QTableMetaData table, QRecord record, String auditMessage)
{
appendToInput(auditInput, table.getName(), record.getValueInteger(table.getPrimaryKeyField()), getRecordSecurityKeyValues(table, record, Optional.empty()), auditMessage);
}
/*******************************************************************************
** Add 1 auditSingleInput to an AuditInput object - with no details (child records).
*******************************************************************************/
public static AuditInput appendToInput(AuditInput auditInput, String tableName, Integer recordId, Map<String, Serializable> securityKeyValues, String message)
public static void appendToInput(AuditInput auditInput, String tableName, Integer recordId, Map<String, Serializable> securityKeyValues, String message)
{
return (appendToInput(auditInput, tableName, recordId, securityKeyValues, message, null));
appendToInput(auditInput, tableName, recordId, securityKeyValues, message, null);
}
@ -139,6 +155,44 @@ public class AuditAction extends AbstractQActionFunction<AuditInput, AuditOutput
/*******************************************************************************
** For a given record, from a given table, build a map of the record's security
** key values.
**
** If, in case, the record has null value(s), and the oldRecord is given (e.g.,
** for the case of an update, where the record may not have all fields set, and
** oldRecord should be known for doing field-diffs), then try to get the value(s)
** from oldRecord.
**
** Currently, will leave values null if they aren't found after that.
**
** An alternative could be to re-fetch the record from its source if needed...
*******************************************************************************/
public static Map<String, Serializable> getRecordSecurityKeyValues(QTableMetaData table, QRecord record, Optional<QRecord> oldRecord)
{
Map<String, Serializable> securityKeyValues = new HashMap<>();
for(RecordSecurityLock recordSecurityLock : RecordSecurityLockFilters.filterForReadLocks(CollectionUtils.nonNullList(table.getRecordSecurityLocks())))
{
Serializable keyValue = record == null ? null : record.getValue(recordSecurityLock.getFieldName());
if(keyValue == null && oldRecord.isPresent())
{
LOG.debug("Table with a securityLock, but value not found in field", logPair("table", table.getName()), logPair("field", recordSecurityLock.getFieldName()));
keyValue = oldRecord.get().getValue(recordSecurityLock.getFieldName());
}
if(keyValue == null)
{
LOG.debug("Table with a securityLock, but value not found in field", logPair("table", table.getName()), logPair("field", recordSecurityLock.getFieldName()), logPair("oldRecordIsPresent", oldRecord.isPresent()));
}
securityKeyValues.put(recordSecurityLock.getSecurityKeyType(), keyValue);
}
return securityKeyValues;
}
/*******************************************************************************
**
*******************************************************************************/

395
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/audits/DMLAuditAction.java
View File

@ -23,8 +23,10 @@ package com.kingsrook.qqq.backend.core.actions.audits;
import java.io.Serializable;
import java.math.BigDecimal;
import java.time.Instant;
import java.time.ZoneId;
import java.time.temporal.ChronoUnit;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.HashMap;
@ -52,13 +54,12 @@ import com.kingsrook.qqq.backend.core.model.metadata.audits.AuditLevel;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.security.RecordSecurityLock;
import com.kingsrook.qqq.backend.core.model.metadata.security.RecordSecurityLockFilters;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.model.session.QSession;
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
import com.kingsrook.qqq.backend.core.utils.StringUtils;
import com.kingsrook.qqq.backend.core.utils.ValueUtils;
import static com.kingsrook.qqq.backend.core.actions.audits.AuditAction.getRecordSecurityKeyValues;
import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
@ -70,6 +71,8 @@ public class DMLAuditAction extends AbstractQActionFunction<DMLAuditInput, DMLAu
{
private static final QLogger LOG = QLogger.getLogger(DMLAuditAction.class);
public static final String AUDIT_CONTEXT_FIELD_NAME = "auditContext";
/*******************************************************************************
@ -99,34 +102,7 @@ public class DMLAuditAction extends AbstractQActionFunction<DMLAuditInput, DMLAu
return (output);
}
String contextSuffix = "";
if(StringUtils.hasContent(input.getAuditContext()))
{
contextSuffix = " " + input.getAuditContext();
}
Optional<AbstractActionInput> actionInput = QContext.getFirstActionInStack();
if(actionInput.isPresent() && actionInput.get() instanceof RunProcessInput runProcessInput)
{
String processName = runProcessInput.getProcessName();
QProcessMetaData process = QContext.getQInstance().getProcess(processName);
if(process != null)
{
contextSuffix = " during process: " + process.getLabel();
}
}
QSession qSession = QContext.getQSession();
String apiVersion = qSession.getValue("apiVersion");
if(apiVersion != null)
{
String apiLabel = qSession.getValue("apiLabel");
if(!StringUtils.hasContent(apiLabel))
{
apiLabel = "API";
}
contextSuffix += (" via " + apiLabel + " Version: " + apiVersion);
}
String contextSuffix = getContentSuffix(input);
AuditInput auditInput = new AuditInput();
if(auditLevel.equals(AuditLevel.RECORD) || (auditLevel.equals(AuditLevel.FIELD) && !dmlType.supportsFields))
@ -137,7 +113,7 @@ public class DMLAuditAction extends AbstractQActionFunction<DMLAuditInput, DMLAu
////////////////////////////////////////////////////////////////////////////////////////////////////////////
for(QRecord record : recordList)
{
AuditAction.appendToInput(auditInput, table.getName(), record.getValueInteger(table.getPrimaryKeyField()), getRecordSecurityKeyValues(table, record), "Record was " + dmlType.pastTenseVerb + contextSuffix);
AuditAction.appendToInput(auditInput, table.getName(), record.getValueInteger(table.getPrimaryKeyField()), getRecordSecurityKeyValues(table, record, Optional.empty()), "Record was " + dmlType.pastTenseVerb + contextSuffix);
}
}
else if(auditLevel.equals(AuditLevel.FIELD))
@ -147,7 +123,7 @@ public class DMLAuditAction extends AbstractQActionFunction<DMLAuditInput, DMLAu
///////////////////////////////////////////////////////////////////
// do many audits, all with field level details, for FIELD level //
///////////////////////////////////////////////////////////////////
QPossibleValueTranslator qPossibleValueTranslator = new QPossibleValueTranslator(QContext.getQInstance(), qSession);
QPossibleValueTranslator qPossibleValueTranslator = new QPossibleValueTranslator(QContext.getQInstance(), QContext.getQSession());
qPossibleValueTranslator.translatePossibleValuesInRecords(table, CollectionUtils.mergeLists(recordList, oldRecordList));
//////////////////////////////////////////
@ -169,92 +145,8 @@ public class DMLAuditAction extends AbstractQActionFunction<DMLAuditInput, DMLAu
List<QRecord> details = new ArrayList<>();
for(String fieldName : sortedFieldNames)
{
if(!record.getValues().containsKey(fieldName))
{
////////////////////////////////////////////////////////////////////////////////////////////////
// if the stored record doesn't have this field name, then don't audit anything about it //
// this is to deal with our Patch style updates not looking like every field was cleared out. //
////////////////////////////////////////////////////////////////////////////////////////////////
continue;
}
if(fieldName.equals("modifyDate") || fieldName.equals("createDate") || fieldName.equals("automationStatus"))
{
continue;
}
QFieldMetaData field = table.getField(fieldName);
Serializable value = ValueUtils.getValueAsFieldType(field.getType(), record.getValue(fieldName));
Serializable oldValue = oldRecord == null ? null : ValueUtils.getValueAsFieldType(field.getType(), oldRecord.getValue(fieldName));
QRecord detailRecord = null;
if(oldRecord == null)
{
if(DMLType.INSERT.equals(dmlType) && value == null)
{
continue;
}
if(field.getType().equals(QFieldType.BLOB) || field.getType().needsMasked())
{
detailRecord = new QRecord().withValue("message", "Set " + field.getLabel());
}
else
{
String formattedValue = getFormattedValueForAuditDetail(record, fieldName, field, value);
detailRecord = new QRecord().withValue("message", "Set " + field.getLabel() + " to " + formattedValue);
detailRecord.withValue("newValue", formattedValue);
}
}
else
{
if(!Objects.equals(oldValue, value))
{
if(field.getType().equals(QFieldType.BLOB) || field.getType().needsMasked())
{
if(oldValue == null)
{
detailRecord = new QRecord().withValue("message", "Set " + field.getLabel());
}
else if(value == null)
{
detailRecord = new QRecord().withValue("message", "Removed " + field.getLabel());
}
else
{
detailRecord = new QRecord().withValue("message", "Changed " + field.getLabel());
}
}
else
{
String formattedValue = getFormattedValueForAuditDetail(record, fieldName, field, value);
String formattedOldValue = getFormattedValueForAuditDetail(oldRecord, fieldName, field, oldValue);
if(oldValue == null)
{
detailRecord = new QRecord().withValue("message", "Set " + field.getLabel() + " to " + formatFormattedValueForDetailMessage(field, formattedValue));
detailRecord.withValue("newValue", formattedValue);
}
else if(value == null)
{
detailRecord = new QRecord().withValue("message", "Removed " + formatFormattedValueForDetailMessage(field, formattedOldValue) + " from " + field.getLabel());
detailRecord.withValue("oldValue", formattedOldValue);
}
else
{
detailRecord = new QRecord().withValue("message", "Changed " + field.getLabel() + " from " + formatFormattedValueForDetailMessage(field, formattedOldValue) + " to " + formatFormattedValueForDetailMessage(field, formattedValue));
detailRecord.withValue("oldValue", formattedOldValue);
detailRecord.withValue("newValue", formattedValue);
}
}
}
}
if(detailRecord != null)
{
detailRecord.withValue("fieldName", fieldName);
details.add(detailRecord);
}
makeAuditDetailRecordForField(fieldName, table, dmlType, record, oldRecord)
.ifPresent(details::add);
}
if(details.isEmpty() && DMLType.UPDATE.equals(dmlType))
@ -264,7 +156,7 @@ public class DMLAuditAction extends AbstractQActionFunction<DMLAuditInput, DMLAu
}
else
{
AuditAction.appendToInput(auditInput, table.getName(), record.getValueInteger(table.getPrimaryKeyField()), getRecordSecurityKeyValues(table, record), "Record was " + dmlType.pastTenseVerb + contextSuffix, details);
AuditAction.appendToInput(auditInput, table.getName(), record.getValueInteger(table.getPrimaryKeyField()), getRecordSecurityKeyValues(table, record, Optional.ofNullable(oldRecord)), "Record was " + dmlType.pastTenseVerb + contextSuffix, details);
}
}
}
@ -284,6 +176,254 @@ public class DMLAuditAction extends AbstractQActionFunction<DMLAuditInput, DMLAu
/*******************************************************************************
**
*******************************************************************************/
static String getContentSuffix(DMLAuditInput input)
{
StringBuilder contextSuffix = new StringBuilder();
/////////////////////////////////////////////////////////////////////////////
// start with context from the input wrapper //
// note, these contexts get propagated down from Input/Update/Delete Input //
/////////////////////////////////////////////////////////////////////////////
if(StringUtils.hasContent(input.getAuditContext()))
{
contextSuffix.append(" ").append(input.getAuditContext());
}
/////////////////////////////////////////////////////////////////////////////////////
// note process label (and a possible context from the process's state) if present //
/////////////////////////////////////////////////////////////////////////////////////
Optional<AbstractActionInput> actionInput = QContext.getFirstActionInStack();
if(actionInput.isPresent() && actionInput.get() instanceof RunProcessInput runProcessInput)
{
String processAuditContext = ValueUtils.getValueAsString(runProcessInput.getValue(AUDIT_CONTEXT_FIELD_NAME));
if(StringUtils.hasContent(processAuditContext))
{
contextSuffix.append(" ").append(processAuditContext);
}
String processName = runProcessInput.getProcessName();
QProcessMetaData process = QContext.getQInstance().getProcess(processName);
if(process != null)
{
contextSuffix.append(" during process: ").append(process.getLabel());
}
}
///////////////////////////////////////////////////
// use api label & version if present in session //
///////////////////////////////////////////////////
QSession qSession = QContext.getQSession();
String apiVersion = qSession.getValue("apiVersion");
if(apiVersion != null)
{
String apiLabel = qSession.getValue("apiLabel");
if(!StringUtils.hasContent(apiLabel))
{
apiLabel = "API";
}
contextSuffix.append(" via ").append(apiLabel).append(" Version: ").append(apiVersion);
}
return (contextSuffix.toString());
}
/*******************************************************************************
**
*******************************************************************************/
static Optional<QRecord> makeAuditDetailRecordForField(String fieldName, QTableMetaData table, DMLType dmlType, QRecord record, QRecord oldRecord)
{
if(!record.getValues().containsKey(fieldName))
{
////////////////////////////////////////////////////////////////////////////////////////////////
// if the stored record doesn't have this field name, then don't audit anything about it //
// this is to deal with our Patch style updates not looking like every field was cleared out. //
////////////////////////////////////////////////////////////////////////////////////////////////
return (Optional.empty());
}
if(fieldName.equals("modifyDate") || fieldName.equals("createDate") || fieldName.equals("automationStatus"))
{
return (Optional.empty());
}
QFieldMetaData field = table.getField(fieldName);
Serializable value = ValueUtils.getValueAsFieldType(field.getType(), record.getValue(fieldName));
Serializable oldValue = oldRecord == null ? null : ValueUtils.getValueAsFieldType(field.getType(), oldRecord.getValue(fieldName));
QRecord detailRecord = null;
if(oldRecord == null)
{
if(DMLType.INSERT.equals(dmlType) && value == null)
{
return (Optional.empty());
}
if(field.getType().equals(QFieldType.BLOB) || field.getType().needsMasked())
{
detailRecord = new QRecord().withValue("message", "Set " + field.getLabel());
}
else
{
String formattedValue = getFormattedValueForAuditDetail(record, fieldName, field, value);
detailRecord = new QRecord().withValue("message", "Set " + field.getLabel() + " to " + formattedValue);
detailRecord.withValue("newValue", formattedValue);
}
}
else
{
if(areValuesDifferentForAudit(field, value, oldValue))
{
if(field.getType().equals(QFieldType.BLOB) || field.getType().needsMasked())
{
if(oldValue == null)
{
detailRecord = new QRecord().withValue("message", "Set " + field.getLabel());
}
else if(value == null)
{
detailRecord = new QRecord().withValue("message", "Removed " + field.getLabel());
}
else
{
detailRecord = new QRecord().withValue("message", "Changed " + field.getLabel());
}
}
else
{
String formattedValue = getFormattedValueForAuditDetail(record, fieldName, field, value);
String formattedOldValue = getFormattedValueForAuditDetail(oldRecord, fieldName, field, oldValue);
if(oldValue == null)
{
detailRecord = new QRecord().withValue("message", "Set " + field.getLabel() + " to " + formatFormattedValueForDetailMessage(field, formattedValue));
detailRecord.withValue("newValue", formattedValue);
}
else if(value == null)
{
detailRecord = new QRecord().withValue("message", "Removed " + formatFormattedValueForDetailMessage(field, formattedOldValue) + " from " + field.getLabel());
detailRecord.withValue("oldValue", formattedOldValue);
}
else
{
detailRecord = new QRecord().withValue("message", "Changed " + field.getLabel() + " from " + formatFormattedValueForDetailMessage(field, formattedOldValue) + " to " + formatFormattedValueForDetailMessage(field, formattedValue));
detailRecord.withValue("oldValue", formattedOldValue);
detailRecord.withValue("newValue", formattedValue);
}
}
}
}
if(detailRecord != null)
{
LOG.debug("Returning with message: " + detailRecord.getValueString("message"));
detailRecord.withValue("fieldName", fieldName);
return (Optional.of(detailRecord));
}
return (Optional.empty());
}
/*******************************************************************************
**
*******************************************************************************/
static boolean areValuesDifferentForAudit(QFieldMetaData field, Serializable value, Serializable oldValue)
{
try
{
///////////////////
// decimal rules //
///////////////////
if(field.getType().equals(QFieldType.DECIMAL))
{
BigDecimal newBD = ValueUtils.getValueAsBigDecimal(value);
BigDecimal oldBD = ValueUtils.getValueAsBigDecimal(oldValue);
if(newBD == null && oldBD == null)
{
return (false);
}
if(newBD == null || oldBD == null)
{
return (true);
}
return (newBD.compareTo(oldBD) != 0);
}
////////////////////
// dateTime rules //
////////////////////
if(field.getType().equals(QFieldType.DATE_TIME))
{
Instant newI = ValueUtils.getValueAsInstant(value);
Instant oldI = ValueUtils.getValueAsInstant(oldValue);
if(newI == null && oldI == null)
{
return (false);
}
if(newI == null || oldI == null)
{
return (true);
}
////////////////////////////////
// just compare to the second //
////////////////////////////////
return (newI.truncatedTo(ChronoUnit.SECONDS).compareTo(oldI.truncatedTo(ChronoUnit.SECONDS)) != 0);
}
//////////////////
// string rules //
//////////////////
if(field.getType().isStringLike())
{
String newString = ValueUtils.getValueAsString(value);
String oldString = ValueUtils.getValueAsString(oldValue);
boolean newIsNullOrEmpty = !StringUtils.hasContent(newString);
boolean oldIsNullOrEmpty = !StringUtils.hasContent(oldString);
if(newIsNullOrEmpty && oldIsNullOrEmpty)
{
return (false);
}
if(newIsNullOrEmpty || oldIsNullOrEmpty)
{
return (true);
}
return (newString.compareTo(oldString) != 0);
}
/////////////////////////////////////
// default just use Objects.equals //
/////////////////////////////////////
return !Objects.equals(oldValue, value);
}
catch(Exception e)
{
LOG.debug("Error checking areValuesDifferentForAudit", e, logPair("fieldName", field.getName()), logPair("value", value), logPair("oldValue", oldValue));
}
////////////////////////////////////
// default to something simple... //
////////////////////////////////////
return !Objects.equals(oldValue, value);
}
/*******************************************************************************
**
*******************************************************************************/
@ -373,21 +513,6 @@ public class DMLAuditAction extends AbstractQActionFunction<DMLAuditInput, DMLAu
/*******************************************************************************
**
*******************************************************************************/
private static Map<String, Serializable> getRecordSecurityKeyValues(QTableMetaData table, QRecord record)
{
Map<String, Serializable> securityKeyValues = new HashMap<>();
for(RecordSecurityLock recordSecurityLock : RecordSecurityLockFilters.filterForReadLocks(CollectionUtils.nonNullList(table.getRecordSecurityLocks())))
{
securityKeyValues.put(recordSecurityLock.getSecurityKeyType(), record == null ? null : record.getValue(recordSecurityLock.getFieldName()));
}
return securityKeyValues;
}
/*******************************************************************************
**
*******************************************************************************/
@ -407,7 +532,7 @@ public class DMLAuditAction extends AbstractQActionFunction<DMLAuditInput, DMLAu
/*******************************************************************************
**
*******************************************************************************/
private enum DMLType
enum DMLType
{
INSERT("Inserted", true),
UPDATE("Edited", true),

53
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/scripts/RunRecordScriptLoadStep.java
View File

@ -25,11 +25,15 @@ package com.kingsrook.qqq.backend.core.processes.implementations.scripts;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.List;
import com.kingsrook.qqq.backend.core.actions.audits.AuditAction;
import com.kingsrook.qqq.backend.core.actions.audits.DMLAuditAction;
import com.kingsrook.qqq.backend.core.actions.scripts.RunAdHocRecordScriptAction;
import com.kingsrook.qqq.backend.core.actions.scripts.logging.StoreScriptLogAndScriptLogLineExecutionLogger;
import com.kingsrook.qqq.backend.core.actions.tables.GetAction;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.logging.QLogger;
import com.kingsrook.qqq.backend.core.model.actions.audits.AuditInput;
import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryFilterLink;
import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLine;
import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLineInterface;
@ -42,7 +46,9 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetOutput;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.code.AdHocScriptCodeReference;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.model.scripts.Script;
import com.kingsrook.qqq.backend.core.model.scripts.ScriptLog;
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.AbstractLoadStep;
@ -121,16 +127,25 @@ public class RunRecordScriptLoadStep extends AbstractLoadStep implements Process
getInput.setTableName(Script.TABLE_NAME);
getInput.setPrimaryKey(scriptId);
GetOutput getOutput = new GetAction().execute(getInput);
if(getOutput.getRecord() == null)
QRecord script = getOutput.getRecord();
if(script == null)
{
throw (new QException("Could not find script by id: " + scriptId));
}
//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// set an "audit context" - so any DML executed during the script will include the note of what script was running. //
//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
runBackendStepInput.addValue(DMLAuditAction.AUDIT_CONTEXT_FIELD_NAME, "via Script \"" + script.getValue("name") + "\"");
String tableName = script.getValueString("tableName");
RunAdHocRecordScriptInput input = new RunAdHocRecordScriptInput();
input.setRecordList(runBackendStepInput.getRecords());
input.setCodeReference(new AdHocScriptCodeReference().withScriptId(scriptId));
input.setTableName(getOutput.getRecord().getValueString("tableName"));
input.setTableName(tableName);
input.setLogger(scriptLogger);
RunAdHocRecordScriptOutput output = new RunAdHocRecordScriptOutput();
Exception caughtException = null;
try
@ -147,11 +162,17 @@ public class RunRecordScriptLoadStep extends AbstractLoadStep implements Process
caughtException = e;
}
String auditMessage = "Script \"" + script.getValueString("name") + "\" (id: " + scriptId + ") was executed against this record";
//////////////////////////////////////////////////////////
// add the record to the appropriate processSummaryLine //
//////////////////////////////////////////////////////////
if(scriptLogger.getScriptLog() != null)
{
Integer id = scriptLogger.getScriptLog().getValueInteger("id");
if(id != null)
{
auditMessage += ", creating script log: " + id;
boolean hadError = BooleanUtils.isTrue(scriptLogger.getScriptLog().getValueBoolean("hadError"));
(hadError ? errorScriptLogIds : okScriptLogIds).add(id);
}
@ -160,6 +181,34 @@ public class RunRecordScriptLoadStep extends AbstractLoadStep implements Process
{
unloggedExceptionLine.incrementCount(runBackendStepInput.getRecords().size());
}
////////////////////////////////////////////////////////////
// audit that the script was executed against the records //
////////////////////////////////////////////////////////////
audit(runBackendStepInput, tableName, auditMessage);
}
/*******************************************************************************
** for each input record, add an audit stating that the script was executed.
*******************************************************************************/
private static void audit(RunBackendStepInput runBackendStepInput, String tableName, String auditMessage)
{
try
{
QTableMetaData table = QContext.getQInstance().getTable(tableName);
AuditInput auditInput = new AuditInput();
for(QRecord record : runBackendStepInput.getRecords())
{
AuditAction.appendToInput(auditInput, table, record, auditMessage);
}
new AuditAction().execute(auditInput);
}
catch(Exception e)
{
LOG.warn("Error recording audits after running record script", e, logPair("tableName", tableName), logPair("auditMessage", auditMessage));
}
}

2
qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/BaseTest.java
View File

@ -48,6 +48,8 @@ public class BaseTest
@BeforeEach
void baseBeforeEach()
{
System.setProperty("qqq.logger.logSessionId.disabled", "true");
QContext.init(TestUtils.defineInstance(), new QSession()
.withUser(new QUser()
.withIdReference("001")

29
qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/audits/AuditActionTest.java
View File

@ -31,6 +31,7 @@ import com.kingsrook.qqq.backend.core.BaseTest;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.audits.AuditInput;
import com.kingsrook.qqq.backend.core.model.actions.audits.AuditSingleInput;
import com.kingsrook.qqq.backend.core.model.audits.AuditsMetaDataProvider;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
@ -38,6 +39,7 @@ import com.kingsrook.qqq.backend.core.model.session.QSession;
import com.kingsrook.qqq.backend.core.model.session.QUser;
import com.kingsrook.qqq.backend.core.processes.utils.GeneralProcessUtils;
import com.kingsrook.qqq.backend.core.utils.TestUtils;
import com.kingsrook.qqq.backend.core.utils.collections.MapBuilder;
import org.junit.jupiter.api.Test;
import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.jupiter.api.Assertions.assertEquals;
@ -197,4 +199,31 @@ class AuditActionTest extends BaseTest
assertThat(auditDetails).anyMatch(r -> r.getValueString("message").equals("Detail3"));
}
/*******************************************************************************
**
*******************************************************************************/
@Test
void testAppendToInputThatTakesRecordNotIdAndSecurityKeyValues()
{
AuditInput auditInput = new AuditInput();
//////////////////////////////////////////////////////////////
// make sure the recordId & securityKey got build correctly //
//////////////////////////////////////////////////////////////
AuditAction.appendToInput(auditInput, QContext.getQInstance().getTable(TestUtils.TABLE_NAME_ORDER), new QRecord().withValue("id", 47).withValue("storeId", 42), "Test");
AuditSingleInput auditSingleInput = auditInput.getAuditSingleInputList().get(0);
assertEquals(47, auditSingleInput.getRecordId());
assertEquals(MapBuilder.of(TestUtils.SECURITY_KEY_TYPE_STORE, 42), auditSingleInput.getSecurityKeyValues());
///////////////////////////////////////////////////////////////////////////////////////////
// acknowledge that we might get back a null key value if the record doesn't have it set //
///////////////////////////////////////////////////////////////////////////////////////////
AuditAction.appendToInput(auditInput, QContext.getQInstance().getTable(TestUtils.TABLE_NAME_ORDER), new QRecord().withValue("id", 47), "Test");
auditSingleInput = auditInput.getAuditSingleInputList().get(1);
assertEquals(47, auditSingleInput.getRecordId());
assertEquals(MapBuilder.of(TestUtils.SECURITY_KEY_TYPE_STORE, null), auditSingleInput.getSecurityKeyValues());
}
}

202
qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/audits/DMLAuditActionTest.java
View File

@ -22,12 +22,14 @@
package com.kingsrook.qqq.backend.core.actions.audits;
import java.math.BigDecimal;
import java.time.Instant;
import java.util.List;
import com.kingsrook.qqq.backend.core.BaseTest;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.audits.DMLAuditInput;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
@ -36,10 +38,18 @@ import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.audits.AuditLevel;
import com.kingsrook.qqq.backend.core.model.metadata.audits.QAuditRules;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.model.session.QSession;
import com.kingsrook.qqq.backend.core.model.statusmessages.BadInputStatusMessage;
import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryRecordStore;
import com.kingsrook.qqq.backend.core.utils.TestUtils;
import com.kingsrook.qqq.backend.core.utils.ValueUtils;
import org.junit.jupiter.api.Test;
import static com.kingsrook.qqq.backend.core.actions.audits.DMLAuditAction.DMLType.INSERT;
import static com.kingsrook.qqq.backend.core.actions.audits.DMLAuditAction.DMLType.UPDATE;
import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNull;
@ -200,4 +210,194 @@ class DMLAuditActionTest extends BaseTest
}
}
}
/*******************************************************************************
**
*******************************************************************************/
@Test
void testMakeAuditDetailRecordForField()
{
QTableMetaData table = new QTableMetaData()
.withField(new QFieldMetaData("createDate", QFieldType.DATE_TIME).withLabel("Create Date"))
.withField(new QFieldMetaData("modifyDate", QFieldType.DATE_TIME).withLabel("Modify Date"))
.withField(new QFieldMetaData("someTimestamp", QFieldType.DATE_TIME).withLabel("Some Timestamp"))
.withField(new QFieldMetaData("name", QFieldType.STRING).withLabel("Name"))
.withField(new QFieldMetaData("seqNo", QFieldType.INTEGER).withLabel("Sequence No."))
.withField(new QFieldMetaData("price", QFieldType.DECIMAL).withLabel("Price"));
///////////////////////////////
// create date - never audit //
///////////////////////////////
assertThat(DMLAuditAction.makeAuditDetailRecordForField("createDate", table, INSERT,
new QRecord().withValue("createDate", Instant.now()),
new QRecord().withValue("createDate", Instant.now().minusSeconds(100))))
.isEmpty();
///////////////////////////////
// modify date - never audit //
///////////////////////////////
assertThat(DMLAuditAction.makeAuditDetailRecordForField("modifyDate", table, UPDATE,
new QRecord().withValue("modifyDate", Instant.now()),
new QRecord().withValue("modifyDate", Instant.now().minusSeconds(100))))
.isEmpty();
////////////////////////////////////////////////////////
// datetime different only in precision - don't audit //
////////////////////////////////////////////////////////
assertThat(DMLAuditAction.makeAuditDetailRecordForField("someTimestamp", table, UPDATE,
new QRecord().withValue("someTimestamp", ValueUtils.getValueAsInstant("2023-04-17T14:33:08.777")),
new QRecord().withValue("someTimestamp", Instant.parse("2023-04-17T14:33:08Z"))))
.isEmpty();
/////////////////////////////////////////////
// datetime actually different - audit it. //
/////////////////////////////////////////////
assertThat(DMLAuditAction.makeAuditDetailRecordForField("someTimestamp", table, UPDATE,
new QRecord().withValue("someTimestamp", Instant.parse("2023-04-17T14:33:09Z")),
new QRecord().withValue("someTimestamp", Instant.parse("2023-04-17T14:33:08Z"))))
.isPresent()
.get().extracting(r -> r.getValueString("message"))
.matches(s -> s.matches("Changed Some Timestamp from 2023.* to 2023.*"));
////////////////////////////////////////////////
// datetime changing null to not null - audit //
////////////////////////////////////////////////
assertThat(DMLAuditAction.makeAuditDetailRecordForField("someTimestamp", table, UPDATE,
new QRecord().withValue("someTimestamp", ValueUtils.getValueAsInstant("2023-04-17T14:33:08.777")),
new QRecord().withValue("someTimestamp", null)))
.isPresent()
.get().extracting(r -> r.getValueString("message"))
.matches(s -> s.matches("Set Some Timestamp to 2023.*"));
////////////////////////////////////////////////
// datetime changing not null to null - audit //
////////////////////////////////////////////////
assertThat(DMLAuditAction.makeAuditDetailRecordForField("someTimestamp", table, UPDATE,
new QRecord().withValue("someTimestamp", null),
new QRecord().withValue("someTimestamp", Instant.parse("2023-04-17T14:33:08Z"))))
.isPresent()
.get().extracting(r -> r.getValueString("message"))
.matches(s -> s.matches("Removed 2023.*from Some Timestamp"));
////////////////////////////////////////
// string that is the same - no audit //
////////////////////////////////////////
assertThat(DMLAuditAction.makeAuditDetailRecordForField("name", table, UPDATE,
new QRecord().withValue("name", "Homer"),
new QRecord().withValue("name", "Homer")))
.isEmpty();
//////////////////////////////////////////
// string from null to empty - no audit //
//////////////////////////////////////////
assertThat(DMLAuditAction.makeAuditDetailRecordForField("name", table, UPDATE,
new QRecord().withValue("name", null),
new QRecord().withValue("name", "")))
.isEmpty();
//////////////////////////////////////////
// string from empty to null - no audit //
//////////////////////////////////////////
assertThat(DMLAuditAction.makeAuditDetailRecordForField("name", table, UPDATE,
new QRecord().withValue("name", ""),
new QRecord().withValue("name", null)))
.isEmpty();
//////////////////////////////////////////////////////////
// decimal that only changes in precision - don't audit //
//////////////////////////////////////////////////////////
assertThat(DMLAuditAction.makeAuditDetailRecordForField("price", table, UPDATE,
new QRecord().withValue("price", "10"),
new QRecord().withValue("price", new BigDecimal("10.00"))))
.isEmpty();
//////////////////////////////////////////////////
// decimal that's actually different - do audit //
//////////////////////////////////////////////////
assertThat(DMLAuditAction.makeAuditDetailRecordForField("price", table, UPDATE,
new QRecord().withValue("price", "10.01"),
new QRecord().withValue("price", new BigDecimal("10.00"))))
.isPresent()
.get().extracting(r -> r.getValueString("message"))
.matches(s -> s.matches("Changed Price from 10.00 to 10.01"));
///////////////////////////////////////
// decimal null, input "" - no audit //
///////////////////////////////////////
assertThat(DMLAuditAction.makeAuditDetailRecordForField("price", table, UPDATE,
new QRecord().withValue("price", ""),
new QRecord().withValue("price", null)))
.isEmpty();
/////////////////////////////////////////
// decimal not-null to null - do audit //
/////////////////////////////////////////
assertThat(DMLAuditAction.makeAuditDetailRecordForField("price", table, UPDATE,
new QRecord().withValue("price", BigDecimal.ONE),
new QRecord().withValue("price", null)))
.isPresent()
.get().extracting(r -> r.getValueString("message"))
.matches(s -> s.matches("Set Price to 1"));
/////////////////////////////////////////
// decimal null to not-null - do audit //
/////////////////////////////////////////
assertThat(DMLAuditAction.makeAuditDetailRecordForField("price", table, UPDATE,
new QRecord().withValue("price", null),
new QRecord().withValue("price", BigDecimal.ONE)))
.isPresent()
.get().extracting(r -> r.getValueString("message"))
.matches(s -> s.matches("Removed 1 from Price"));
///////////////////////////////////////
// integer null, input "" - no audit //
///////////////////////////////////////
assertThat(DMLAuditAction.makeAuditDetailRecordForField("seqNo", table, UPDATE,
new QRecord().withValue("seqNo", ""),
new QRecord().withValue("seqNo", null)))
.isEmpty();
////////////////////////////////
// integer changed - do audit //
////////////////////////////////
assertThat(DMLAuditAction.makeAuditDetailRecordForField("seqNo", table, UPDATE,
new QRecord().withValue("seqNo", 2),
new QRecord().withValue("seqNo", 1)))
.isPresent()
.get().extracting(r -> r.getValueString("message"))
.matches(s -> s.matches("Changed Sequence No. from 1 to 2"));
}
/*******************************************************************************
**
*******************************************************************************/
@Test
void testGetContextSuffix()
{
assertEquals("", DMLAuditAction.getContentSuffix(new DMLAuditInput()));
assertEquals(" while shipping an order", DMLAuditAction.getContentSuffix(new DMLAuditInput().withAuditContext("while shipping an order")));
QContext.pushAction(new RunProcessInput().withValue(DMLAuditAction.AUDIT_CONTEXT_FIELD_NAME, "via Script \"My Script\""));
assertEquals(" via Script \"My Script\"", DMLAuditAction.getContentSuffix(new DMLAuditInput()));
QContext.popAction();
QContext.pushAction(new RunProcessInput().withProcessName(TestUtils.PROCESS_NAME_GREET_PEOPLE));
assertEquals(" during process: Greet", DMLAuditAction.getContentSuffix(new DMLAuditInput()));
QContext.popAction();
QContext.setQSession(new QSession().withValue("apiVersion", "1.0"));
assertEquals(" via API Version: 1.0", DMLAuditAction.getContentSuffix(new DMLAuditInput()));
QContext.setQSession(new QSession().withValue("apiVersion", "20230921").withValue("apiLabel", "Our Public API"));
assertEquals(" via Our Public API Version: 20230921", DMLAuditAction.getContentSuffix(new DMLAuditInput()));
QContext.pushAction(new RunProcessInput().withProcessName(TestUtils.PROCESS_NAME_GREET_PEOPLE).withValue(DMLAuditAction.AUDIT_CONTEXT_FIELD_NAME, "via Script \"My Script\""));
QContext.setQSession(new QSession().withValue("apiVersion", "20230921").withValue("apiLabel", "Our Public API"));
assertEquals(" while shipping an order via Script \"My Script\" during process: Greet via Our Public API Version: 20230921", DMLAuditAction.getContentSuffix(new DMLAuditInput().withAuditContext("while shipping an order")));
QContext.popAction();
}
}