Merge branch 'feature/script-audit-and-audit-change-cleanup' into integration/20230921

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/audits/AuditAction.java

@@ -29,6 +29,7 @@ import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
+import java.util.Optional;
 import com.kingsrook.qqq.backend.core.actions.AbstractQActionFunction;
 import com.kingsrook.qqq.backend.core.actions.tables.GetAction;
 import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
@@ -49,6 +50,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 import com.kingsrook.qqq.backend.core.model.session.QUser;
 import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
 import com.kingsrook.qqq.backend.core.utils.Pair;
+import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
 
 
 /*******************************************************************************
@@ -108,12 +110,26 @@ public class AuditAction extends AbstractQActionFunction<AuditInput, AuditOutput
 
 
 
+   /*******************************************************************************
+    ** Simple overload that internally figures out primary key and security key values
+    **
+    ** Be aware - if the record doesn't have its security key values set (say it's a
+    ** partial record as part of an update), then those values won't be in the
+    ** security key map...  This should probably be considered a bug.
+    *******************************************************************************/
+   public static void appendToInput(AuditInput auditInput, QTableMetaData table, QRecord record, String auditMessage)
+   {
+      appendToInput(auditInput, table.getName(), record.getValueInteger(table.getPrimaryKeyField()), getRecordSecurityKeyValues(table, record, Optional.empty()), auditMessage);
+   }
+
+
+
    /*******************************************************************************
     ** Add 1 auditSingleInput to an AuditInput object - with no details (child records).
     *******************************************************************************/
-   public static AuditInput appendToInput(AuditInput auditInput, String tableName, Integer recordId, Map<String, Serializable> securityKeyValues, String message)
+   public static void appendToInput(AuditInput auditInput, String tableName, Integer recordId, Map<String, Serializable> securityKeyValues, String message)
    {
-      return (appendToInput(auditInput, tableName, recordId, securityKeyValues, message, null));
+      appendToInput(auditInput, tableName, recordId, securityKeyValues, message, null);
    }
 
 
@@ -139,6 +155,44 @@ public class AuditAction extends AbstractQActionFunction<AuditInput, AuditOutput
 
 
 
+   /*******************************************************************************
+    ** For a given record, from a given table, build a map of the record's security
+    ** key values.
+    **
+    ** If, in case, the record has null value(s), and the oldRecord is given (e.g.,
+    ** for the case of an update, where the record may not have all fields set, and
+    ** oldRecord should be known for doing field-diffs), then try to get the value(s)
+    ** from oldRecord.
+    **
+    ** Currently, will leave values null if they aren't found after that.
+    **
+    ** An alternative could be to re-fetch the record from its source if needed...
+    *******************************************************************************/
+   public static Map<String, Serializable> getRecordSecurityKeyValues(QTableMetaData table, QRecord record, Optional<QRecord> oldRecord)
+   {
+      Map<String, Serializable> securityKeyValues = new HashMap<>();
+      for(RecordSecurityLock recordSecurityLock : RecordSecurityLockFilters.filterForReadLocks(CollectionUtils.nonNullList(table.getRecordSecurityLocks())))
+      {
+         Serializable keyValue = record == null ? null : record.getValue(recordSecurityLock.getFieldName());
+
+         if(keyValue == null && oldRecord.isPresent())
+         {
+            LOG.debug("Table with a securityLock, but value not found in field", logPair("table", table.getName()), logPair("field", recordSecurityLock.getFieldName()));
+            keyValue = oldRecord.get().getValue(recordSecurityLock.getFieldName());
+         }
+
+         if(keyValue == null)
+         {
+            LOG.debug("Table with a securityLock, but value not found in field", logPair("table", table.getName()), logPair("field", recordSecurityLock.getFieldName()), logPair("oldRecordIsPresent", oldRecord.isPresent()));
+         }
+
+         securityKeyValues.put(recordSecurityLock.getSecurityKeyType(), keyValue);
+      }
+      return securityKeyValues;
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/audits/DMLAuditAction.java

@@ -23,8 +23,10 @@ package com.kingsrook.qqq.backend.core.actions.audits;
 
 
 import java.io.Serializable;
+import java.math.BigDecimal;
 import java.time.Instant;
 import java.time.ZoneId;
+import java.time.temporal.ChronoUnit;
 import java.util.ArrayList;
 import java.util.Comparator;
 import java.util.HashMap;
@@ -52,13 +54,12 @@ import com.kingsrook.qqq.backend.core.model.metadata.audits.AuditLevel;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
 import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
-import com.kingsrook.qqq.backend.core.model.metadata.security.RecordSecurityLock;
-import com.kingsrook.qqq.backend.core.model.metadata.security.RecordSecurityLockFilters;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 import com.kingsrook.qqq.backend.core.model.session.QSession;
 import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
 import com.kingsrook.qqq.backend.core.utils.StringUtils;
 import com.kingsrook.qqq.backend.core.utils.ValueUtils;
+import static com.kingsrook.qqq.backend.core.actions.audits.AuditAction.getRecordSecurityKeyValues;
 import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
 
 
@@ -70,6 +71,8 @@ public class DMLAuditAction extends AbstractQActionFunction<DMLAuditInput, DMLAu
 {
    private static final QLogger LOG = QLogger.getLogger(DMLAuditAction.class);
 
+   public static final String AUDIT_CONTEXT_FIELD_NAME = "auditContext";
+
 
 
    /*******************************************************************************
@@ -99,34 +102,7 @@ public class DMLAuditAction extends AbstractQActionFunction<DMLAuditInput, DMLAu
             return (output);
          }
 
-         String contextSuffix = "";
+         String contextSuffix = getContentSuffix(input);
-         if(StringUtils.hasContent(input.getAuditContext()))
-         {
-            contextSuffix = " " + input.getAuditContext();
-         }
-
-         Optional<AbstractActionInput> actionInput = QContext.getFirstActionInStack();
-         if(actionInput.isPresent() && actionInput.get() instanceof RunProcessInput runProcessInput)
-         {
-            String           processName = runProcessInput.getProcessName();
-            QProcessMetaData process     = QContext.getQInstance().getProcess(processName);
-            if(process != null)
-            {
-               contextSuffix = " during process: " + process.getLabel();
-            }
-         }
-
-         QSession qSession   = QContext.getQSession();
-         String   apiVersion = qSession.getValue("apiVersion");
-         if(apiVersion != null)
-         {
-            String apiLabel = qSession.getValue("apiLabel");
-            if(!StringUtils.hasContent(apiLabel))
-            {
-               apiLabel = "API";
-            }
-            contextSuffix += (" via " + apiLabel + " Version: " + apiVersion);
-         }
 
          AuditInput auditInput = new AuditInput();
          if(auditLevel.equals(AuditLevel.RECORD) || (auditLevel.equals(AuditLevel.FIELD) && !dmlType.supportsFields))
@@ -137,7 +113,7 @@ public class DMLAuditAction extends AbstractQActionFunction<DMLAuditInput, DMLAu
             ////////////////////////////////////////////////////////////////////////////////////////////////////////////
             for(QRecord record : recordList)
             {
-               AuditAction.appendToInput(auditInput, table.getName(), record.getValueInteger(table.getPrimaryKeyField()), getRecordSecurityKeyValues(table, record), "Record was " + dmlType.pastTenseVerb + contextSuffix);
+               AuditAction.appendToInput(auditInput, table.getName(), record.getValueInteger(table.getPrimaryKeyField()), getRecordSecurityKeyValues(table, record, Optional.empty()), "Record was " + dmlType.pastTenseVerb + contextSuffix);
             }
          }
          else if(auditLevel.equals(AuditLevel.FIELD))
@@ -147,7 +123,7 @@ public class DMLAuditAction extends AbstractQActionFunction<DMLAuditInput, DMLAu
             ///////////////////////////////////////////////////////////////////
             // do many audits, all with field level details, for FIELD level //
             ///////////////////////////////////////////////////////////////////
-            QPossibleValueTranslator qPossibleValueTranslator = new QPossibleValueTranslator(QContext.getQInstance(), qSession);
+            QPossibleValueTranslator qPossibleValueTranslator = new QPossibleValueTranslator(QContext.getQInstance(), QContext.getQSession());
             qPossibleValueTranslator.translatePossibleValuesInRecords(table, CollectionUtils.mergeLists(recordList, oldRecordList));
 
             //////////////////////////////////////////
@@ -169,92 +145,8 @@ public class DMLAuditAction extends AbstractQActionFunction<DMLAuditInput, DMLAu
                List<QRecord> details = new ArrayList<>();
                for(String fieldName : sortedFieldNames)
                {
-                  if(!record.getValues().containsKey(fieldName))
+                  makeAuditDetailRecordForField(fieldName, table, dmlType, record, oldRecord)
-                  {
+                     .ifPresent(details::add);
-                     ////////////////////////////////////////////////////////////////////////////////////////////////
-                     // if the stored record doesn't have this field name, then don't audit anything about it      //
-                     // this is to deal with our Patch style updates not looking like every field was cleared out. //
-                     ////////////////////////////////////////////////////////////////////////////////////////////////
-                     continue;
-                  }
-
-                  if(fieldName.equals("modifyDate") || fieldName.equals("createDate") || fieldName.equals("automationStatus"))
-                  {
-                     continue;
-                  }
-
-                  QFieldMetaData field        = table.getField(fieldName);
-                  Serializable   value        = ValueUtils.getValueAsFieldType(field.getType(), record.getValue(fieldName));
-                  Serializable   oldValue     = oldRecord == null ? null : ValueUtils.getValueAsFieldType(field.getType(), oldRecord.getValue(fieldName));
-                  QRecord        detailRecord = null;
-
-                  if(oldRecord == null)
-                  {
-                     if(DMLType.INSERT.equals(dmlType) && value == null)
-                     {
-                        continue;
-                     }
-
-                     if(field.getType().equals(QFieldType.BLOB) || field.getType().needsMasked())
-                     {
-                        detailRecord = new QRecord().withValue("message", "Set " + field.getLabel());
-                     }
-                     else
-                     {
-                        String formattedValue = getFormattedValueForAuditDetail(record, fieldName, field, value);
-                        detailRecord = new QRecord().withValue("message", "Set " + field.getLabel() + " to " + formattedValue);
-                        detailRecord.withValue("newValue", formattedValue);
-                     }
-                  }
-                  else
-                  {
-                     if(!Objects.equals(oldValue, value))
-                     {
-                        if(field.getType().equals(QFieldType.BLOB) || field.getType().needsMasked())
-                        {
-                           if(oldValue == null)
-                           {
-                              detailRecord = new QRecord().withValue("message", "Set " + field.getLabel());
-                           }
-                           else if(value == null)
-                           {
-                              detailRecord = new QRecord().withValue("message", "Removed " + field.getLabel());
-                           }
-                           else
-                           {
-                              detailRecord = new QRecord().withValue("message", "Changed " + field.getLabel());
-                           }
-                        }
-                        else
-                        {
-                           String formattedValue    = getFormattedValueForAuditDetail(record, fieldName, field, value);
-                           String formattedOldValue = getFormattedValueForAuditDetail(oldRecord, fieldName, field, oldValue);
-
-                           if(oldValue == null)
-                           {
-                              detailRecord = new QRecord().withValue("message", "Set " + field.getLabel() + " to " + formatFormattedValueForDetailMessage(field, formattedValue));
-                              detailRecord.withValue("newValue", formattedValue);
-                           }
-                           else if(value == null)
-                           {
-                              detailRecord = new QRecord().withValue("message", "Removed " + formatFormattedValueForDetailMessage(field, formattedOldValue) + " from " + field.getLabel());
-                              detailRecord.withValue("oldValue", formattedOldValue);
-                           }
-                           else
-                           {
-                              detailRecord = new QRecord().withValue("message", "Changed " + field.getLabel() + " from " + formatFormattedValueForDetailMessage(field, formattedOldValue) + " to " + formatFormattedValueForDetailMessage(field, formattedValue));
-                              detailRecord.withValue("oldValue", formattedOldValue);
-                              detailRecord.withValue("newValue", formattedValue);
-                           }
-                        }
-                     }
-                  }
-
-                  if(detailRecord != null)
-                  {
-                     detailRecord.withValue("fieldName", fieldName);
-                     details.add(detailRecord);
-                  }
                }
 
                if(details.isEmpty() && DMLType.UPDATE.equals(dmlType))
@@ -264,7 +156,7 @@ public class DMLAuditAction extends AbstractQActionFunction<DMLAuditInput, DMLAu
                }
                else
                {
-                  AuditAction.appendToInput(auditInput, table.getName(), record.getValueInteger(table.getPrimaryKeyField()), getRecordSecurityKeyValues(table, record), "Record was " + dmlType.pastTenseVerb + contextSuffix, details);
+                  AuditAction.appendToInput(auditInput, table.getName(), record.getValueInteger(table.getPrimaryKeyField()), getRecordSecurityKeyValues(table, record, Optional.ofNullable(oldRecord)), "Record was " + dmlType.pastTenseVerb + contextSuffix, details);
                }
             }
          }
@@ -284,6 +176,254 @@ public class DMLAuditAction extends AbstractQActionFunction<DMLAuditInput, DMLAu
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   static String getContentSuffix(DMLAuditInput input)
+   {
+      StringBuilder contextSuffix = new StringBuilder();
+
+      /////////////////////////////////////////////////////////////////////////////
+      // start with context from the input wrapper                               //
+      // note, these contexts get propagated down from Input/Update/Delete Input //
+      /////////////////////////////////////////////////////////////////////////////
+      if(StringUtils.hasContent(input.getAuditContext()))
+      {
+         contextSuffix.append(" ").append(input.getAuditContext());
+      }
+
+      /////////////////////////////////////////////////////////////////////////////////////
+      // note process label (and a possible context from the process's state) if present //
+      /////////////////////////////////////////////////////////////////////////////////////
+      Optional<AbstractActionInput> actionInput = QContext.getFirstActionInStack();
+      if(actionInput.isPresent() && actionInput.get() instanceof RunProcessInput runProcessInput)
+      {
+         String processAuditContext = ValueUtils.getValueAsString(runProcessInput.getValue(AUDIT_CONTEXT_FIELD_NAME));
+         if(StringUtils.hasContent(processAuditContext))
+         {
+            contextSuffix.append(" ").append(processAuditContext);
+         }
+
+         String           processName = runProcessInput.getProcessName();
+         QProcessMetaData process     = QContext.getQInstance().getProcess(processName);
+         if(process != null)
+         {
+            contextSuffix.append(" during process: ").append(process.getLabel());
+         }
+      }
+
+      ///////////////////////////////////////////////////
+      // use api label & version if present in session //
+      ///////////////////////////////////////////////////
+      QSession qSession   = QContext.getQSession();
+      String   apiVersion = qSession.getValue("apiVersion");
+      if(apiVersion != null)
+      {
+         String apiLabel = qSession.getValue("apiLabel");
+         if(!StringUtils.hasContent(apiLabel))
+         {
+            apiLabel = "API";
+         }
+         contextSuffix.append(" via ").append(apiLabel).append(" Version: ").append(apiVersion);
+      }
+      return (contextSuffix.toString());
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   static Optional<QRecord> makeAuditDetailRecordForField(String fieldName, QTableMetaData table, DMLType dmlType, QRecord record, QRecord oldRecord)
+   {
+      if(!record.getValues().containsKey(fieldName))
+      {
+         ////////////////////////////////////////////////////////////////////////////////////////////////
+         // if the stored record doesn't have this field name, then don't audit anything about it      //
+         // this is to deal with our Patch style updates not looking like every field was cleared out. //
+         ////////////////////////////////////////////////////////////////////////////////////////////////
+         return (Optional.empty());
+      }
+
+      if(fieldName.equals("modifyDate") || fieldName.equals("createDate") || fieldName.equals("automationStatus"))
+      {
+         return (Optional.empty());
+      }
+
+      QFieldMetaData field        = table.getField(fieldName);
+      Serializable   value        = ValueUtils.getValueAsFieldType(field.getType(), record.getValue(fieldName));
+      Serializable   oldValue     = oldRecord == null ? null : ValueUtils.getValueAsFieldType(field.getType(), oldRecord.getValue(fieldName));
+      QRecord        detailRecord = null;
+
+      if(oldRecord == null)
+      {
+         if(DMLType.INSERT.equals(dmlType) && value == null)
+         {
+            return (Optional.empty());
+         }
+
+         if(field.getType().equals(QFieldType.BLOB) || field.getType().needsMasked())
+         {
+            detailRecord = new QRecord().withValue("message", "Set " + field.getLabel());
+         }
+         else
+         {
+            String formattedValue = getFormattedValueForAuditDetail(record, fieldName, field, value);
+            detailRecord = new QRecord().withValue("message", "Set " + field.getLabel() + " to " + formattedValue);
+            detailRecord.withValue("newValue", formattedValue);
+         }
+      }
+      else
+      {
+         if(areValuesDifferentForAudit(field, value, oldValue))
+         {
+            if(field.getType().equals(QFieldType.BLOB) || field.getType().needsMasked())
+            {
+               if(oldValue == null)
+               {
+                  detailRecord = new QRecord().withValue("message", "Set " + field.getLabel());
+               }
+               else if(value == null)
+               {
+                  detailRecord = new QRecord().withValue("message", "Removed " + field.getLabel());
+               }
+               else
+               {
+                  detailRecord = new QRecord().withValue("message", "Changed " + field.getLabel());
+               }
+            }
+            else
+            {
+               String formattedValue    = getFormattedValueForAuditDetail(record, fieldName, field, value);
+               String formattedOldValue = getFormattedValueForAuditDetail(oldRecord, fieldName, field, oldValue);
+
+               if(oldValue == null)
+               {
+                  detailRecord = new QRecord().withValue("message", "Set " + field.getLabel() + " to " + formatFormattedValueForDetailMessage(field, formattedValue));
+                  detailRecord.withValue("newValue", formattedValue);
+               }
+               else if(value == null)
+               {
+                  detailRecord = new QRecord().withValue("message", "Removed " + formatFormattedValueForDetailMessage(field, formattedOldValue) + " from " + field.getLabel());
+                  detailRecord.withValue("oldValue", formattedOldValue);
+               }
+               else
+               {
+                  detailRecord = new QRecord().withValue("message", "Changed " + field.getLabel() + " from " + formatFormattedValueForDetailMessage(field, formattedOldValue) + " to " + formatFormattedValueForDetailMessage(field, formattedValue));
+                  detailRecord.withValue("oldValue", formattedOldValue);
+                  detailRecord.withValue("newValue", formattedValue);
+               }
+            }
+         }
+      }
+
+      if(detailRecord != null)
+      {
+         LOG.debug("Returning with message: " + detailRecord.getValueString("message"));
+
+         detailRecord.withValue("fieldName", fieldName);
+         return (Optional.of(detailRecord));
+      }
+
+      return (Optional.empty());
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   static boolean areValuesDifferentForAudit(QFieldMetaData field, Serializable value, Serializable oldValue)
+   {
+      try
+      {
+         ///////////////////
+         // decimal rules //
+         ///////////////////
+         if(field.getType().equals(QFieldType.DECIMAL))
+         {
+            BigDecimal newBD = ValueUtils.getValueAsBigDecimal(value);
+            BigDecimal oldBD = ValueUtils.getValueAsBigDecimal(oldValue);
+
+            if(newBD == null && oldBD == null)
+            {
+               return (false);
+            }
+
+            if(newBD == null || oldBD == null)
+            {
+               return (true);
+            }
+
+            return (newBD.compareTo(oldBD) != 0);
+         }
+
+         ////////////////////
+         // dateTime rules //
+         ////////////////////
+         if(field.getType().equals(QFieldType.DATE_TIME))
+         {
+            Instant newI = ValueUtils.getValueAsInstant(value);
+            Instant oldI = ValueUtils.getValueAsInstant(oldValue);
+
+            if(newI == null && oldI == null)
+            {
+               return (false);
+            }
+
+            if(newI == null || oldI == null)
+            {
+               return (true);
+            }
+
+            ////////////////////////////////
+            // just compare to the second //
+            ////////////////////////////////
+            return (newI.truncatedTo(ChronoUnit.SECONDS).compareTo(oldI.truncatedTo(ChronoUnit.SECONDS)) != 0);
+         }
+
+         //////////////////
+         // string rules //
+         //////////////////
+         if(field.getType().isStringLike())
+         {
+            String newString = ValueUtils.getValueAsString(value);
+            String oldString = ValueUtils.getValueAsString(oldValue);
+
+            boolean newIsNullOrEmpty = !StringUtils.hasContent(newString);
+            boolean oldIsNullOrEmpty = !StringUtils.hasContent(oldString);
+
+            if(newIsNullOrEmpty && oldIsNullOrEmpty)
+            {
+               return (false);
+            }
+
+            if(newIsNullOrEmpty || oldIsNullOrEmpty)
+            {
+               return (true);
+            }
+
+            return (newString.compareTo(oldString) != 0);
+         }
+
+         /////////////////////////////////////
+         // default just use Objects.equals //
+         /////////////////////////////////////
+         return !Objects.equals(oldValue, value);
+      }
+      catch(Exception e)
+      {
+         LOG.debug("Error checking areValuesDifferentForAudit", e, logPair("fieldName", field.getName()), logPair("value", value), logPair("oldValue", oldValue));
+      }
+
+      ////////////////////////////////////
+      // default to something simple... //
+      ////////////////////////////////////
+      return !Objects.equals(oldValue, value);
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/
@@ -373,21 +513,6 @@ public class DMLAuditAction extends AbstractQActionFunction<DMLAuditInput, DMLAu
 
 
 
-   /*******************************************************************************
-    **
-    *******************************************************************************/
-   private static Map<String, Serializable> getRecordSecurityKeyValues(QTableMetaData table, QRecord record)
-   {
-      Map<String, Serializable> securityKeyValues = new HashMap<>();
-      for(RecordSecurityLock recordSecurityLock : RecordSecurityLockFilters.filterForReadLocks(CollectionUtils.nonNullList(table.getRecordSecurityLocks())))
-      {
-         securityKeyValues.put(recordSecurityLock.getSecurityKeyType(), record == null ? null : record.getValue(recordSecurityLock.getFieldName()));
-      }
-      return securityKeyValues;
-   }
-
-
-
    /*******************************************************************************
     **
     *******************************************************************************/
@@ -407,7 +532,7 @@ public class DMLAuditAction extends AbstractQActionFunction<DMLAuditInput, DMLAu
    /*******************************************************************************
     **
     *******************************************************************************/
-   private enum DMLType
+   enum DMLType
    {
       INSERT("Inserted", true),
       UPDATE("Edited", true),

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/scripts/RunRecordScriptLoadStep.java

@@ -25,11 +25,15 @@ package com.kingsrook.qqq.backend.core.processes.implementations.scripts;
 import java.io.Serializable;
 import java.util.ArrayList;
 import java.util.List;
+import com.kingsrook.qqq.backend.core.actions.audits.AuditAction;
+import com.kingsrook.qqq.backend.core.actions.audits.DMLAuditAction;
 import com.kingsrook.qqq.backend.core.actions.scripts.RunAdHocRecordScriptAction;
 import com.kingsrook.qqq.backend.core.actions.scripts.logging.StoreScriptLogAndScriptLogLineExecutionLogger;
 import com.kingsrook.qqq.backend.core.actions.tables.GetAction;
+import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.logging.QLogger;
+import com.kingsrook.qqq.backend.core.model.actions.audits.AuditInput;
 import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryFilterLink;
 import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLine;
 import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLineInterface;
@@ -42,7 +46,9 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetOutput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
 import com.kingsrook.qqq.backend.core.model.metadata.code.AdHocScriptCodeReference;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 import com.kingsrook.qqq.backend.core.model.scripts.Script;
 import com.kingsrook.qqq.backend.core.model.scripts.ScriptLog;
 import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.AbstractLoadStep;
@@ -121,16 +127,25 @@ public class RunRecordScriptLoadStep extends AbstractLoadStep implements Process
       getInput.setTableName(Script.TABLE_NAME);
       getInput.setPrimaryKey(scriptId);
       GetOutput getOutput = new GetAction().execute(getInput);
-      if(getOutput.getRecord() == null)
+      QRecord   script    = getOutput.getRecord();
+      if(script == null)
       {
          throw (new QException("Could not find script by id: " + scriptId));
       }
 
+      //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      // set an "audit context" - so any DML executed during the script will include the note of what script was running. //
+      //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      runBackendStepInput.addValue(DMLAuditAction.AUDIT_CONTEXT_FIELD_NAME, "via Script \"" + script.getValue("name") + "\"");
+
+      String tableName = script.getValueString("tableName");
+
       RunAdHocRecordScriptInput input = new RunAdHocRecordScriptInput();
       input.setRecordList(runBackendStepInput.getRecords());
       input.setCodeReference(new AdHocScriptCodeReference().withScriptId(scriptId));
-      input.setTableName(getOutput.getRecord().getValueString("tableName"));
+      input.setTableName(tableName);
       input.setLogger(scriptLogger);
+
       RunAdHocRecordScriptOutput output          = new RunAdHocRecordScriptOutput();
       Exception                  caughtException = null;
       try
@@ -147,11 +162,17 @@ public class RunRecordScriptLoadStep extends AbstractLoadStep implements Process
          caughtException = e;
       }
 
+      String auditMessage = "Script \"" + script.getValueString("name") + "\" (id: " + scriptId + ") was executed against this record";
+
+      //////////////////////////////////////////////////////////
+      // add the record to the appropriate processSummaryLine //
+      //////////////////////////////////////////////////////////
       if(scriptLogger.getScriptLog() != null)
       {
          Integer id = scriptLogger.getScriptLog().getValueInteger("id");
          if(id != null)
          {
+            auditMessage += ", creating script log: " + id;
             boolean hadError = BooleanUtils.isTrue(scriptLogger.getScriptLog().getValueBoolean("hadError"));
             (hadError ? errorScriptLogIds : okScriptLogIds).add(id);
          }
@@ -160,6 +181,34 @@ public class RunRecordScriptLoadStep extends AbstractLoadStep implements Process
       {
          unloggedExceptionLine.incrementCount(runBackendStepInput.getRecords().size());
       }
+
+      ////////////////////////////////////////////////////////////
+      // audit that the script was executed against the records //
+      ////////////////////////////////////////////////////////////
+      audit(runBackendStepInput, tableName, auditMessage);
+   }
+
+
+
+   /*******************************************************************************
+    ** for each input record, add an audit stating that the script was executed.
+    *******************************************************************************/
+   private static void audit(RunBackendStepInput runBackendStepInput, String tableName, String auditMessage)
+   {
+      try
+      {
+         QTableMetaData table      = QContext.getQInstance().getTable(tableName);
+         AuditInput     auditInput = new AuditInput();
+         for(QRecord record : runBackendStepInput.getRecords())
+         {
+            AuditAction.appendToInput(auditInput, table, record, auditMessage);
+         }
+         new AuditAction().execute(auditInput);
+      }
+      catch(Exception e)
+      {
+         LOG.warn("Error recording audits after running record script", e, logPair("tableName", tableName), logPair("auditMessage", auditMessage));
+      }
    }
 
 

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/BaseTest.java

@@ -48,6 +48,8 @@ public class BaseTest
    @BeforeEach
    void baseBeforeEach()
    {
+      System.setProperty("qqq.logger.logSessionId.disabled", "true");
+
       QContext.init(TestUtils.defineInstance(), new QSession()
          .withUser(new QUser()
             .withIdReference("001")

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/audits/AuditActionTest.java

@@ -31,6 +31,7 @@ import com.kingsrook.qqq.backend.core.BaseTest;
 import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.model.actions.audits.AuditInput;
+import com.kingsrook.qqq.backend.core.model.actions.audits.AuditSingleInput;
 import com.kingsrook.qqq.backend.core.model.audits.AuditsMetaDataProvider;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
 import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
@@ -38,6 +39,7 @@ import com.kingsrook.qqq.backend.core.model.session.QSession;
 import com.kingsrook.qqq.backend.core.model.session.QUser;
 import com.kingsrook.qqq.backend.core.processes.utils.GeneralProcessUtils;
 import com.kingsrook.qqq.backend.core.utils.TestUtils;
+import com.kingsrook.qqq.backend.core.utils.collections.MapBuilder;
 import org.junit.jupiter.api.Test;
 import static org.assertj.core.api.Assertions.assertThat;
 import static org.junit.jupiter.api.Assertions.assertEquals;
@@ -197,4 +199,31 @@ class AuditActionTest extends BaseTest
       assertThat(auditDetails).anyMatch(r -> r.getValueString("message").equals("Detail3"));
    }
 
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testAppendToInputThatTakesRecordNotIdAndSecurityKeyValues()
+   {
+      AuditInput auditInput = new AuditInput();
+
+      //////////////////////////////////////////////////////////////
+      // make sure the recordId & securityKey got build correctly //
+      //////////////////////////////////////////////////////////////
+      AuditAction.appendToInput(auditInput, QContext.getQInstance().getTable(TestUtils.TABLE_NAME_ORDER), new QRecord().withValue("id", 47).withValue("storeId", 42), "Test");
+      AuditSingleInput auditSingleInput = auditInput.getAuditSingleInputList().get(0);
+      assertEquals(47, auditSingleInput.getRecordId());
+      assertEquals(MapBuilder.of(TestUtils.SECURITY_KEY_TYPE_STORE, 42), auditSingleInput.getSecurityKeyValues());
+
+      ///////////////////////////////////////////////////////////////////////////////////////////
+      // acknowledge that we might get back a null key value if the record doesn't have it set //
+      ///////////////////////////////////////////////////////////////////////////////////////////
+      AuditAction.appendToInput(auditInput, QContext.getQInstance().getTable(TestUtils.TABLE_NAME_ORDER), new QRecord().withValue("id", 47), "Test");
+      auditSingleInput = auditInput.getAuditSingleInputList().get(1);
+      assertEquals(47, auditSingleInput.getRecordId());
+      assertEquals(MapBuilder.of(TestUtils.SECURITY_KEY_TYPE_STORE, null), auditSingleInput.getSecurityKeyValues());
+   }
+
 }

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/audits/DMLAuditActionTest.java

@@ -22,12 +22,14 @@
 package com.kingsrook.qqq.backend.core.actions.audits;
 
 
+import java.math.BigDecimal;
 import java.time.Instant;
 import java.util.List;
 import com.kingsrook.qqq.backend.core.BaseTest;
 import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.model.actions.audits.DMLAuditInput;
+import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
@@ -36,10 +38,18 @@ import com.kingsrook.qqq.backend.core.model.data.QRecord;
 import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
 import com.kingsrook.qqq.backend.core.model.metadata.audits.AuditLevel;
 import com.kingsrook.qqq.backend.core.model.metadata.audits.QAuditRules;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+import com.kingsrook.qqq.backend.core.model.session.QSession;
 import com.kingsrook.qqq.backend.core.model.statusmessages.BadInputStatusMessage;
 import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryRecordStore;
 import com.kingsrook.qqq.backend.core.utils.TestUtils;
+import com.kingsrook.qqq.backend.core.utils.ValueUtils;
 import org.junit.jupiter.api.Test;
+import static com.kingsrook.qqq.backend.core.actions.audits.DMLAuditAction.DMLType.INSERT;
+import static com.kingsrook.qqq.backend.core.actions.audits.DMLAuditAction.DMLType.UPDATE;
+import static org.assertj.core.api.Assertions.assertThat;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertFalse;
 import static org.junit.jupiter.api.Assertions.assertNull;
@@ -200,4 +210,194 @@ class DMLAuditActionTest extends BaseTest
       }
    }
 
-}
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testMakeAuditDetailRecordForField()
+   {
+      QTableMetaData table = new QTableMetaData()
+         .withField(new QFieldMetaData("createDate", QFieldType.DATE_TIME).withLabel("Create Date"))
+         .withField(new QFieldMetaData("modifyDate", QFieldType.DATE_TIME).withLabel("Modify Date"))
+         .withField(new QFieldMetaData("someTimestamp", QFieldType.DATE_TIME).withLabel("Some Timestamp"))
+         .withField(new QFieldMetaData("name", QFieldType.STRING).withLabel("Name"))
+         .withField(new QFieldMetaData("seqNo", QFieldType.INTEGER).withLabel("Sequence No."))
+         .withField(new QFieldMetaData("price", QFieldType.DECIMAL).withLabel("Price"));
+
+      ///////////////////////////////
+      // create date - never audit //
+      ///////////////////////////////
+      assertThat(DMLAuditAction.makeAuditDetailRecordForField("createDate", table, INSERT,
+         new QRecord().withValue("createDate", Instant.now()),
+         new QRecord().withValue("createDate", Instant.now().minusSeconds(100))))
+         .isEmpty();
+
+      ///////////////////////////////
+      // modify date - never audit //
+      ///////////////////////////////
+      assertThat(DMLAuditAction.makeAuditDetailRecordForField("modifyDate", table, UPDATE,
+         new QRecord().withValue("modifyDate", Instant.now()),
+         new QRecord().withValue("modifyDate", Instant.now().minusSeconds(100))))
+         .isEmpty();
+
+      ////////////////////////////////////////////////////////
+      // datetime different only in precision - don't audit //
+      ////////////////////////////////////////////////////////
+      assertThat(DMLAuditAction.makeAuditDetailRecordForField("someTimestamp", table, UPDATE,
+         new QRecord().withValue("someTimestamp", ValueUtils.getValueAsInstant("2023-04-17T14:33:08.777")),
+         new QRecord().withValue("someTimestamp", Instant.parse("2023-04-17T14:33:08Z"))))
+         .isEmpty();
+
+      /////////////////////////////////////////////
+      // datetime actually different - audit it. //
+      /////////////////////////////////////////////
+      assertThat(DMLAuditAction.makeAuditDetailRecordForField("someTimestamp", table, UPDATE,
+         new QRecord().withValue("someTimestamp", Instant.parse("2023-04-17T14:33:09Z")),
+         new QRecord().withValue("someTimestamp", Instant.parse("2023-04-17T14:33:08Z"))))
+         .isPresent()
+         .get().extracting(r -> r.getValueString("message"))
+         .matches(s -> s.matches("Changed Some Timestamp from 2023.* to 2023.*"));
+
+      ////////////////////////////////////////////////
+      // datetime changing null to not null - audit //
+      ////////////////////////////////////////////////
+      assertThat(DMLAuditAction.makeAuditDetailRecordForField("someTimestamp", table, UPDATE,
+         new QRecord().withValue("someTimestamp", ValueUtils.getValueAsInstant("2023-04-17T14:33:08.777")),
+         new QRecord().withValue("someTimestamp", null)))
+         .isPresent()
+         .get().extracting(r -> r.getValueString("message"))
+         .matches(s -> s.matches("Set Some Timestamp to 2023.*"));
+
+      ////////////////////////////////////////////////
+      // datetime changing not null to null - audit //
+      ////////////////////////////////////////////////
+      assertThat(DMLAuditAction.makeAuditDetailRecordForField("someTimestamp", table, UPDATE,
+         new QRecord().withValue("someTimestamp", null),
+         new QRecord().withValue("someTimestamp", Instant.parse("2023-04-17T14:33:08Z"))))
+         .isPresent()
+         .get().extracting(r -> r.getValueString("message"))
+         .matches(s -> s.matches("Removed 2023.*from Some Timestamp"));
+
+      ////////////////////////////////////////
+      // string that is the same - no audit //
+      ////////////////////////////////////////
+      assertThat(DMLAuditAction.makeAuditDetailRecordForField("name", table, UPDATE,
+         new QRecord().withValue("name", "Homer"),
+         new QRecord().withValue("name", "Homer")))
+         .isEmpty();
+
+      //////////////////////////////////////////
+      // string from null to empty - no audit //
+      //////////////////////////////////////////
+      assertThat(DMLAuditAction.makeAuditDetailRecordForField("name", table, UPDATE,
+         new QRecord().withValue("name", null),
+         new QRecord().withValue("name", "")))
+         .isEmpty();
+
+      //////////////////////////////////////////
+      // string from empty to null - no audit //
+      //////////////////////////////////////////
+      assertThat(DMLAuditAction.makeAuditDetailRecordForField("name", table, UPDATE,
+         new QRecord().withValue("name", ""),
+         new QRecord().withValue("name", null)))
+         .isEmpty();
+
+      //////////////////////////////////////////////////////////
+      // decimal that only changes in precision - don't audit //
+      //////////////////////////////////////////////////////////
+      assertThat(DMLAuditAction.makeAuditDetailRecordForField("price", table, UPDATE,
+         new QRecord().withValue("price", "10"),
+         new QRecord().withValue("price", new BigDecimal("10.00"))))
+         .isEmpty();
+
+      //////////////////////////////////////////////////
+      // decimal that's actually different - do audit //
+      //////////////////////////////////////////////////
+      assertThat(DMLAuditAction.makeAuditDetailRecordForField("price", table, UPDATE,
+         new QRecord().withValue("price", "10.01"),
+         new QRecord().withValue("price", new BigDecimal("10.00"))))
+         .isPresent()
+         .get().extracting(r -> r.getValueString("message"))
+         .matches(s -> s.matches("Changed Price from 10.00 to 10.01"));
+
+      ///////////////////////////////////////
+      // decimal null, input "" - no audit //
+      ///////////////////////////////////////
+      assertThat(DMLAuditAction.makeAuditDetailRecordForField("price", table, UPDATE,
+         new QRecord().withValue("price", ""),
+         new QRecord().withValue("price", null)))
+         .isEmpty();
+
+      /////////////////////////////////////////
+      // decimal not-null to null - do audit //
+      /////////////////////////////////////////
+      assertThat(DMLAuditAction.makeAuditDetailRecordForField("price", table, UPDATE,
+         new QRecord().withValue("price", BigDecimal.ONE),
+         new QRecord().withValue("price", null)))
+         .isPresent()
+         .get().extracting(r -> r.getValueString("message"))
+         .matches(s -> s.matches("Set Price to 1"));
+
+      /////////////////////////////////////////
+      // decimal null to not-null - do audit //
+      /////////////////////////////////////////
+      assertThat(DMLAuditAction.makeAuditDetailRecordForField("price", table, UPDATE,
+         new QRecord().withValue("price", null),
+         new QRecord().withValue("price", BigDecimal.ONE)))
+         .isPresent()
+         .get().extracting(r -> r.getValueString("message"))
+         .matches(s -> s.matches("Removed 1 from Price"));
+
+      ///////////////////////////////////////
+      // integer null, input "" - no audit //
+      ///////////////////////////////////////
+      assertThat(DMLAuditAction.makeAuditDetailRecordForField("seqNo", table, UPDATE,
+         new QRecord().withValue("seqNo", ""),
+         new QRecord().withValue("seqNo", null)))
+         .isEmpty();
+
+      ////////////////////////////////
+      // integer changed - do audit //
+      ////////////////////////////////
+      assertThat(DMLAuditAction.makeAuditDetailRecordForField("seqNo", table, UPDATE,
+         new QRecord().withValue("seqNo", 2),
+         new QRecord().withValue("seqNo", 1)))
+         .isPresent()
+         .get().extracting(r -> r.getValueString("message"))
+         .matches(s -> s.matches("Changed Sequence No. from 1 to 2"));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testGetContextSuffix()
+   {
+      assertEquals("", DMLAuditAction.getContentSuffix(new DMLAuditInput()));
+      assertEquals(" while shipping an order", DMLAuditAction.getContentSuffix(new DMLAuditInput().withAuditContext("while shipping an order")));
+
+      QContext.pushAction(new RunProcessInput().withValue(DMLAuditAction.AUDIT_CONTEXT_FIELD_NAME, "via Script \"My Script\""));
+      assertEquals(" via Script \"My Script\"", DMLAuditAction.getContentSuffix(new DMLAuditInput()));
+      QContext.popAction();
+
+      QContext.pushAction(new RunProcessInput().withProcessName(TestUtils.PROCESS_NAME_GREET_PEOPLE));
+      assertEquals(" during process: Greet", DMLAuditAction.getContentSuffix(new DMLAuditInput()));
+      QContext.popAction();
+
+      QContext.setQSession(new QSession().withValue("apiVersion", "1.0"));
+      assertEquals(" via API Version: 1.0", DMLAuditAction.getContentSuffix(new DMLAuditInput()));
+
+      QContext.setQSession(new QSession().withValue("apiVersion", "20230921").withValue("apiLabel", "Our Public API"));
+      assertEquals(" via Our Public API Version: 20230921", DMLAuditAction.getContentSuffix(new DMLAuditInput()));
+
+      QContext.pushAction(new RunProcessInput().withProcessName(TestUtils.PROCESS_NAME_GREET_PEOPLE).withValue(DMLAuditAction.AUDIT_CONTEXT_FIELD_NAME, "via Script \"My Script\""));
+      QContext.setQSession(new QSession().withValue("apiVersion", "20230921").withValue("apiLabel", "Our Public API"));
+      assertEquals(" while shipping an order via Script \"My Script\" during process: Greet via Our Public API Version: 20230921", DMLAuditAction.getContentSuffix(new DMLAuditInput().withAuditContext("while shipping an order")));
+      QContext.popAction();
+   }
+
+}