CE-970: fixed bug around getting new ApiTableMetaData

Add new "Collecting Logger", that can be activated to "collect" logs …

docs/.gitignore

@@ -0,0 +1 @@
+*.html

docs/actions/GetAction.adoc

@@ -0,0 +1,64 @@
+== GetAction
+include::../variables.adoc[]
+
+The `*GetAction*` is essentially a subset of the <<QueryAction>>, only specifically meant to get just a single record from a {link-table}.
+In SQL/RDBMS terms, it is analogous to a `SELECT` statement, where a single record may be found - or - it may not be found.
+
+For all tables, `GetAction` can do a lookup by primary key.
+In addition, for tables that define a `UniqueKey`, name/value pairs (in the form of a `Map`) can be used as input for `GetAction`.
+
+=== Examples
+[source,java]
+.Basic form - by primary key
+----
+GetInput input = new GetInput();
+input.setTableName("orders");
+input.setPrimaryKey(1);
+GetOutput output = new GetAction.execute(input);
+QRecord record = output.getRecord();
+----
+
+[source,java]
+.Secondary form - by Unique Key
+----
+GetInput input = new GetInput();
+input.setTableName("products");
+input.setUniqueKey(Map.of("storeId", 1701, "sku", "ABCD"));
+GetOutput output = new GetAction.execute(input);
+QRecord record = output.getRecord();
+----
+
+=== GetInput
+* `table` - *String, Required* - Name of the table being queried against.
+* `primaryKey` - *Serializable, Conditional* - Value for the primary key field of the table being queried.
+Type should match the table's primary key field's type.
+If a `primaryKey` is not given, then a `uniqueKey` must be given.
+* `uniqueKey` - *Map of String -> Serializable, Conditional* - Map of name-value pairs that define the record to be fetcheed.
+Keys in the map must be field names from the table being queried.
+Values in the map should should be of types that correspond to the fields.
+If a `primaryKey` is not given, then a `uniqueKey` must be given.
+If both `primaryKey` and `uniqueKey` are given, then `uniqueKey` is ignored.
+* `transaction` - *QBackendTransaction object* - Optional transaction object.
+** Behavior for this object is backend-dependant.
+In an RDBMS backend, this object is generally needed if you want your query to see data that may have been modified within the same transaction.
+
+* `shouldTranslatePossibleValues` - *boolean, default: false* - Controls whether any fields in the table with a *possibleValueSource* assigned to them should have those possible values looked up
+(e.g., to provide text translations in the generated records' `displayValues` map).
+** For example, if getting a record to present to a user, this would generally need to be *true*.
+But if getting a record as part of a process, then this can generally be left as *false*.
+* `shouldGenerateDisplayValues` - *boolean, default: false* - Controls whether field level *displayFormats* should be used to populate the generated records' `displayValues` map.
+** For example, if getting a record to present to a user, this would generally need to be *true*.
+But if getting a record as part of a process, then this can generally be left as *false*.
+* `shouldFetchHeavyFields` - *boolean, default: true* - Controls whether or not fields marked as `isHeavy` should be fetched & returned or not.
+* `shouldOmitHiddenFields` - *boolean, default: true* - Controls whether or not fields marked as `isHidden` should be included in the result or not.
+* `shouldMaskPassword` - *boolean, default: true* - Controls whether or not fields with `type` = `PASSWORD` should be masked, or if their actual values should be returned.
+* `queryJoins` - *List of <<QueryJoin>> objects* - Optional list of tables to be joined with the main table being queried.
+See QueryJoin under <<QueryAction>> for further details.
+* `includeAssociations` - *boolean, default: false* - Control whether or not records from tables defined as `associations` under the table being queried should be included in the result or not.
+* `associationNamesToInclude` - *Collection of String* - If `includeAssociations` is true, then this field can be used to limit which associated tables are included.
+If this field is null, then all associated tables are included.
+Otherwise, a table is only included if its name is in this collection.
+
+=== GetOutput
+* `record` - *QRecord* - The record that was specified by the input `primaryKey` or `uniqueKey`.
+Will be null if the record was not found.

docs/actions/InsertAction.adoc

@@ -0,0 +1,86 @@
+== InsertAction
+include::../variables.adoc[]
+
+To insert (add, create) new records into any {link-table}, the `*InsertAction*` is used.
+In SQL/RDBMS terms, it is analogous to a `INSERT` statement, where one or more records can be provided as input.
+
+=== Examples
+[source,java]
+.Canonical InsertAction invocation
+----
+InsertInput insertInput = new InsertInput();
+insertInput.setTableName("person");
+insertInput.setRecords(personRecordList);
+InsertOutput insertOutput = new InsertAction().execute(insertInput);
+List<QRecord> insertedPersonRecords = insertOutput.getRecords();
+----
+
+=== Details
+`InsertAction` does several things beyond just inserting records into the specified table.
+A high-level flow of its internal logic is:
+
+. For tables using an automation status field, set its value to `PENDING_INSERT_AUTOMATIONS` for all `records` that are going to be inserted.
+. Perform the following validations, which include running the table's `PRE_INSERT_CUSTOMIZER`, if one is defined, at the time that is specified by the customizer's `WhenToRun` property (default is `AFTER_ALL_VALIDATIONS`):
+.. Ensure that default values specified in the table's fields are present if needed.
+.. Apply per-field behaviors, as defined in {link-field} meta-data, such as truncating strings that are longer than their specified max.
+.. Check for unique key violations (done here instead of in the backend, to provide better error messaging, and to allow a subset of records to be stored while some fail).
+_We might want to make an input control in the future to specify that either the full input set should succeed or fail..._
+.. Validate that required fields (again, per {link-field} meta-data) are set, generating per-record errors if they are not.
+.. Validate any security fields in the records - e.g., ensure that the user has permission to insert records with the values they are attempting to insert.
+. Send the records to the table's backend module to actually insert them into the backend storage.
+. If the table has any associations defined, and if associated records are present, then recursively run `InsertAction` on the associated records.
+.. In particular, before these recursive `InsertAction` calls are made, values that were generated by the original insert may need to be propagated down into the associated records.
+*** For example, if inserting `order` and `lineItem` records, where a {link-join} exists between the two tables on `order.id` and `lineItem.orderId`, and `order.id` values were generated in the first `InsertAction`, then those values are propagated down into the associated `lineItem.orderId` fields.
+. If the {link-instance} has an `audit` table, then based on the {link-table}'s audit rules, audits about the inserted records are created.
+. If the table has a `POST_INSERT_CUSTOMIZER`, it is executed.
+
+=== Overloads
+
+`InsertAction` can be called in a few alternate forms, mostly just for convenience:
+
+[source,java]
+.If inserting a single record, get that record back instead of the InsertOutput:
+----
+InsertInput insertInput = new InsertInput();
+insertInput.setTableName("person");
+insertInput.setRecords(List.of(personRecord));
+QRecord insertedRecord = new InsertAction().executeForRecord(insertInput);
+
+// or more compactly, using InsertInput.withRecord (instead of withRecords)
+QRecord insertedRecord = new InsertAction()
+   .executeForRecord(new InsertInput("person").withRecord(personRecord));
+----
+
+[source,java]
+.Taking QRecordEntity objects as inputs instead of QRecords:
+----
+// insert a list of person entities:
+InsertInput insertInput = new InsertInput("person").withRecordEntities(personList);
+InsertOutput insertOutput = new InsertAction().execute(insertInput);
+
+// or for a single person entity (also mapping the output record back to an entity):
+Person insertedPerson = new Person(new InsertAction()
+   .executeForRecord(new InsertInput("person").withRecordEntity(person)));
+
+----
+
+=== InsertInput
+* `table` - *String, Required* - Name of the table that records are to be inserted into.
+* `records` - *List of QRecord, Required* - List of records to be inserted into the table.
+If the list is empty, the insert action does a `noop`.
+
+.Less common options
+* `inputSource` - *InputSource object, default: QInputSource.SYSTEM* - an indicator of what the source of the action is - generally, a `SYSTEM` triggered action, or a `USER` triggered action.
+** `InsertAction` will call the `shouldValidateRequiredFields()` method on this object to determine if it should validate that required fields on the records have values.
+Both `QInputSource.SYSTEM` and `QInputSource.USER` return `true` from this method, but application can define their own `InputSource` objects with different behavior.
+** In addition, this field can be used in pre- and post-insert customizers to drive further custom logic.
+* `skipUniqueKeyCheck` - *boolean, default: false* - control whether or not `InsertAction` should check for unique key violations before attempting to insert its records.
+** In a context where one has previously done this validation, or is okay letting the backend provide such checks, they may wish to avoid re-doing this work, and thus may set this property to `true`.
+* `omitDmlAudit` - *boolean, default: false* - control if the automatic DML audit that `InsertAction` generally performs should be omitted.
+* `auditContext` - *String* - optional message which can be included in DML audit messages, to give users more context about why the insert occurred.
+
+
+=== InsertOutput
+* `records` - *List of QRecord* - Copy of the input list of records, with details added based on the results of the input action.
+** If there were warnings or errors, the corresponding field (`warnings` or `errors`) will be set in the records.
+** If the insert action generated any values (such as a serial id or a default value), those values will be in the record's `fields` map.

docs/justfile

@@ -0,0 +1,12 @@
+## https://github.com/casey/just
+
+default:
+   just --list
+
+build-index-html:
+   asciidoctor -a docinfo=shared index.adoc
+   line=$(grep 'Last updated' index.html) && sed -i "s/id=\"content\">/&$line/" index.html
+
+build-and-publish-index-html: build-index-html
+   scp index.html first-node:/mnt/first-volume/dkelkhoff/nginx/html/justinsgotskinnylegs.com/qqq-docs.html
+   @echo "Updated: https://justinsgotskinnylegs.com/qqq-docs.html"

docs/metaData/Icons.adoc

@@ -0,0 +1,20 @@
+[#Icons]
+== Icons
+include::../variables.adoc[]
+
+#TODO#
+
+=== QIcon
+Icons are defined in a QQQ Instance in a `*QIcon*` object.
+
+#TODO#
+
+*QIcon Properties:*
+
+* `name` - *String* - Name of an icon from the https://mui.com/material-ui/material-icons/[Material UI Icon set]
+** Note that icon names from the link above need to be converted from _CamelCase_ to _underscore_case_...
+* `path` - *String* - Path to a file served under the application's web root, to be used as the icon.
+
+_Either `name` or `path` must be specified. If both are given, then name is used._
+
+

docs/metaData/PermissionRules.adoc

@@ -0,0 +1,13 @@
+[#PermissionRules]
+== Permission Rules
+include::../variables.adoc[]
+
+#TODO#
+
+=== PermissionRule
+#TODO#
+
+*PermissionRule Properties:*
+
+#TODO#
+

docs/metaData/QInstance.adoc

@@ -0,0 +1,39 @@
+[#QInstance]
+== QInstance
+include::../variables.adoc[]
+
+An application in QQQ is defined as a set of Meta Data objects.  These objects are all stored together in an object called a `*QInstance*`.
+
+Currently, a `QInstance` must be programmatically constructed in java code - e.g., by constructing objects which get added to the QInstance, for example:
+
+[source,java]
+.Adding meta-data for two tables and one process to a QInstance
+----
+QInstance qInstance = new QInstance();
+qInstance.addTable(definePersonTable());
+qInstance.addTable(defineHomeTable());
+qInstance.addProcess(defineSendPersonHomeProcess());
+----
+
+It is on the QQQ roadmap to allow meta-data to be defined in a non-programmatic way, for example, in YAML or JSON files, or even from a dynamic data source (e.g. a document or relational database).
+
+The middleware and/or frontends being used in an application will drive how the `QInstance` is connected to the running server/process.
+For example, using the `qqq-middleware-javalin` module, a the `QJavalinImplementation` class () has a constructor which takes a `QInstance` as an argument:
+
+[source,java]
+.Starting a QQQ Javalin middleware server - passing a QInstance as a parameter to the new QJavalinImplementation
+----
+QJavalinImplementation qJavalinImplementation = new QJavalinImplementation(qInstance);
+Javalin service = Javalin.create();
+service.routes(qJavalinImplementation.getRoutes());
+service.start();
+----
+
+*QBackendMetaData Setup Methods:*
+
+These are the methods that one is most likely to use when setting up (defining) a `QInstance` object:
+
+* asdf
+
+*QBackendMetaData Usage Methods:*
+

docs/misc/ProcessBackendSteps.adoc

@@ -0,0 +1,155 @@
+== Process Backend Steps
+include::../variables.adoc[]
+
+In many QQQ applications, much of the code that engineers write will take the form of Backend Steps for {link-processes}.
+Such code is defined in classes which implement the interface `BackendStep`.  
+This interface defines only a single method:
+[source,java]
+.BackendStep.java
+----
+public interface BackendStep
+{
+   /*******************************************************************************
+    ** Execute the backend step - using the request as input, and the result as output.
+    **
+    *******************************************************************************/
+   void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException;
+}
+
+----
+
+Process backend steps have access to state information - specifically, a list of records, and a map of name=value pairs - in the input & output objects.
+This state data is persisted by QQQ between steps (e.g., if a frontend step is presented to a user between backend steps).
+
+
+=== RunBackendStepInput
+All input data to the step is available in the `RunBackendStepInput` object.
+Key methods in this class are:
+
+* `getRecords()` - Returns the List of <<QRecords>> that are currently being acted on in the process.
+* `getValues()` - Returns a Map of String -> Serializable; name=value pairs that are the state-data of the process.
+** Values can be added to this state from a process's meta-data, from a screen, or from another backend step.
+* `getValue(String fieldName)` - Returns a specific value, by name, from the process's state.
+** This method has several variations that return the value as a specific type, such as `getValueString`, `getValueInteger`, `getValueBoolean`...
+* `getAsyncJobCallback()` - Accessor for an `AsyncJobCallback` object, which provides a way for the process backend step to communicate about its status or progress with a user running the process in a frontend.
+Provides methods:
+** `updateStatus(String message)` - Where general status messages can be given.
+For example, `"Loading census data"`
+** `updateStatus(int current, int total)` - For updating a progress meter.
+e.g., "47 of 1701" would be display by calling `.updateStatus(47, 1701)`
+* `getFrontendStepBehavior()` - Enum, indicating what should happen when a frontend step is encountered as the process's next step to run.
+Possible values are:
+** `BREAK` - Indicates that the process's execution should be suspended, so that the screen represented by the frontend step can be presented to a user.
+This would be the expected behavior if a process is being run by a user from a UI.
+** `SKIP` - Indicates that frontend steps should be skipped.
+This would be the expected behavior if a process is running from a scheduled job (without a user present to drive it), for example.
+** `FAIL` - Indicates that the process should end with an exception if a frontend step is encountered.
+** A backend step may want to act differently based on its frontendStepBehavior.
+For example, additional data may be looked up for displaying to a user if the behavior is `BREAK`.
+* `getBasepullLastRunTime()` - For <<BasepullConfiguration,Basepull>> processes, this is the `Instant` stored in the basepull table as the process's last run time.
+
+=== RunBackendStepOutput
+All output from a process step should be placed in its `RunBackendStepOutput` object (and/or stored to a backend, as appropriate).
+Key methods in this class are:
+
+* `addValue(String fieldName, Serializable value)` - Adds a single named value to the process's state, overwriting it the value if it already exists.
+* `addRecord(QRecord record)` - Add a `<<QRecord>>` to the process's output.
+* `addAuditSingleInput(AuditSingleInput auditSingleInput)` - Add a new entry to the process's list of audit inputs, to be stored at the completion of the process.
+** An `AuditSingleInput` object can most easily be built with the constructor: `AuditSingleInput(String tableName, QRecord record, String auditMessage)`.
+** Additional audit details messages (sub-bullets that accompany the primary `auditMessage`) can be added to an `AuditSingleInput` via the `addDetail(String message)` method.
+** _Note that at this time, the automatic storing of these audits is only provided by the execute step of a StreamedETLWithFrontendProcesses._
+
+=== Example
+
+[source,java]
+.Example of a BackendStep
+----
+/*******************************************************************************
+ ** For the "person" table's "Add Age" process -
+ ** For each input person record, add the specified yearsToAdd to their age.
+ *******************************************************************************/
+public class AddAge implements BackendStep
+{
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput)
+   {
+      /////////////////////////////////////////////////////////////////
+      // get the yearsToAdd input field value from the process input //
+      /////////////////////////////////////////////////////////////////
+      Integer yearsToAdd = runBackendStepInput.getValueInteger("yearsToAdd");
+      int totalYearsAdded = 0;
+
+      ///////////////////////////////////////////////////
+      // loop over the records passed into the process //
+      ///////////////////////////////////////////////////
+      for(QRecord record : runBackendStepInput.getRecords())
+      {
+         Integer age = record.getValueInteger("age");
+         age += yearsToAdd;
+         totalYearsAdded += yearsToAdd;
+
+         ////////////////////////////////////////////////////////////////////////////////////////////
+         // update the record with the new "age" value.                                            //
+         // note that this update record object will implicitly be available to the process's next //
+         // backend step, via the sharing of the processState object.                              //
+         ////////////////////////////////////////////////////////////////////////////////////////////
+         record.setValue("age", age);
+      }
+
+      /////////////////////////////////////////
+      // set an output value for the process //
+      /////////////////////////////////////////
+      runBackendStepOutput.addValue("totalYearsAdded", totalYearsAdded);
+   }
+}
+
+----
+
+
+=== Backend Steps for StreamedETLWithFrontendProcesses
+
+For <<StreamedETLWithFrontendProcess>> type processes, backend steps are defined a little bit differently than they are for other process types.
+In this type of process, the process meta-data defines 3 backend steps which are built-in to QQQ, and which do not have any custom application logic.
+These steps are:
+
+* `StreamedETLPreviewStep`
+* `StreamedETLValidateStep`
+* `StreamedETLExecuteStep`
+
+For custom application logic to be implemented in a StreamedETLWithFrontendProcesses, an application engineer must define (up to) 3 backend step classes which are loaded by the steps listed above.
+These application-defined steps must extend specific classes (which themselves implement the `BackendStep` interface), to provide the needed logic of this style of process.
+These steps are:
+
+* *Extract* - a subclass of `AbstractExtractStep` - is responsible for Extracting records from the source table.
+** For this step, we can often use the QQQ-provided class `ExtractViaQueryStep`, or sometimes a subclass of it.
+** The Extract step is called before the Preview, Validate, and Result screens, though for the Preview screen, it is set to only extract a small number of records (10).
+* *Transform* - a subclass of `AbstractTransformStep` - is responsible for applying the majority of the business logic of the process.
+In ETL terminology, this is the "Transform" action - which means applying some type of logical transformation an input record (found by the Extract step) to generate an output record (stored by the Load step).
+** A Transform step's `run` method will be called, potentially, multiple times, each time with a page of records in the `runBackendStepInput` parameter.
+** This method is responsible for adding records to the `runBackendStepOutput`, which will then be passed to the *Load* step.
+** This class is also responsible for implementing the method `getProcessSummary`, which provides the data to the *Validate* screen.
+** The run method will generally update ProcessSummaryLine objects to facilitate this functionality.
+** The Transform step is called before the Preview, Validate, and Result screens, consuming all records selected by the Extract step.
+* *Load* - a subclass of `AbstractLoadStep` - is responsible for the Load function of the ETL job.
+_A quick word on terminology - this step is actually doing what we are more likely to think of as storing data - which feels like the opposite of “loading” - but we use the name Load to keep in line with the ETL naming convention…_
+** The Load step is ONLY called before the Result screen is presented (possibly after Preview, if the user chose to skip validation, otherwise, after validation).
+** Similar to the Transform step, the Load step's `run` method will be called potentially multiple times, with pages of records in its input.
+** As such, the Load step is generally the only step where data writes should occur.
+*** e.g., a Transform step should not do any writes, as it will be called when the user is going to the Preview & Validate screens - e.g., before the user confirmed that they want to execute the action!
+** A common pattern is that the Load step just needs to insert or update the list of records output by the Transform step, in which case the QQQ-provided `LoadViaInsertStep` or `LoadViaUpdateStep` can be used, but custom use-cases can be built as well.
+
+Another distinction between StreamedELTWithFrontendProcess steps and general QQQ process backend steps, is that the list of records in the input & output objects is NOT shared for StreamedELTWithFrontendProcess steps.
+The direct implication of this is, that a Transform step MUST explicitly call `output.addRecord()` for any records that it wants to pass along to the Load step.
+
+==== Example
+
+[source,java]
+.Examples of a Transform and Load step for a StreamedELTWithFrontendProcess
+----
+   // todo!
+----
+
+#todo: more details on these 3 specialized types of process steps (e.g., method to overload, when stuff like pre-action is called; how summaries work).#

docs/misc/QContext.adoc

@@ -0,0 +1,30 @@
+== QContext
+include::../variables.adoc[]
+
+The class `QContext` contains a collection of thread-local variables, to define the current context of the QQQ code that is currently running.
+For example, what `QInstance` (meta-data container) is being used, what `QSession` (user attributes) is active, etc.
+
+Most of the time, main-line application code does not need to worry about setting up the `QContext` - although unit-test code can is a common exception to that rule.
+This is because all of QQQ's entry-points into execution (e.g., web context handlers, CLI startup methods, schedulers, and multi-threaded executors) take care of initializing the context.
+
+It is more common though, for application code to need to get data from the context, such as the current session or any piece of meta-data from the QInstance.
+The methods to access data from the `QContext` are fairly straightforward:
+
+=== Examples
+==== Get a QTableMetaData from the active QInstance
+[source,java]
+----
+QTableMeataData myTable = QContext.getQInstance().getTable("myTable");
+for(QFieldMeataData field : myTable.getFields().values())
+{
+   // ...
+}
+----
+
+==== Get a security key value for the current user session
+[source,java]
+----
+QSession           session   = QContext.getQSession();
+List<Serializable> clientIds = session.getSecurityKeyValues("clientId");
+----
+

docs/misc/QRecordEntities.adoc

@@ -0,0 +1,116 @@
+== QRecordEntities
+include::../variables.adoc[]
+
+While `<<QRecords>>` provide a flexible mechanism for passing around record data in a QQQ Application, they have one big disadvantage from the point-of-view of a Java Application:
+They do not provide a mechanism to ensure compile-time checks of field names or field types.
+
+As such, an alternative mechanism exists, which allows records in a QQQ application to be worked with following a more familiar Java Bean (Entity Bean) like pattern.
+This mechanism is known as a `QRecordEntity`.
+
+Specifically speaking, `QRecordEntity` is an abstract base class, whose purpose is to be the base class for entity-bean classes.
+Using reflection, `QRecordEntity` is able to convert objects back and forth from `QRecord` to specific entity-bean subtypes.
+
+For example, the method `QRecordEntity::toQRecord()` converts a record entity object into a `QRecord`.
+
+Inversely, `QRecordEntity::populateFromQRecord(QRecord record)` sets fields in a record entity object, based on the values in the supplied `QRecord`.
+It is conventional for a subclass of `QRecordEntity` to have both a no-arg (default) constructor, and a constructor that takes a `QRecord` as a parameter, and calls `populateFromQRecord`.
+
+In addition to these constructors, a `QRecordEntity` subclass will generally contain:
+
+* A `public static final String TABLE_NAME`, used throughout the application as a constant reference to the name for the {link-table}.
+* A series of `private` fields, corresponding to the fields in the table that the entity represents.
+** If these fields are annotated as `@QField()`, then the {link-table} meta-data for the table that the entity represents can in part be inferred by QQQ, by using the method `QTableMetaData::withFieldsFromEntity`.
+* `getX()`, `setX()`, and `withX()` methods for all of the entity's fields.
+
+=== Examples
+[source,java]
+.Example Definition of a QRecordEntity subclass:  Person.java
+----
+/*******************************************************************************
+ ** QRecordEntity for the person table.
+ *******************************************************************************/
+public class Person extends QRecordEntity
+{
+   public static final String TABLE_NAME = "person";
+
+   @QField(isEditable = false)
+   private Integer id;
+
+   @QField()
+   private String firstName;
+
+   @QField()
+   private String lastName;
+
+   @QField()
+   private Integer age;
+
+   // all other fields
+
+   /*******************************************************************************
+    ** Default constructor
+    *******************************************************************************/
+   public Person()
+   {
+   }
+
+
+   /*******************************************************************************
+    ** Constructor that takes a QRecord
+    *******************************************************************************/
+   public Person(QRecord record)
+   {
+      populateFromQRecord(record);
+   }
+
+
+   /*******************************************************************************
+    ** Custom method to concatenate firstName and lastName
+    *******************************************************************************/
+   public String getFullName()
+   {
+      //////////////////////////////////////////////////////////////////////
+      // if there were more than an example, we could do some null-checks //
+      // here to avoid silly output like "Bobby null" :)                  //
+      //////////////////////////////////////////////////////////////////////
+      return (firstName + " " + lastName);
+   }
+
+   // all getter, setter, and fluent setter (wither) methods
+}
+----
+
+The core ORM actions and process-execution actions of QQQ work with `QRecords`.
+However, application engineers may want to apply patterns like the following example, to gain the compile-time safety of `QRecordEntities`:
+
+[source,java]
+.Example Usage of a QRecordEntity
+----
+//////////////////////////////////////////////////////////////////////
+// assume we're working with the "person" table & entity from above //
+//////////////////////////////////////////////////////////////////////
+List<QRecord> recordsToUpdate = new ArrayList<>();
+for(QRecord record : inputRecordList)
+{
+   /////////////////////////////////////////////////////////////////////////////
+   // call the constructor that copies values from the record into the entity //
+   /////////////////////////////////////////////////////////////////////////////
+   Person person = new Person(record);
+
+   ////////////////////////////////////////////////
+   // call a custom method defined in the entity //
+   ////////////////////////////////////////////////
+   LOG.info("Processing: " + person.getFullName());
+
+   /////////////////////////////////////////////////////////////
+   // age is an Integer, so no type-conversion is needed here //
+   /////////////////////////////////////////////////////////////
+   person.setAge(person.getAge() + 1);
+
+   ///////////////////////////////////////////////////////////////////////////
+   // to pass the updated records to UpdateAction, convert them to QRecords //
+   ///////////////////////////////////////////////////////////////////////////
+   recordsToUpdate.add(person.toQRecord());
+}
+----
+

docs/misc/QRecords.adoc

@@ -0,0 +1,68 @@
+== QRecords
+include::../variables.adoc[]
+
+Almost all code inside a QQQ application will be dealing with *Records* (aka Tuples or Rows).
+That is: a collection of named values, representing a single Entity, Fact, or, Row from a {link-table}.
+
+The class that QQQ uses to work with records is called:  `QRecord`.
+
+=== Values
+At its core, a `QRecord` is a wrapper around a `Map<String, Serializable> values`.
+These are the *actual* values for the fields in the table for the record.
+That is, direct representations of the values as they are stored in the {link-backend}.
+
+The keys in the `values` map are names from the {link-fields} in the {link-table}.
+
+The values in `values` map are declared as `Serializable` (to help ensure the serializability of the `QRecord` as a whole).
+In practice, their types will be based on the `QFieldType` of the {link-field} that they correspond to.
+That will typically be one of: `String`, `Integer`, `Boolean`, `BigDecimal`, `Instant`, `LocalDate`, `LocalTime`, or `byte[]`.
+Be aware that `null` values may be in the `values` map, especially/per if the backend/table support `null`.
+
+To work with the `values` map, the following methods are provided:
+
+* `setValue(String fieldName, Serializable value)` - Sets a value for the specified field in the record.
+** Overloaded as `setValue(String fieldName, Object value)` - For some cases where the value may not be known to be `Serializable`.
+In this overload, if the value is `null` or `Serializable`, the primary version of `setValue` is called.
+Otherwise, the `value` is passed through `String::valueOf`, and the result is stored.
+** Overloaded as `setValue(QFieldMetaData field, Serializable value)` - which simply defers to the primary version of `setValue`, passing `field.getName()` as the first parameter.
+* `removeValue(String fieldName)` - Remove the given field from the `values` map.
+** Note that in some situations this is an important distinction from having a `null` value in the map (See <<UpdateAction)>>).
+* `setValues(Map<String, Serializable> values)` - Sets the full map of `values`.
+* `getValues()` - Returns the full map of `values`.
+* `getValue(String fieldName)` - Returns the value for the named field - possibly `null` - as a `Serializable`.
+* Several type-specific variations of `getValueX(String fieldName)`, where internally, values will be not exactly type-cast, but effectively converted (if possible) to the requested type.
+These conversions are done using the `ValueUtils.getValueAsX(Object)` methods.
+These methods are generally the preferred/cleanest way to get record values in application code, when it is needed in a type-specific way .
+** `getValueString(String fieldName)`
+** `getValueInteger(String fieldName)`
+** `getValueBoolean(String fieldName)`
+** `getValueBigDecimal(String fieldName)`
+** `getValueInstant(String fieldName)`
+** `getValueLocalDate(String fieldName)`
+** `getValueLocalTime(String fieldName)`
+** `getValueByteArray(String fieldName)`
+
+=== Display Values
+In addition to the `values` map, a `QRecord` contains another map called `displayValues`, which only stores `String` values.
+That is to say, values for other types are stringified, based on their {link-field}'s type and `displayFormat` property.
+In addition, fields which have a `possibleValueSource` property will have their translated values set in the `displayValues` map.
+
+By default, a `QRecord` will not have its `displayValues` populated.
+To populate `displayValues`, the <<QueryAction>> and <<GetAction>> classes take a property in their inputs called `shouldGenerateDisplayValues`, which must be set to `true` to generate `displayValues`.
+In addition, these two actions also have a property `shouldTranslatePossibleValues` in their inputs, which needs to be set to `true` if possible-value lookups are to be performed.
+
+As an alternative to the `shouldGenerateDisplayValues` and `shouldTranslatePossibleValues` inputs to <<QueryAction>> and <<GetAction>>, one can directly call the `QValueFormatter.setDisplayValuesInRecords` and/or `qPossibleValueTranslator.translatePossibleValuesInRecords` methods.
+Or, for special cases, `setDisplayValue(String fieldName, String displayValue)` or `setDisplayValues(Map<String, String> displayValues)` can be called directly.
+
+=== Backend Details
+Sometimes a backend may want to place additional data in a `QRecord` that doesn't exactly correspond to a field.
+To do this, the `Map<String, Serializable> backendDetails` member is used.
+
+For example, an API backend may store the full JSON `String` that came from the API as a backend detail in a `QRecord`.
+Or fields that are marked as `isHeavy`, if the full (heavy) value of the field hasn't been fetched, then the lengths of any such heavy fields may be stored in `backendDetails`.
+
+=== Errors and Warnings
+#todo#
+
+=== Associated Records
+#todo#

pom.xml

@@ -330,28 +330,28 @@ fi
 
    <!-- mvn javadoc:aggregate -->
    <reporting>
-    <plugins>
-      <plugin>
-        <groupId>org.apache.maven.plugins</groupId>
-        <artifactId>maven-javadoc-plugin</artifactId>
-        <version>3.6.2</version>
-        <reportSets>
-          <reportSet>
-            <id>aggregate</id>
-            <inherited>false</inherited>
-            <reports>
-              <report>aggregate</report>
-            </reports>
-          </reportSet>
-          <reportSet>
-            <id>default</id>
-            <reports>
-              <report>javadoc</report>
-            </reports>
-          </reportSet>
-        </reportSets>
-      </plugin>
-    </plugins>
+      <plugins>
+         <plugin>
+            <groupId>org.apache.maven.plugins</groupId>
+            <artifactId>maven-javadoc-plugin</artifactId>
+            <version>3.6.2</version>
+            <reportSets>
+               <reportSet>
+                  <id>aggregate</id>
+                  <inherited>false</inherited>
+                  <reports>
+                     <report>aggregate</report>
+                  </reports>
+               </reportSet>
+               <reportSet>
+                  <id>default</id>
+                  <reports>
+                     <report>javadoc</report>
+                  </reports>
+               </reportSet>
+            </reportSets>
+         </plugin>
+      </plugins>
    </reporting>
 
    <repositories>

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/audits/AuditAction.java

@@ -295,7 +295,7 @@ public class AuditAction extends AbstractQActionFunction<AuditInput, AuditOutput
          }
          catch(Exception e)
          {
-            LOG.error("Error performing an audit", e);
+            LOG.warn("Error performing an audit", e);
          }
       }
 

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/audits/DMLAuditAction.java

@@ -187,7 +187,7 @@ public class DMLAuditAction extends AbstractQActionFunction<DMLAuditInput, DMLAu
       }
       catch(Exception e)
       {
-         LOG.error("Error performing DML audit", e, logPair("type", String.valueOf(dmlType)), logPair("table", table.getName()));
+         LOG.warn("Error performing DML audit", e, logPair("type", String.valueOf(dmlType)), logPair("table", table.getName()));
       }
 
       return (output);

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/InsertAction.java

@@ -242,7 +242,7 @@ public class InsertAction extends AbstractQActionFunction<InsertInput, InsertOut
 
       setDefaultValuesInRecords(table, insertInput.getRecords());
 
-      ValueBehaviorApplier.applyFieldBehaviors(ValueBehaviorApplier.Action.INSERT, insertInput.getInstance(), table, insertInput.getRecords());
+      ValueBehaviorApplier.applyFieldBehaviors(ValueBehaviorApplier.Action.INSERT, insertInput.getInstance(), table, insertInput.getRecords(), null);
 
       runPreInsertCustomizerIfItIsTime(insertInput, preInsertCustomizer, AbstractPreInsertCustomizer.WhenToRun.BEFORE_UNIQUE_KEY_CHECKS);
       setErrorsIfUniqueKeyErrors(insertInput, table);
@@ -455,7 +455,7 @@ public class InsertAction extends AbstractQActionFunction<InsertInput, InsertOut
    private QBackendModuleInterface getBackendModuleInterface(QBackendMetaData backend) throws QException
    {
       QBackendModuleDispatcher qBackendModuleDispatcher = new QBackendModuleDispatcher();
-      QBackendModuleInterface qModule = qBackendModuleDispatcher.getQBackendModule(backend);
+      QBackendModuleInterface  qModule                  = qBackendModuleDispatcher.getQBackendModule(backend);
       return (qModule);
    }
 

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/ReplaceAction.java

@@ -136,19 +136,22 @@ public class ReplaceAction extends AbstractQActionFunction<ReplaceInput, Replace
          UpdateOutput updateOutput = new UpdateAction().execute(updateInput);
          output.setUpdateOutput(updateOutput);
 
-         QQueryFilter deleteFilter = new QQueryFilter(new QFilterCriteria(primaryKeyField, QCriteriaOperator.NOT_IN, primaryKeysToKeep));
-         if(input.getFilter() != null)
+         if(input.getPerformDeletes())
          {
-            deleteFilter.addSubFilter(input.getFilter());
-         }
+            QQueryFilter deleteFilter = new QQueryFilter(new QFilterCriteria(primaryKeyField, QCriteriaOperator.NOT_IN, primaryKeysToKeep));
+            if(input.getFilter() != null)
+            {
+               deleteFilter.addSubFilter(input.getFilter());
+            }
 
-         DeleteInput deleteInput = new DeleteInput();
-         deleteInput.setTableName(table.getName());
-         deleteInput.setQueryFilter(deleteFilter);
-         deleteInput.setTransaction(transaction);
-         deleteInput.setOmitDmlAudit(input.getOmitDmlAudit());
-         DeleteOutput deleteOutput = new DeleteAction().execute(deleteInput);
-         output.setDeleteOutput(deleteOutput);
+            DeleteInput deleteInput = new DeleteInput();
+            deleteInput.setTableName(table.getName());
+            deleteInput.setQueryFilter(deleteFilter);
+            deleteInput.setTransaction(transaction);
+            deleteInput.setOmitDmlAudit(input.getOmitDmlAudit());
+            DeleteOutput deleteOutput = new DeleteAction().execute(deleteInput);
+            output.setDeleteOutput(deleteOutput);
+         }
 
          if(weOwnTheTransaction)
          {

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/UpdateAction.java

@@ -57,6 +57,8 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateOutput;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.DynamicDefaultValueBehavior;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.FieldBehavior;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
 import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinOn;
@@ -72,6 +74,7 @@ import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleDispatcher;
 import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleInterface;
 import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
 import com.kingsrook.qqq.backend.core.utils.ValueUtils;
+import org.apache.commons.lang.BooleanUtils;
 import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
 
 
@@ -244,7 +247,13 @@ public class UpdateAction
       /////////////////////////////
       // run standard validators //
       /////////////////////////////
-      ValueBehaviorApplier.applyFieldBehaviors(ValueBehaviorApplier.Action.UPDATE, updateInput.getInstance(), table, updateInput.getRecords());
+      Set<FieldBehavior<?>> behaviorsToOmit = null;
+      if(BooleanUtils.isTrue(updateInput.getOmitModifyDateUpdate()))
+      {
+         behaviorsToOmit = Set.of(DynamicDefaultValueBehavior.MODIFY_DATE);
+      }
+
+      ValueBehaviorApplier.applyFieldBehaviors(ValueBehaviorApplier.Action.UPDATE, updateInput.getInstance(), table, updateInput.getRecords(), behaviorsToOmit);
       validatePrimaryKeysAreGiven(updateInput);
 
       if(oldRecordList.isPresent())

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/ValueBehaviorApplier.java

@@ -23,6 +23,7 @@ package com.kingsrook.qqq.backend.core.actions.values;
 
 
 import java.util.List;
+import java.util.Set;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
 import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.FieldBehavior;
@@ -32,7 +33,7 @@ import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
 
 
 /*******************************************************************************
- ** Utility class to apply value behaviors to records.  
+ ** Utility class to apply value behaviors to records.
  *******************************************************************************/
 public class ValueBehaviorApplier
 {
@@ -51,7 +52,7 @@ public class ValueBehaviorApplier
    /*******************************************************************************
     **
     *******************************************************************************/
-   public static void applyFieldBehaviors(Action action, QInstance instance, QTableMetaData table, List<QRecord> recordList)
+   public static void applyFieldBehaviors(Action action, QInstance instance, QTableMetaData table, List<QRecord> recordList, Set<FieldBehavior<?>> behaviorsToOmit)
    {
       if(CollectionUtils.nullSafeIsEmpty(recordList))
       {
@@ -62,7 +63,7 @@ public class ValueBehaviorApplier
       {
          for(FieldBehavior<?> fieldBehavior : CollectionUtils.nonNullCollection(field.getBehaviors()))
          {
-            fieldBehavior.apply(action, recordList, instance, table, field);
+            fieldBehavior.apply(action, recordList, instance, table, field, behaviorsToOmit);
          }
       }
    }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/context/QContext.java

@@ -22,6 +22,9 @@
 package com.kingsrook.qqq.backend.core.context;
 
 
+import java.io.Serializable;
+import java.util.HashMap;
+import java.util.Map;
 import java.util.Optional;
 import java.util.Stack;
 import com.kingsrook.qqq.backend.core.actions.QBackendTransaction;
@@ -31,6 +34,7 @@ import com.kingsrook.qqq.backend.core.logging.QLogger;
 import com.kingsrook.qqq.backend.core.model.actions.AbstractActionInput;
 import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
 import com.kingsrook.qqq.backend.core.model.session.QSession;
+import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
 
 
 /*******************************************************************************
@@ -47,6 +51,7 @@ public class QContext
    private static ThreadLocal<QBackendTransaction>        qBackendTransactionThreadLocal = new ThreadLocal<>();
    private static ThreadLocal<Stack<AbstractActionInput>> actionStackThreadLocal         = new ThreadLocal<>();
 
+   private static ThreadLocal<Map<String, Serializable>> objectsThreadLocal = new ThreadLocal<>();
 
 
    /*******************************************************************************
@@ -132,6 +137,7 @@ public class QContext
       qSessionThreadLocal.remove();
       qBackendTransactionThreadLocal.remove();
       actionStackThreadLocal.remove();
+      objectsThreadLocal.remove();
    }
 
 
@@ -259,4 +265,92 @@ public class QContext
 
       return (Optional.of(actionStackThreadLocal.get().get(0)));
    }
+
+
+   /*******************************************************************************
+    ** get one named object from the Context for the current thread.  may return null.
+    *******************************************************************************/
+   public static Serializable getObject(String key)
+   {
+      if(objectsThreadLocal.get() == null)
+      {
+         return null;
+      }
+      return objectsThreadLocal.get().get(key);
+   }
+
+
+   /*******************************************************************************
+    ** get one named object from the Context for the current thread, cast to the
+    ** specified type if possible.  if not found, or wrong type, empty is returned.
+    *******************************************************************************/
+   public static <T extends Serializable> Optional<T> getObject(String key, Class<T> type)
+   {
+      Serializable object = getObject(key);
+
+      if(type.isInstance(object))
+      {
+         return Optional.of(type.cast(object));
+      }
+      else if(object == null)
+      {
+         return Optional.empty();
+      }
+      else
+      {
+         LOG.warn("Unexpected type of object found in session under key [" + key + "]",
+            logPair("expectedType", type.getName()),
+            logPair("actualType", object.getClass().getName())
+         );
+         return Optional.empty();
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** put a named object into the Context for the current thread.
+    *******************************************************************************/
+   public static void setObject(String key, Serializable object)
+   {
+      if(objectsThreadLocal.get() == null)
+      {
+         objectsThreadLocal.set(new HashMap<>());
+      }
+      objectsThreadLocal.get().put(key, object);
+   }
+
+
+
+   /*******************************************************************************
+    ** remove a named object from the Context of the current thread.
+    *******************************************************************************/
+   public static void removeObject(String key)
+   {
+      if(objectsThreadLocal.get() != null)
+      {
+         objectsThreadLocal.get().remove(key);
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** get the full map of named objects for the current thread (possibly null).
+    *******************************************************************************/
+   public static Map<String, Serializable> getObjects()
+   {
+      return objectsThreadLocal.get();
+   }
+
+
+
+   /*******************************************************************************
+    ** fully replace the map of named objets for the current thread.
+    *******************************************************************************/
+   public static void setObjects(Map<String, Serializable> objects)
+   {
+      objectsThreadLocal.set(objects);
+   }
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidator.java

@@ -24,8 +24,10 @@ package com.kingsrook.qqq.backend.core.instances;
 
 import java.io.Serializable;
 import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
 import java.lang.reflect.Modifier;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashSet;
 import java.util.List;
@@ -43,6 +45,7 @@ import com.kingsrook.qqq.backend.core.actions.processes.BackendStep;
 import com.kingsrook.qqq.backend.core.actions.scripts.TestScriptActionInterface;
 import com.kingsrook.qqq.backend.core.actions.values.QCustomPossibleValueProvider;
 import com.kingsrook.qqq.backend.core.exceptions.QInstanceValidationException;
+import com.kingsrook.qqq.backend.core.instances.validation.plugins.QInstanceValidatorPluginInterface;
 import com.kingsrook.qqq.backend.core.logging.QLogger;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy;
@@ -52,9 +55,11 @@ import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
 import com.kingsrook.qqq.backend.core.model.metadata.QSupplementalInstanceMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.authentication.QAuthenticationMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.automation.QAutomationProviderMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
 import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeType;
 import com.kingsrook.qqq.backend.core.model.metadata.dashboard.ParentWidgetMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.dashboard.QWidgetMetaDataInterface;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.AdornmentType;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.FieldAdornment;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
@@ -64,16 +69,21 @@ import com.kingsrook.qqq.backend.core.model.metadata.joins.QJoinMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppChildMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppSection;
+import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSource;
 import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.processes.QStepMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.processes.QSupplementalProcessMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.queues.QQueueMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.queues.QQueueProviderMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.queues.SQSQueueProviderMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportDataSource;
 import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportField;
+import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportView;
 import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.QScheduleMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.security.FieldSecurityLock;
+import com.kingsrook.qqq.backend.core.model.metadata.security.QSecurityKeyType;
 import com.kingsrook.qqq.backend.core.model.metadata.security.RecordSecurityLock;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.AssociatedScript;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.Association;
@@ -90,6 +100,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.tables.cache.CacheOf;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.cache.CacheUseCase;
 import com.kingsrook.qqq.backend.core.modules.authentication.QAuthenticationModuleCustomizerInterface;
 import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
+import com.kingsrook.qqq.backend.core.utils.ListingHash;
 import com.kingsrook.qqq.backend.core.utils.StringUtils;
 import com.kingsrook.qqq.backend.core.utils.ValueUtils;
 import com.kingsrook.qqq.backend.core.utils.lambdas.UnsafeLambda;
@@ -111,6 +122,8 @@ public class QInstanceValidator
 
    private boolean printWarnings = false;
 
+   private static ListingHash<Class<?>, QInstanceValidatorPluginInterface<?>> validatorPlugins = new ListingHash<>();
+
    private List<String> errors = new ArrayList<>();
 
 
@@ -171,6 +184,8 @@ public class QInstanceValidator
          validateSupplementalMetaData(qInstance);
 
          validateUniqueTopLevelNames(qInstance);
+
+         runPlugins(QInstance.class, qInstance, qInstance);
       }
       catch(Exception e)
       {
@@ -189,6 +204,57 @@ public class QInstanceValidator
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public static void addValidatorPlugin(QInstanceValidatorPluginInterface<?> plugin)
+   {
+      Optional<Method> validateMethod = Arrays.stream(plugin.getClass().getDeclaredMethods())
+         .filter(m -> m.getName().equals("validate")
+            && m.getParameterCount() == 3
+            && !m.getParameterTypes()[0].equals(Object.class)
+            && m.getParameterTypes()[1].equals(QInstance.class)
+            && m.getParameterTypes()[2].equals(QInstanceValidator.class)
+         ).findFirst();
+
+      if(validateMethod.isPresent())
+      {
+         Class<?> parameterType = validateMethod.get().getParameterTypes()[0];
+         validatorPlugins.add(parameterType, plugin);
+      }
+      else
+      {
+         LOG.warn("Could not find validate method on validator plugin [" + plugin.getClass().getName() + "] (to infer type being validated) - this plugin will not be used.");
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public static void removeAllValidatorPlugins()
+   {
+      validatorPlugins.clear();
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private <T> void runPlugins(Class<T> c, T t, QInstance qInstance)
+   {
+      for(QInstanceValidatorPluginInterface<?> plugin : CollectionUtils.nonNullList(validatorPlugins.get(c)))
+      {
+         @SuppressWarnings("unchecked")
+         QInstanceValidatorPluginInterface<T> processPlugin = (QInstanceValidatorPluginInterface<T>) plugin;
+         processPlugin.validate(t, qInstance, this);
+      }
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/
@@ -197,6 +263,8 @@ public class QInstanceValidator
       for(QSupplementalInstanceMetaData supplementalInstanceMetaData : CollectionUtils.nonNullMap(qInstance.getSupplementalMetaData()).values())
       {
          supplementalInstanceMetaData.validate(qInstance, this);
+
+         runPlugins(QSupplementalInstanceMetaData.class, supplementalInstanceMetaData, qInstance);
       }
    }
 
@@ -234,6 +302,8 @@ public class QInstanceValidator
             {
                assertCondition(qInstance.getPossibleValueSource(securityKeyType.getPossibleValueSourceName()) != null, "Unrecognized possibleValueSourceName in securityKeyType: " + name);
             }
+
+            runPlugins(QSecurityKeyType.class, securityKeyType, qInstance);
          }
       });
    }
@@ -280,6 +350,8 @@ public class QInstanceValidator
                assertNoException(() -> qInstance.getTable(join.getRightTable()).getField(orderBy.getFieldName()), "Field name " + orderBy.getFieldName() + " in orderBy for join " + joinName + " is not a defined field in the right-table " + join.getRightTable());
             }
          }
+
+         runPlugins(QJoinMetaData.class, join, qInstance);
       });
    }
 
@@ -343,6 +415,8 @@ public class QInstanceValidator
                assertCondition(StringUtils.hasContent(sqsQueueProvider.getBaseURL()), "Missing baseURL for SQSQueueProvider: " + name);
                assertCondition(StringUtils.hasContent(sqsQueueProvider.getRegion()), "Missing region for SQSQueueProvider: " + name);
             }
+
+            runPlugins(QQueueProviderMetaData.class, queueProvider, qInstance);
          });
       }
 
@@ -357,6 +431,8 @@ public class QInstanceValidator
             {
                assertCondition(qInstance.getProcesses() != null && qInstance.getProcess(queue.getProcessName()) != null, "Unrecognized processName for queue: " + name);
             }
+
+            runPlugins(QQueueMetaData.class, queue, qInstance);
          });
       }
    }
@@ -375,6 +451,8 @@ public class QInstanceValidator
             assertCondition(Objects.equals(backendName, backend.getName()), "Inconsistent naming for backend: " + backendName + "/" + backend.getName() + ".");
 
             backend.performValidation(this);
+
+            runPlugins(QBackendMetaData.class, backend, qInstance);
          });
       }
    }
@@ -392,6 +470,8 @@ public class QInstanceValidator
          {
             assertCondition(Objects.equals(name, automationProvider.getName()), "Inconsistent naming for automationProvider: " + name + "/" + automationProvider.getName() + ".");
             assertCondition(automationProvider.getType() != null, "Missing type for automationProvider: " + name);
+
+            runPlugins(QAutomationProviderMetaData.class, automationProvider, qInstance);
          });
       }
    }
@@ -410,6 +490,8 @@ public class QInstanceValidator
          {
             validateSimpleCodeReference("Instance Authentication meta data customizer ", authentication.getCustomizer(), QAuthenticationModuleCustomizerInterface.class);
          }
+
+         runPlugins(QAuthenticationMetaData.class, authentication, qInstance);
       }
    }
 
@@ -532,6 +614,8 @@ public class QInstanceValidator
             {
                supplementalTableMetaData.validate(qInstance, table, this);
             }
+
+            runPlugins(QTableMetaData.class, table, qInstance);
          });
       }
    }
@@ -1330,6 +1414,7 @@ public class QInstanceValidator
                supplementalProcessMetaData.validate(qInstance, process, this);
             }
 
+            runPlugins(QProcessMetaData.class, process, qInstance);
          });
       }
    }
@@ -1436,6 +1521,8 @@ public class QInstanceValidator
                   // view.getTitleFormat(); view.getTitleFields(); // validate these match?
                }
             }
+
+            runPlugins(QReportMetaData.class, report, qInstance);
          });
       }
    }
@@ -1556,9 +1643,9 @@ public class QInstanceValidator
                }
             }
 
-            //////////////////////////////////////////
-            // validate field sections in the table //
-            //////////////////////////////////////////
+            ////////////////////////////////////////
+            // validate field sections in the app //
+            ////////////////////////////////////////
             Set<String> childNamesInSections = new HashSet<>();
             if(app.getSections() != null)
             {
@@ -1586,6 +1673,8 @@ public class QInstanceValidator
             {
                assertCondition(qInstance.getWidget(widgetName) != null, "App " + appName + " widget " + widgetName + " is not a recognized widget.");
             }
+
+            runPlugins(QAppMetaData.class, app, qInstance);
          });
       }
    }
@@ -1618,6 +1707,8 @@ public class QInstanceValidator
                      }
                   }
                }
+
+               runPlugins(QWidgetMetaDataInterface.class, widget, qInstance);
             }
          );
       }
@@ -1703,6 +1794,8 @@ public class QInstanceValidator
                   }
                   default -> errors.add("Unexpected possibleValueSource type: " + possibleValueSource.getType());
                }
+
+               runPlugins(QPossibleValueSource.class, possibleValueSource, qInstance);
             }
          });
       }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/validation/plugins/BasepullExtractStepValidator.java

@@ -0,0 +1,87 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.instances.validation.plugins;
+
+
+import com.kingsrook.qqq.backend.core.actions.customizers.QCodeLoader;
+import com.kingsrook.qqq.backend.core.instances.QInstanceValidator;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
+import com.kingsrook.qqq.backend.core.processes.implementations.basepull.BasepullExtractStepInterface;
+
+
+/*******************************************************************************
+ ** instance validator plugin, to ensure that a process which is a basepull uses
+ ** an extract step marked for basepulls.
+ *******************************************************************************/
+public class BasepullExtractStepValidator implements QInstanceValidatorPluginInterface<QProcessMetaData>
+{
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public void validate(QProcessMetaData process, QInstance qInstance, QInstanceValidator qInstanceValidator)
+   {
+      ///////////////////////////////////////////////////////////////////////////
+      // if there's no basepull config on the process, don't do any validation //
+      ///////////////////////////////////////////////////////////////////////////
+      if(process.getBasepullConfiguration() == null)
+      {
+         return;
+      }
+
+      //////////////////////////////////////////////////////////////////////////////////////////
+      // try to find an input field in the process, w/ a defaultValue that's a QCodeReference //
+      // and is an instance of BasepullExtractStepInterface                                   //
+      //////////////////////////////////////////////////////////////////////////////////////////
+      boolean foundBasepullExtractStep = false;
+      for(QFieldMetaData field : process.getInputFields())
+      {
+         if(field.getDefaultValue() != null && field.getDefaultValue() instanceof QCodeReference codeReference)
+         {
+            try
+            {
+               BasepullExtractStepInterface extractStep = QCodeLoader.getAdHoc(BasepullExtractStepInterface.class, codeReference);
+               if(extractStep != null)
+               {
+                  foundBasepullExtractStep = true;
+               }
+            }
+            catch(Exception e)
+            {
+               //////////////////////////////////////////////////////
+               // ok, just means we haven't found our extract step //
+               //////////////////////////////////////////////////////
+            }
+         }
+      }
+
+      ///////////////////////////////////////////////////////////
+      // validate we could find a BasepullExtractStepInterface //
+      ///////////////////////////////////////////////////////////
+      qInstanceValidator.assertCondition(foundBasepullExtractStep, "Process [" + process.getName() + "] has a basepullConfiguration, but does not have a field with a default value that is a BasepullExtractStepInterface CodeReference");
+   }
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/validation/plugins/QInstanceValidatorPluginInterface.java

@@ -0,0 +1,41 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.instances.validation.plugins;
+
+
+import com.kingsrook.qqq.backend.core.instances.QInstanceValidator;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+
+
+/*******************************************************************************
+ ** Interface for additional / optional q instance validators.  Some will be
+ ** provided by QQQ - others can be defined by applications.
+ *******************************************************************************/
+public interface QInstanceValidatorPluginInterface<T>
+{
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   void validate(T object, QInstance qInstance, QInstanceValidator qInstanceValidator);
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/logging/CollectedLogMessage.java

@@ -0,0 +1,153 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.logging;
+
+
+import org.apache.logging.log4j.Level;
+import org.json.JSONException;
+import org.json.JSONObject;
+
+
+/*******************************************************************************
+ ** A log message, which can be "collected" by the QCollectingLogger.
+ *******************************************************************************/
+public class CollectedLogMessage
+{
+   private Level     level;
+   private String    message;
+   private Throwable exception;
+
+
+
+   /*******************************************************************************
+    ** Constructor
+    **
+    *******************************************************************************/
+   public CollectedLogMessage()
+   {
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for message
+    *******************************************************************************/
+   public String getMessage()
+   {
+      return (this.message);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for message
+    *******************************************************************************/
+   public void setMessage(String message)
+   {
+      this.message = message;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for message
+    *******************************************************************************/
+   public CollectedLogMessage withMessage(String message)
+   {
+      this.message = message;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for exception
+    *******************************************************************************/
+   public Throwable getException()
+   {
+      return (this.exception);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for exception
+    *******************************************************************************/
+   public void setException(Throwable exception)
+   {
+      this.exception = exception;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for exception
+    *******************************************************************************/
+   public CollectedLogMessage withException(Throwable exception)
+   {
+      this.exception = exception;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for level
+    **
+    *******************************************************************************/
+   public Level getLevel()
+   {
+      return level;
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for level
+    **
+    *******************************************************************************/
+   public void setLevel(Level level)
+   {
+      this.level = level;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for level
+    **
+    *******************************************************************************/
+   public CollectedLogMessage withLevel(Level level)
+   {
+      this.level = level;
+      return (this);
+   }
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public JSONObject getMessageAsJSONObject() throws JSONException
+   {
+      return (new JSONObject(getMessage()));
+   }
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/logging/QCollectingLogger.java

@@ -0,0 +1,155 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.logging;
+
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Properties;
+import org.apache.logging.log4j.Level;
+import org.apache.logging.log4j.Logger;
+import org.apache.logging.log4j.Marker;
+import org.apache.logging.log4j.message.Message;
+import org.apache.logging.log4j.message.ObjectMessage;
+import org.apache.logging.log4j.message.SimpleMessageFactory;
+import org.apache.logging.log4j.simple.SimpleLogger;
+import org.apache.logging.log4j.util.PropertiesUtil;
+
+
+/*******************************************************************************
+ ** QQQ log4j implementation, used within a QLogger, to "collect" log messages
+ ** in an internal list - the idea being - for tests, to assert that logs happened.
+ *******************************************************************************/
+public class QCollectingLogger extends SimpleLogger
+{
+   private List<CollectedLogMessage> collectedMessages = new ArrayList<>();
+
+   ///////////////////////////////////////////////////////////////////////////////////////////////////////////
+   // just in case one of these gets activated, and left on, put a limit on how many messages we'll collect //
+   ///////////////////////////////////////////////////////////////////////////////////////////////////////////
+   private int capacity = 100;
+
+   private Logger logger;
+
+
+
+   /*******************************************************************************
+    ** Constructor
+    **
+    *******************************************************************************/
+   public QCollectingLogger(Logger logger)
+   {
+      super(logger.getName(), logger.getLevel(), false, false, true, false, "", new SimpleMessageFactory(), new PropertiesUtil(new Properties()), System.out);
+      this.logger = logger;
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public void logMessage(String fqcn, Level level, Marker marker, Message message, Throwable throwable)
+   {
+      ////////////////////////////////////////////
+      // add this log message to our collection //
+      ////////////////////////////////////////////
+      collectedMessages.add(new CollectedLogMessage()
+         .withLevel(level)
+         .withMessage(message.getFormattedMessage())
+         .withException(throwable));
+
+      ////////////////////////////////////////////////////////////////////////////////////////
+      // if we've gone over our capacity, remove the 1st entry until we're back at capacity //
+      ////////////////////////////////////////////////////////////////////////////////////////
+      while(collectedMessages.size() > capacity)
+      {
+         collectedMessages.remove(0);
+      }
+
+      //////////////////////////////////////////////////////////////////////
+      // update the message that we log to indicate that we collected it. //
+      // if it looks like JSON, insert as a name:value pair; else text.   //
+      //////////////////////////////////////////////////////////////////////
+      String formattedMessage = message.getFormattedMessage();
+      String updatedMessage;
+      if(formattedMessage.startsWith("{"))
+      {
+         updatedMessage = """
+            {"collected":true,""" + formattedMessage.substring(1);
+      }
+      else
+      {
+         updatedMessage = "[Collected] " + formattedMessage;
+      }
+      ObjectMessage myMessage = new ObjectMessage(updatedMessage);
+
+      ///////////////////////////////////////////////////////////////////////////////////////
+      // log the message with the original log4j logger, with our slightly updated message //
+      ///////////////////////////////////////////////////////////////////////////////////////
+      logger.logMessage(level, marker, fqcn, null, myMessage, throwable);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for logger
+    **
+    *******************************************************************************/
+   public void setLogger(Logger logger)
+   {
+      this.logger = logger;
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for collectedMessages
+    **
+    *******************************************************************************/
+   public List<CollectedLogMessage> getCollectedMessages()
+   {
+      return collectedMessages;
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public void clear()
+   {
+      this.collectedMessages.clear();
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for capacity
+    **
+    *******************************************************************************/
+   public void setCapacity(int capacity)
+   {
+      this.capacity = capacity;
+   }
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/logging/QLogger.java

@@ -119,6 +119,34 @@ public class QLogger
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public static QCollectingLogger activateCollectingLoggerForClass(Class<?> c)
+   {
+      Logger            loggerFromLogManager = LogManager.getLogger(c);
+      QCollectingLogger collectingLogger     = new QCollectingLogger(loggerFromLogManager);
+
+      QLogger qLogger = getLogger(c);
+      qLogger.setLogger(collectingLogger);
+
+      return collectingLogger;
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public static void deactivateCollectingLoggerForClass(Class<?> c)
+   {
+      Logger  loggerFromLogManager = LogManager.getLogger(c);
+      QLogger qLogger              = getLogger(c);
+      qLogger.setLogger(loggerFromLogManager);
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/
@@ -518,7 +546,7 @@ public class QLogger
    /*******************************************************************************
     **
     *******************************************************************************/
-   private String makeJsonString(String message, Throwable t, List<LogPair> logPairList)
+   protected String makeJsonString(String message, Throwable t, List<LogPair> logPairList)
    {
       if(logPairList == null)
       {
@@ -620,4 +648,15 @@ public class QLogger
       exceptionList.get(0).setHasLoggedLevel(level);
       return (level);
    }
+
+
+
+   /*******************************************************************************
+    ** Setter for logger
+    **
+    *******************************************************************************/
+   private void setLogger(Logger logger)
+   {
+      this.logger = logger;
+   }
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/audits/AuditDetailAccumulator.java

@@ -0,0 +1,156 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.model.actions.audits;
+
+
+import java.io.Serializable;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Optional;
+import com.kingsrook.qqq.backend.core.context.QContext;
+import com.kingsrook.qqq.backend.core.logging.QLogger;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.data.QRecordEntity;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
+
+
+/*******************************************************************************
+ ** Object to accumulate multiple audit-details to be recorded under a single
+ ** audit per-record, within a process step.  Especially useful if/when the
+ ** process step spreads its work out through multiple classes.
+ **
+ ** Pattern of usage looks like:
+ **
+ ** <pre>
+ ** // declare as a field (or local) w/ message for the audit headers
+ ** private AuditDetailAccumulator auditDetailAccumulator = new AuditDetailAccumulator("Audit header message");
+ **
+ **  // put into thread context
+ **  AuditDetailAccumulator.setInContext(auditDetailAccumulator);
+ **
+ **  // add a detail message for a record
+ **  auditDetailAccumulator.addAuditDetail(tableName, record, "Detail message");
+ **
+ **  // in another class, get the accumulator from context and safely add a detail message
+ **  AuditDetailAccumulator.getFromContext().ifPresent(ada -> ada.addAuditDetail(tableName, record, "More Details"));
+ **
+ **  // at the end of a step run/runOnePage method, add the accumulated audit details to step output
+ **  auditDetailAccumulator.getAccumulatedAuditSingleInputs().forEach(runBackendStepOutput::addAuditSingleInput);
+ **  auditDetailAccumulator.clear();
+ ** </pre>
+ *******************************************************************************/
+public class AuditDetailAccumulator implements Serializable
+{
+   private static final QLogger LOG = QLogger.getLogger(AuditDetailAccumulator.class);
+
+   private static final String objectKey = AuditDetailAccumulator.class.getSimpleName();
+
+   private String header;
+
+   private Map<TableNameAndPrimaryKey, AuditSingleInput> recordAuditInputMap = new HashMap<>();
+
+
+
+   /*******************************************************************************
+    ** Constructor
+    **
+    *******************************************************************************/
+   public AuditDetailAccumulator(String header)
+   {
+      this.header = header;
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public void setInContext()
+   {
+      QContext.setObject(objectKey, this);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public static Optional<AuditDetailAccumulator> getFromContext()
+   {
+      return QContext.getObject(objectKey, AuditDetailAccumulator.class);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public void addAuditDetail(String tableName, QRecordEntity entity, String message)
+   {
+      if(entity != null)
+      {
+         addAuditDetail(tableName, entity.toQRecord(), message);
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public void addAuditDetail(String tableName, QRecord record, String message)
+   {
+      QTableMetaData table      = QContext.getQInstance().getTable(tableName);
+      Serializable   primaryKey = record.getValue(table.getPrimaryKeyField());
+      if(primaryKey == null)
+      {
+         LOG.info("Missing primary key in input record - audit detail message will not be recorded.", logPair("message", message));
+         return;
+      }
+
+      AuditSingleInput auditSingleInput = recordAuditInputMap.computeIfAbsent(new TableNameAndPrimaryKey(tableName, primaryKey), (key) -> new AuditSingleInput(table, record, header));
+      auditSingleInput.addDetail(message);
+   }
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public Collection<AuditSingleInput> getAccumulatedAuditSingleInputs()
+   {
+      return (recordAuditInputMap.values());
+   }
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public void clear()
+   {
+      recordAuditInputMap.clear();
+   }
+
+
+   private record TableNameAndPrimaryKey(String tableName, Serializable primaryKey) {}
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/audits/AuditSingleInput.java

@@ -41,7 +41,7 @@ import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
 /*******************************************************************************
  ** Input data to insert a single audit record (with optional child record)..
  *******************************************************************************/
-public class AuditSingleInput
+public class AuditSingleInput implements Serializable
 {
    private String  auditTableName;
    private String  auditUserName;

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/replace/ReplaceInput.java

@@ -39,6 +39,7 @@ public class ReplaceInput extends AbstractTableActionInput
    private UniqueKey           key;
    private List<QRecord>       records;
    private QQueryFilter        filter;
+   private boolean             performDeletes = true;
 
    private boolean omitDmlAudit = false;
 
@@ -207,4 +208,35 @@ public class ReplaceInput extends AbstractTableActionInput
       return (this);
    }
 
+
+
+   /*******************************************************************************
+    ** Getter for performDeletes
+    *******************************************************************************/
+   public boolean getPerformDeletes()
+   {
+      return (this.performDeletes);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for performDeletes
+    *******************************************************************************/
+   public void setPerformDeletes(boolean performDeletes)
+   {
+      this.performDeletes = performDeletes;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for performDeletes
+    *******************************************************************************/
+   public ReplaceInput withPerformDeletes(boolean performDeletes)
+   {
+      this.performDeletes = performDeletes;
+      return (this);
+   }
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/update/UpdateInput.java

@@ -52,8 +52,9 @@ public class UpdateInput extends AbstractTableActionInput
    private Boolean areAllValuesBeingUpdatedTheSame = null;
 
    private boolean omitTriggeringAutomations = false;
-   private boolean omitDmlAudit = false;
-   private String  auditContext = null;
+   private boolean omitDmlAudit              = false;
+   private boolean omitModifyDateUpdate      = false;
+   private String  auditContext              = null;
 
 
 
@@ -353,4 +354,35 @@ public class UpdateInput extends AbstractTableActionInput
       return (this);
    }
 
+
+
+   /*******************************************************************************
+    ** Getter for omitModifyDateUpdate
+    *******************************************************************************/
+   public boolean getOmitModifyDateUpdate()
+   {
+      return (this.omitModifyDateUpdate);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for omitModifyDateUpdate
+    *******************************************************************************/
+   public void setOmitModifyDateUpdate(boolean omitModifyDateUpdate)
+   {
+      this.omitModifyDateUpdate = omitModifyDateUpdate;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for omitModifyDateUpdate
+    *******************************************************************************/
+   public UpdateInput withOmitModifyDateUpdate(boolean omitModifyDateUpdate)
+   {
+      this.omitModifyDateUpdate = omitModifyDateUpdate;
+      return (this);
+   }
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/dashboard/widgets/QWidgetData.java

@@ -34,6 +34,7 @@ import java.util.Map;
 public abstract class QWidgetData
 {
    private String       label;
+   private String       sublabel;
    private String       footerHTML;
    private List<String> dropdownNameList;
    private List<String> dropdownLabelList;
@@ -51,6 +52,7 @@ public abstract class QWidgetData
    private List<List<Serializable>> csvData;
 
 
+
    /*******************************************************************************
     ** Getter for type
     *******************************************************************************/
@@ -356,4 +358,35 @@ public abstract class QWidgetData
       return (this);
    }
 
+
+
+   /*******************************************************************************
+    ** Getter for sublabel
+    *******************************************************************************/
+   public String getSublabel()
+   {
+      return (this.sublabel);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for sublabel
+    *******************************************************************************/
+   public void setSublabel(String sublabel)
+   {
+      this.sublabel = sublabel;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for sublabel
+    *******************************************************************************/
+   public QWidgetData withSublabel(String sublabel)
+   {
+      this.sublabel = sublabel;
+      return (this);
+   }
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/dashboard/widgets/blocks/AbstractBlockWidgetData.java

@@ -40,6 +40,8 @@ public abstract class AbstractBlockWidgetData<
    S extends BlockSlotsInterface,
    SX extends BlockStylesInterface> extends QWidgetData
 {
+   private String blockId;
+
    private BlockTooltip tooltip;
    private BlockLink    link;
 
@@ -383,4 +385,35 @@ public abstract class AbstractBlockWidgetData<
       return (T) this;
    }
 
+
+   /*******************************************************************************
+    ** Getter for blockId
+    *******************************************************************************/
+   public String getBlockId()
+   {
+      return (this.blockId);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for blockId
+    *******************************************************************************/
+   public void setBlockId(String blockId)
+   {
+      this.blockId = blockId;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for blockId
+    *******************************************************************************/
+   public T withBlockId(String blockId)
+   {
+      this.blockId = blockId;
+      return (T) this;
+   }
+
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/DynamicDefaultValueBehavior.java

@@ -26,6 +26,7 @@ import java.io.Serializable;
 import java.time.Instant;
 import java.time.LocalDate;
 import java.util.List;
+import java.util.Set;
 import com.kingsrook.qqq.backend.core.actions.values.ValueBehaviorApplier;
 import com.kingsrook.qqq.backend.core.logging.QLogger;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
@@ -65,12 +66,16 @@ public enum DynamicDefaultValueBehavior implements FieldBehavior<DynamicDefaultV
     **
     *******************************************************************************/
    @Override
-   public void apply(ValueBehaviorApplier.Action action, List<QRecord> recordList, QInstance instance, QTableMetaData table, QFieldMetaData field)
+   public void apply(ValueBehaviorApplier.Action action, List<QRecord> recordList, QInstance instance, QTableMetaData table, QFieldMetaData field, Set<FieldBehavior<?>> behaviorsToOmit)
    {
       if(this.equals(NONE))
       {
          return;
       }
+      if(behaviorsToOmit != null && behaviorsToOmit.contains(this))
+      {
+         return;
+      }
 
       switch(this)
       {

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/FieldBehavior.java

@@ -23,6 +23,7 @@ package com.kingsrook.qqq.backend.core.model.metadata.fields;
 
 
 import java.util.List;
+import java.util.Set;
 import com.kingsrook.qqq.backend.core.actions.values.ValueBehaviorApplier;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
 import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
@@ -49,7 +50,7 @@ public interface FieldBehavior<T extends FieldBehavior<T>>
    /*******************************************************************************
     ** Apply this behavior to a list of records
     *******************************************************************************/
-   void apply(ValueBehaviorApplier.Action action, List<QRecord> recordList, QInstance instance, QTableMetaData table, QFieldMetaData field);
+   void apply(ValueBehaviorApplier.Action action, List<QRecord> recordList, QInstance instance, QTableMetaData table, QFieldMetaData field, Set<FieldBehavior<?>> behaviorsToOmit);
 
    /*******************************************************************************
     ** control if multiple behaviors of this type should be allowed together on a field.

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/ValueTooLongBehavior.java

@@ -23,6 +23,7 @@ package com.kingsrook.qqq.backend.core.model.metadata.fields;
 
 
 import java.util.List;
+import java.util.Set;
 import com.kingsrook.qqq.backend.core.actions.values.ValueBehaviorApplier;
 import com.kingsrook.qqq.backend.core.logging.QLogger;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
@@ -65,12 +66,16 @@ public enum ValueTooLongBehavior implements FieldBehavior<ValueTooLongBehavior>
     **
     *******************************************************************************/
    @Override
-   public void apply(ValueBehaviorApplier.Action action, List<QRecord> recordList, QInstance instance, QTableMetaData table, QFieldMetaData field)
+   public void apply(ValueBehaviorApplier.Action action, List<QRecord> recordList, QInstance instance, QTableMetaData table, QFieldMetaData field, Set<FieldBehavior<?>> behaviorsToOmit)
    {
       if(this.equals(PASS_THROUGH))
       {
          return;
       }
+      if(behaviorsToOmit != null && behaviorsToOmit.contains(this))
+      {
+         return;
+      }
 
       String fieldName = field.getName();
       if(!QFieldType.STRING.equals(field.getType()))

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/automation/HealBadRecordAutomationStatusesProcessStep.java

@@ -28,6 +28,7 @@ import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
+import java.util.function.Function;
 import com.kingsrook.qqq.backend.core.actions.automation.AutomationStatus;
 import com.kingsrook.qqq.backend.core.actions.processes.BackendStep;
 import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
@@ -101,6 +102,31 @@ public class HealBadRecordAutomationStatusesProcessStep implements BackendStep,
    @Override
    public QProcessMetaData produce(QInstance qInstance) throws QException
    {
+      Function<String, QFrontendStepMetaData> makeReviewOrResultStep = (String name) -> new QFrontendStepMetaData()
+         .withName(name)
+         .withComponent(new NoCodeWidgetFrontendComponentMetaData()
+            .withOutput(new WidgetHtmlLine()
+               .withCondition(new QFilterCriteria("warningCount", QCriteriaOperator.GREATER_THAN, 0))
+               .withWrapper(HtmlWrapper.divWithStyles(HtmlWrapper.STYLE_YELLOW))
+               .withVelocityTemplate("<b>Warning:</b>"))
+            .withOutput(new WidgetHtmlLine()
+               .withCondition(new QFilterCriteria("warningCount", QCriteriaOperator.GREATER_THAN, 0))
+               .withWrapper(HtmlWrapper.divWithStyles(HtmlWrapper.STYLE_INDENT_1))
+               .withWrapper(HtmlWrapper.divWithStyles(HtmlWrapper.STYLE_YELLOW))
+               .withVelocityTemplate("""
+                  <ul>
+                     #foreach($string in $warnings)
+                        <li>$string</li>
+                     #end
+                  </ul>
+                  """)))
+         .withComponent(new QFrontendComponentMetaData().withType(QComponentType.VIEW_FORM))
+         .withViewField(new QFieldMetaData("review".equals(name) ? "totalRecordsToUpdate" : "totalRecordsUpdated", QFieldType.INTEGER) /* todo - didn't display commas... .withDisplayFormat(DisplayFormat.COMMAS) */)
+         .withComponent(new QFrontendComponentMetaData().withType(QComponentType.RECORD_LIST))
+         .withRecordListField(new QFieldMetaData("tableName", QFieldType.STRING))
+         .withRecordListField(new QFieldMetaData("badStatus", QFieldType.STRING))
+         .withRecordListField(new QFieldMetaData("count", QFieldType.INTEGER).withDisplayFormat(DisplayFormat.COMMAS) /* todo - didn't display commas... */);
+
       QProcessMetaData processMetaData = new QProcessMetaData()
          .withName(NAME)
          .withStepList(List.of(
@@ -109,37 +135,14 @@ public class HealBadRecordAutomationStatusesProcessStep implements BackendStep,
                .withComponent(new QFrontendComponentMetaData().withType(QComponentType.EDIT_FORM))
                .withFormField(new QFieldMetaData("tableName", QFieldType.STRING).withPossibleValueSourceName(TablesPossibleValueSourceMetaDataProvider.NAME))
                .withFormField(new QFieldMetaData("minutesOldLimit", QFieldType.INTEGER).withDefaultValue(60)),
+            new QBackendStepMetaData()
+               .withName("preview")
+               .withCode(new QCodeReference(getClass())),
+            makeReviewOrResultStep.apply("review"),
             new QBackendStepMetaData()
                .withName("run")
                .withCode(new QCodeReference(getClass())),
-            new QFrontendStepMetaData()
-               .withName("output")
-
-               .withComponent(new NoCodeWidgetFrontendComponentMetaData()
-                  .withOutput(new WidgetHtmlLine()
-                     .withCondition(new QFilterCriteria("warningCount", QCriteriaOperator.GREATER_THAN, 0))
-                     .withWrapper(HtmlWrapper.divWithStyles(HtmlWrapper.STYLE_YELLOW))
-                     .withVelocityTemplate("<b>Warning:</b>"))
-                  .withOutput(new WidgetHtmlLine()
-                     .withCondition(new QFilterCriteria("warningCount", QCriteriaOperator.GREATER_THAN, 0))
-                     .withWrapper(HtmlWrapper.divWithStyles(HtmlWrapper.STYLE_INDENT_1))
-                     .withWrapper(HtmlWrapper.divWithStyles(HtmlWrapper.STYLE_YELLOW))
-                     .withVelocityTemplate("""
-                        <ul>
-                           #foreach($string in $warnings)
-                              <li>$string</li>
-                           #end
-                        </ul>
-                        """)))
-
-               .withComponent(new QFrontendComponentMetaData().withType(QComponentType.VIEW_FORM))
-               .withViewField(new QFieldMetaData("totalRecordsUpdated", QFieldType.INTEGER) /* todo - didn't display commas... .withDisplayFormat(DisplayFormat.COMMAS) */)
-
-               .withComponent(new QFrontendComponentMetaData().withType(QComponentType.RECORD_LIST))
-               .withRecordListField(new QFieldMetaData("tableName", QFieldType.STRING))
-               .withRecordListField(new QFieldMetaData("badStatus", QFieldType.STRING))
-               .withRecordListField(new QFieldMetaData("count", QFieldType.INTEGER).withDisplayFormat(DisplayFormat.COMMAS) /* todo - didn't display commas... */)
-
+            makeReviewOrResultStep.apply("result")
          ));
 
       return (processMetaData);
@@ -154,6 +157,7 @@ public class HealBadRecordAutomationStatusesProcessStep implements BackendStep,
    public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException
    {
       int recordsUpdated = 0;
+      boolean isReview = "preview".equals(runBackendStepInput.getStepName());
 
       ////////////////////////////////////////////////////////////////////////
       // if a table name is given, validate it, and run for just that table //
@@ -167,7 +171,7 @@ public class HealBadRecordAutomationStatusesProcessStep implements BackendStep,
             throw (new QException("Unrecognized table name: " + tableName));
          }
 
-         recordsUpdated += processTable(tableName, runBackendStepInput, runBackendStepOutput, warnings);
+         recordsUpdated += processTable(isReview, tableName, runBackendStepInput, runBackendStepOutput, warnings);
       }
       else
       {
@@ -176,11 +180,12 @@ public class HealBadRecordAutomationStatusesProcessStep implements BackendStep,
          //////////////////////////////////////////////////////////////////////////
          for(QTableMetaData table : QContext.getQInstance().getTables().values())
          {
-            recordsUpdated += processTable(table.getName(), runBackendStepInput, runBackendStepOutput, warnings);
+            recordsUpdated += processTable(isReview, table.getName(), runBackendStepInput, runBackendStepOutput, warnings);
          }
       }
 
       runBackendStepOutput.addValue("totalRecordsUpdated", recordsUpdated);
+      runBackendStepOutput.addValue("totalRecordsToUpdate", recordsUpdated);
       runBackendStepOutput.addValue("warnings", warnings);
       runBackendStepOutput.addValue("warningCount", warnings.size());
 
@@ -198,7 +203,7 @@ public class HealBadRecordAutomationStatusesProcessStep implements BackendStep,
    /*******************************************************************************
     **
     *******************************************************************************/
-   private int processTable(String tableName, RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput, List<String> warnings)
+   private int processTable(boolean isReview, String tableName, RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput, List<String> warnings)
    {
       try
       {
@@ -216,34 +221,42 @@ public class HealBadRecordAutomationStatusesProcessStep implements BackendStep,
             // find the modify-date field on the table //
             /////////////////////////////////////////////
             String modifyDateFieldName = null;
+            String createDateFieldName = null;
             for(QFieldMetaData field : table.getFields().values())
             {
                if(DynamicDefaultValueBehavior.MODIFY_DATE.equals(field.getBehaviorOnlyIfSet(DynamicDefaultValueBehavior.class)))
                {
                   modifyDateFieldName = field.getName();
-                  break;
+               }
+               if(DynamicDefaultValueBehavior.CREATE_DATE.equals(field.getBehaviorOnlyIfSet(DynamicDefaultValueBehavior.class)))
+               {
+                  createDateFieldName = field.getName();
                }
             }
 
-            if(modifyDateFieldName == null)
+            //////////////////////////////////////////////////////////////////////////////////////////////////
+            // set up a filter to query for records either FAILED, or RUNNING w/ create/modify date too old //
+            //////////////////////////////////////////////////////////////////////////////////////////////////
+            QQueryFilter filter = new QQueryFilter().withBooleanOperator(QQueryFilter.BooleanOperator.OR);
+            filter.addSubFilter(new QQueryFilter().withCriteria(new QFilterCriteria(automationStatusFieldName, QCriteriaOperator.IN, AutomationStatus.FAILED_INSERT_AUTOMATIONS.getId(), AutomationStatus.FAILED_UPDATE_AUTOMATIONS.getId())));
+
+            if(modifyDateFieldName != null)
             {
-               warnings.add("Could not find a Modify Date field on table: " + tableName);
-               LOG.info("Couldn't find a MODIFY_DATE field on table", logPair("tableName", tableName));
-               return 0;
+               filter.addSubFilter(new QQueryFilter()
+                  .withCriteria(new QFilterCriteria(automationStatusFieldName, QCriteriaOperator.EQUALS, AutomationStatus.RUNNING_UPDATE_AUTOMATIONS.getId()))
+                  .withCriteria(new QFilterCriteria(modifyDateFieldName, QCriteriaOperator.LESS_THAN, NowWithOffset.minus(minutesOldLimit, ChronoUnit.MINUTES))));
+            }
+
+            if(createDateFieldName != null)
+            {
+               filter.addSubFilter(new QQueryFilter()
+                  .withCriteria(new QFilterCriteria(automationStatusFieldName, QCriteriaOperator.EQUALS, AutomationStatus.RUNNING_INSERT_AUTOMATIONS.getId()))
+                  .withCriteria(new QFilterCriteria(createDateFieldName, QCriteriaOperator.LESS_THAN, NowWithOffset.minus(minutesOldLimit, ChronoUnit.MINUTES))));
             }
 
-            ////////////////////////////////////////////////////////////////////////
-            // query for records either FAILED, or RUNNING w/ modify date too old //
-            ////////////////////////////////////////////////////////////////////////
             QueryInput queryInput = new QueryInput();
             queryInput.setTableName(tableName);
-            queryInput.setFilter(new QQueryFilter().withBooleanOperator(QQueryFilter.BooleanOperator.OR)
-               .withSubFilter(new QQueryFilter()
-                  .withCriteria(new QFilterCriteria(automationStatusFieldName, QCriteriaOperator.IN, AutomationStatus.FAILED_INSERT_AUTOMATIONS.getId(), AutomationStatus.FAILED_UPDATE_AUTOMATIONS.getId())))
-               .withSubFilter(new QQueryFilter()
-                  .withCriteria(new QFilterCriteria(automationStatusFieldName, QCriteriaOperator.IN, AutomationStatus.RUNNING_INSERT_AUTOMATIONS.getId(), AutomationStatus.RUNNING_UPDATE_AUTOMATIONS.getId()))
-                  .withCriteria(new QFilterCriteria(modifyDateFieldName, QCriteriaOperator.LESS_THAN, NowWithOffset.minus(minutesOldLimit, ChronoUnit.MINUTES))))
-            );
+            queryInput.setFilter(filter);
             QueryOutput queryOutput = new QueryAction().execute(queryInput);
 
             //////////////////////////////////////////////////////////////////////////////////////////////////////////////////
@@ -269,18 +282,27 @@ public class HealBadRecordAutomationStatusesProcessStep implements BackendStep,
                }
             }
 
-            if(!recordsToUpdate.isEmpty())
+            //////////////////////////////////////////////////////////////////////////////////////
+            // if there are record to update (and this isn't the review step), then update them //
+            //////////////////////////////////////////////////////////////////////////////////////
+            if(!recordsToUpdate.isEmpty() && !isReview)
             {
-               LOG.info("Healing bad record automation statuses", logPair("tableName", tableName), logPair("count", recordsToUpdate.size()));
+               LOG.info("Healing bad record automation statuses", logPair("tableName", tableName), logPair("countByStatus", countByStatus));
                new UpdateAction().execute(new UpdateInput(tableName).withRecords(recordsToUpdate).withOmitTriggeringAutomations(true));
             }
 
-            for(Map.Entry<String, Integer> entry : countByStatus.entrySet())
+            ///////////////////////////////////////////////////
+            // on the review step, add records to the output //
+            ///////////////////////////////////////////////////
+            if(isReview)
             {
-               runBackendStepOutput.addRecord(new QRecord()
-                  .withValue("tableName", tableName)
-                  .withValue("badStatus", entry.getKey())
-                  .withValue("count", entry.getValue()));
+               for(Map.Entry<String, Integer> entry : countByStatus.entrySet())
+               {
+                  runBackendStepOutput.addRecord(new QRecord()
+                     .withValue("tableName", QContext.getQInstance().getTable(tableName).getLabel())
+                     .withValue("badStatus", entry.getKey())
+                     .withValue("count", entry.getValue()));
+               }
             }
 
             return (recordsToUpdate.size());

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/basepull/BasepullExtractStepInterface.java

@@ -0,0 +1,30 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.processes.implementations.basepull;
+
+
+/*******************************************************************************
+ ** marker - used to indicate that a step is meant to be used  for basepull extracts.
+ *******************************************************************************/
+public interface BasepullExtractStepInterface
+{
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/basepull/ExtractViaBasepullQueryStep.java

@@ -38,7 +38,7 @@ import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwith
 /*******************************************************************************
  ** Version of ExtractViaQueryStep that knows how to set up a basepull query.
  *******************************************************************************/
-public class ExtractViaBasepullQueryStep extends ExtractViaQueryStep
+public class ExtractViaBasepullQueryStep extends ExtractViaQueryStep implements BasepullExtractStepInterface
 {
 
    /*******************************************************************************

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/ExtractViaQueryStep.java

@@ -113,6 +113,10 @@ public class ExtractViaQueryStep extends AbstractExtractStep
       {
          queryInput.setShouldFetchHeavyFields(true);
       }
+      if(runBackendStepInput.getValuePrimitiveBoolean(StreamedETLWithFrontendProcess.FIELD_INCLUDE_ASSOCIATIONS))
+      {
+         queryInput.setIncludeAssociations(true);
+      }
 
       customizeInputPreQuery(queryInput);
 

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/LoadViaUpdateStep.java

@@ -32,6 +32,7 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.InputSource;
 import com.kingsrook.qqq.backend.core.model.actions.tables.QInputSource;
 import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateOutput;
+import org.apache.commons.lang.BooleanUtils;
 
 
 /*******************************************************************************
@@ -40,7 +41,9 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateOutput;
  *******************************************************************************/
 public class LoadViaUpdateStep extends AbstractLoadStep
 {
-   public static final String FIELD_DESTINATION_TABLE = "destinationTable";
+   public static final String FIELD_DESTINATION_TABLE               = "destinationTable";
+   public static final String DO_NOT_UPDATE_MODIFY_DATE_FIELD_NAME  = "doNotUpdateModifyDateFieldName";
+   public static final String DO_NOT_TRIGGER_AUTOMATIONS_FIELD_NAME = "doNotTriggerAutomationsFieldName";
 
 
 
@@ -67,6 +70,15 @@ public class LoadViaUpdateStep extends AbstractLoadStep
       updateInput.setRecords(runBackendStepInput.getRecords());
       getTransaction().ifPresent(updateInput::setTransaction);
       updateInput.setAsyncJobCallback(runBackendStepInput.getAsyncJobCallback());
+
+      //////////////////////////////////////////////////////////////////////////////////////////
+      // look for flags in the input to either not update modify dates or not run automations //
+      //////////////////////////////////////////////////////////////////////////////////////////
+      boolean doNotUpdateModifyDate   = BooleanUtils.isTrue(runBackendStepInput.getValueBoolean(LoadViaUpdateStep.DO_NOT_UPDATE_MODIFY_DATE_FIELD_NAME));
+      boolean doNotTriggerAutomations = BooleanUtils.isTrue(runBackendStepInput.getValueBoolean(LoadViaUpdateStep.DO_NOT_TRIGGER_AUTOMATIONS_FIELD_NAME));
+      updateInput.setOmitModifyDateUpdate(doNotUpdateModifyDate);
+      updateInput.setOmitTriggeringAutomations(doNotTriggerAutomations);
+
       UpdateOutput updateOutput = new UpdateAction().execute(updateInput);
       runBackendStepOutput.getRecords().addAll(updateOutput.getRecords());
    }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLWithFrontendProcess.java

@@ -83,6 +83,7 @@ public class StreamedETLWithFrontendProcess
    public static final String FIELD_RECORD_COUNT         = "recordCount"; // Integer
    public static final String FIELD_DEFAULT_QUERY_FILTER = "defaultQueryFilter"; // QQueryFilter or String (json, of q QQueryFilter)
    public static final String FIELD_FETCH_HEAVY_FIELDS   = "fetchHeavyFields"; // Boolean
+   public static final String FIELD_INCLUDE_ASSOCIATIONS = "includeAssociations"; // Boolean
 
    public static final String FIELD_SUPPORTS_FULL_VALIDATION = "supportsFullValidation"; // Boolean
    public static final String FIELD_DO_FULL_VALIDATION       = "doFullValidation"; // Boolean
@@ -144,6 +145,7 @@ public class StreamedETLWithFrontendProcess
          .withCode(new QCodeReference(StreamedETLPreviewStep.class))
          .withInputData(new QFunctionInputMetaData()
             .withField(new QFieldMetaData(FIELD_SOURCE_TABLE, QFieldType.STRING).withDefaultValue(defaultFieldValues.get(FIELD_SOURCE_TABLE)))
+            .withField(new QFieldMetaData(FIELD_INCLUDE_ASSOCIATIONS, QFieldType.BOOLEAN).withDefaultValue(defaultFieldValues.getOrDefault(FIELD_INCLUDE_ASSOCIATIONS, false)))
             .withField(new QFieldMetaData(FIELD_FETCH_HEAVY_FIELDS, QFieldType.BOOLEAN).withDefaultValue(defaultFieldValues.getOrDefault(FIELD_FETCH_HEAVY_FIELDS, false)))
             .withField(new QFieldMetaData(FIELD_DESTINATION_TABLE, QFieldType.STRING).withDefaultValue(defaultFieldValues.get(FIELD_DESTINATION_TABLE)))
             .withField(new QFieldMetaData(FIELD_SUPPORTS_FULL_VALIDATION, QFieldType.BOOLEAN).withDefaultValue(defaultFieldValues.getOrDefault(FIELD_SUPPORTS_FULL_VALIDATION, true)))

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/utils/GeneralProcessUtils.java

@@ -292,9 +292,26 @@ public class GeneralProcessUtils
     ** too many rows...  Caveat emptor.
     *******************************************************************************/
    public static <T extends QRecordEntity> List<T> loadTable(String tableName, Class<T> entityClass) throws QException
+   {
+      return (loadTable(tableName, entityClass, null));
+   }
+
+
+
+   /*******************************************************************************
+    ** Load all rows from a table as a RecordEntity, takes in a filter as well
+    **
+    ** Note, this is inherently unsafe, if you were to call it on a table with
+    ** too many rows...  Caveat emptor.
+    *******************************************************************************/
+   public static <T extends QRecordEntity> List<T> loadTable(String tableName, Class<T> entityClass, QQueryFilter filter) throws QException
    {
       QueryInput queryInput = new QueryInput();
       queryInput.setTableName(tableName);
+      if(filter != null)
+      {
+         queryInput.setFilter(filter);
+      }
       QueryOutput queryOutput = new QueryAction().execute(queryInput);
 
       List<T> rs = new ArrayList<>();

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/tables/UpdateActionTest.java

@@ -23,6 +23,7 @@ package com.kingsrook.qqq.backend.core.actions.tables;
 
 
 import java.math.BigDecimal;
+import java.time.Instant;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Objects;
@@ -787,4 +788,59 @@ class UpdateActionTest extends BaseTest
       }
    }
 
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testUpdateDoNotUpdateModifyDate() throws QException
+   {
+      QContext.getQSession().withSecurityKeyValues(MapBuilder.of(TestUtils.SECURITY_KEY_TYPE_STORE_ALL_ACCESS, ListBuilder.of(true)));
+
+      ////////////////////////////////////////////////////
+      // create a test order and capture its modifyDate //
+      ////////////////////////////////////////////////////
+      InsertInput insertInput = new InsertInput();
+      insertInput.setTableName(TestUtils.TABLE_NAME_ORDER);
+      insertInput.setRecords(List.of(new QRecord()));
+      QRecord qRecord = new InsertAction().execute(insertInput).getRecords().get(0);
+
+      Instant initialModifyDate = qRecord.getValueInstant("modifyDate");
+      assertNotNull(initialModifyDate);
+
+      ///////////////////////////////////////////////////////////////////
+      // update the order, the modify date should be in the future now //
+      ///////////////////////////////////////////////////////////////////
+      {
+         UpdateInput updateInput = new UpdateInput();
+         updateInput.setTableName(TestUtils.TABLE_NAME_ORDER);
+         updateInput.setRecords(List.of(qRecord));
+         QRecord updatedRecord = new UpdateAction().execute(updateInput).getRecords().get(0);
+
+         Instant newModifyDate = updatedRecord.getValueInstant("modifyDate");
+         assertNotNull(initialModifyDate);
+         assertThat(initialModifyDate).isBefore(newModifyDate);
+
+         /////////////////////////////////////////////////////////////////////////////
+         // set the initial modify date to this modify date for the next test below //
+         /////////////////////////////////////////////////////////////////////////////
+         initialModifyDate = newModifyDate;
+      }
+
+      ////////////////////////////////////////////////////////////////////////////////////////////////
+      // now do an update setting flag to not update the modify date, then compare, should be equal //
+      ////////////////////////////////////////////////////////////////////////////////////////////////
+      {
+         UpdateInput updateInput = new UpdateInput();
+         updateInput.setTableName(TestUtils.TABLE_NAME_ORDER);
+         updateInput.setRecords(List.of(qRecord));
+         updateInput.setOmitModifyDateUpdate(true);
+         QRecord updatedRecord = new UpdateAction().execute(updateInput).getRecords().get(0);
+
+         Instant newModifyDate = updatedRecord.getValueInstant("modifyDate");
+         assertNotNull(initialModifyDate);
+         assertThat(initialModifyDate).isEqualTo(newModifyDate);
+      }
+   }
 }

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/values/ValueBehaviorApplierTest.java

@@ -24,10 +24,12 @@ package com.kingsrook.qqq.backend.core.actions.values;
 
 import java.util.List;
 import java.util.Optional;
+import java.util.Set;
 import com.kingsrook.qqq.backend.core.BaseTest;
 import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
 import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.FieldBehavior;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.ValueTooLongBehavior;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 import com.kingsrook.qqq.backend.core.utils.TestUtils;
@@ -35,6 +37,7 @@ import org.junit.jupiter.api.Test;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertFalse;
 import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
 import static org.junit.jupiter.api.Assertions.fail;
 
 
@@ -63,7 +66,7 @@ class ValueBehaviorApplierTest extends BaseTest
          new QRecord().withValue("id", 2).withValue("firstName", "John").withValue("lastName", "Last name too long").withValue("email", "john@smith.com"),
          new QRecord().withValue("id", 3).withValue("firstName", "First name too long").withValue("lastName", "Smith").withValue("email", "john.smith@emaildomainwayytolongtofit.com")
       );
-      ValueBehaviorApplier.applyFieldBehaviors(ValueBehaviorApplier.Action.INSERT, qInstance, table, recordList);
+      ValueBehaviorApplier.applyFieldBehaviors(ValueBehaviorApplier.Action.INSERT, qInstance, table, recordList, null);
 
       assertEquals("First name", getRecordById(recordList, 1).getValueString("firstName"));
       assertEquals("Last na...", getRecordById(recordList, 2).getValueString("lastName"));
@@ -73,6 +76,38 @@ class ValueBehaviorApplierTest extends BaseTest
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testOmitBehaviors()
+   {
+      QInstance      qInstance = QContext.getQInstance();
+      QTableMetaData table     = qInstance.getTable(TestUtils.TABLE_NAME_PERSON_MEMORY);
+      table.getField("firstName").withMaxLength(10).withBehavior(ValueTooLongBehavior.TRUNCATE);
+      table.getField("lastName").withMaxLength(10).withBehavior(ValueTooLongBehavior.TRUNCATE_ELLIPSIS);
+      table.getField("email").withMaxLength(20).withBehavior(ValueTooLongBehavior.ERROR);
+
+      List<QRecord> recordList = List.of(
+         new QRecord().withValue("id", 1).withValue("firstName", "First name too long").withValue("lastName", "Smith").withValue("email", "john@smith.com"),
+         new QRecord().withValue("id", 2).withValue("firstName", "John").withValue("lastName", "Last name too long").withValue("email", "john@smith.com"),
+         new QRecord().withValue("id", 3).withValue("firstName", "First name too long").withValue("lastName", "Smith").withValue("email", "john.smith@emaildomainwayytolongtofit.com")
+      );
+
+      Set<FieldBehavior<?>> behaviorsToOmit = Set.of(ValueTooLongBehavior.ERROR);
+      ValueBehaviorApplier.applyFieldBehaviors(ValueBehaviorApplier.Action.INSERT, qInstance, table, recordList, behaviorsToOmit);
+
+      ///////////////////////////////////////////////////////////////////////////////////////////
+      // the third error behavior was set to be omitted, so no errors should be on that record //
+      ///////////////////////////////////////////////////////////////////////////////////////////
+      assertEquals("First name", getRecordById(recordList, 1).getValueString("firstName"));
+      assertEquals("Last na...", getRecordById(recordList, 2).getValueString("lastName"));
+      assertEquals("john.smith@emaildomainwayytolongtofit.com", getRecordById(recordList, 3).getValueString("email"));
+      assertTrue(getRecordById(recordList, 3).getErrors().isEmpty());
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/
@@ -95,7 +130,7 @@ class ValueBehaviorApplierTest extends BaseTest
          new QRecord().withValue("id", 1).withValue("firstName", "First name too long").withValue("lastName", null).withValue("email", "john@smith.com"),
          new QRecord().withValue("id", 2).withValue("firstName", "").withValue("lastName", "Last name too long").withValue("email", "john@smith.com")
       );
-      ValueBehaviorApplier.applyFieldBehaviors(ValueBehaviorApplier.Action.INSERT, qInstance, table, recordList);
+      ValueBehaviorApplier.applyFieldBehaviors(ValueBehaviorApplier.Action.INSERT, qInstance, table, recordList, null);
 
       assertEquals("First name too long", getRecordById(recordList, 1).getValueString("firstName"));
       assertNull(getRecordById(recordList, 1).getValueString("lastName"));
@@ -118,4 +153,4 @@ class ValueBehaviorApplierTest extends BaseTest
       return (recordOpt.get());
    }
 
-}
+}

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidatorTest.java

@@ -1,6 +1,6 @@
 /*
  * QQQ - Low-code Application Framework for Engineers.
- * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
  * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
  * contact@kingsrook.com
  * https://github.com/Kingsrook/
@@ -38,6 +38,7 @@ import com.kingsrook.qqq.backend.core.actions.processes.BackendStep;
 import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.exceptions.QInstanceValidationException;
+import com.kingsrook.qqq.backend.core.instances.validation.plugins.AlwaysFailsProcessValidatorPlugin;
 import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLineInterface;
 import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
 import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
@@ -97,7 +98,7 @@ import static org.junit.jupiter.api.Assertions.fail;
  ** Unit test for QInstanceValidator.
  **
  *******************************************************************************/
-class QInstanceValidatorTest extends BaseTest
+public class QInstanceValidatorTest extends BaseTest
 {
 
    /*******************************************************************************
@@ -367,6 +368,35 @@ class QInstanceValidatorTest extends BaseTest
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void test_validatorPlugins()
+   {
+      try
+      {
+         QInstanceValidator.addValidatorPlugin(new AlwaysFailsProcessValidatorPlugin());
+
+         ////////////////////////////////////////
+         // make sure our always failer fails. //
+         ////////////////////////////////////////
+         assertValidationFailureReasonsAllowingExtraReasons((qInstance) -> {
+         }, "always fail");
+      }
+      finally
+      {
+         QInstanceValidator.removeAllValidatorPlugins();
+         
+         ////////////////////////////////////////////////////
+         // make sure if remove all plugins, we don't fail //
+         ////////////////////////////////////////////////////
+         assertValidationSuccess((qInstance) -> {});
+      }
+   }
+
+
+
    /*******************************************************************************
     ** Test that a table with no fields fails.
     **
@@ -1949,9 +1979,9 @@ class QInstanceValidatorTest extends BaseTest
     ** failed validation with reasons that match the supplied vararg-reasons (but allow
     ** more reasons - e.g., helpful when one thing we're testing causes other errors).
     *******************************************************************************/
-   private void assertValidationFailureReasonsAllowingExtraReasons(Consumer<QInstance> setup, String... reasons)
+   public static void assertValidationFailureReasonsAllowingExtraReasons(Consumer<QInstance> setup, String... expectedReasons)
    {
-      assertValidationFailureReasons(setup, true, reasons);
+      assertValidationFailureReasons(setup, true, expectedReasons);
    }
 
 
@@ -1961,9 +1991,9 @@ class QInstanceValidatorTest extends BaseTest
     ** failed validation with reasons that match the supplied vararg-reasons (and
     ** require that exact # of reasons).
     *******************************************************************************/
-   private void assertValidationFailureReasons(Consumer<QInstance> setup, String... reasons)
+   public static void assertValidationFailureReasons(Consumer<QInstance> setup, String... expectedReasons)
    {
-      assertValidationFailureReasons(setup, false, reasons);
+      assertValidationFailureReasons(setup, false, expectedReasons);
    }
 
 
@@ -1971,7 +2001,7 @@ class QInstanceValidatorTest extends BaseTest
    /*******************************************************************************
     ** Implementation for the overloads of this name.
     *******************************************************************************/
-   private void assertValidationFailureReasons(Consumer<QInstance> setup, boolean allowExtraReasons, String... reasons)
+   public static void assertValidationFailureReasons(Consumer<QInstance> setup, boolean allowExtraReasons, String... expectedReasons)
    {
       try
       {
@@ -1982,17 +2012,27 @@ class QInstanceValidatorTest extends BaseTest
       }
       catch(QInstanceValidationException e)
       {
-         if(!allowExtraReasons)
-         {
-            int noOfReasons = e.getReasons() == null ? 0 : e.getReasons().size();
-            assertEquals(reasons.length, noOfReasons, "Expected number of validation failure reasons.\nExpected reasons: " + String.join(",", reasons)
-               + "\nActual reasons: " + (noOfReasons > 0 ? String.join("\n", e.getReasons()) : "--"));
-         }
+         assertValidationFailureReasons(allowExtraReasons, e.getReasons(), expectedReasons);
+      }
+   }
 
-         for(String reason : reasons)
-         {
-            assertReason(reason, e);
-         }
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public static void assertValidationFailureReasons(boolean allowExtraReasons, List<String> actualReasons, String... expectedReasons)
+   {
+      if(!allowExtraReasons)
+      {
+         int noOfReasons = actualReasons == null ? 0 : actualReasons.size();
+         assertEquals(expectedReasons.length, noOfReasons, "Expected number of validation failure reasons.\nExpected reasons: " + String.join(",", expectedReasons)
+            + "\nActual reasons: " + (noOfReasons > 0 ? String.join("\n", actualReasons) : "--"));
+      }
+
+      for(String reason : expectedReasons)
+      {
+         assertReason(reason, actualReasons);
       }
    }
 
@@ -2022,11 +2062,11 @@ class QInstanceValidatorTest extends BaseTest
     ** the list of reasons in the QInstanceValidationException.
     **
     *******************************************************************************/
-   private void assertReason(String reason, QInstanceValidationException e)
+   public static void assertReason(String reason, List<String> actualReasons)
    {
-      assertNotNull(e.getReasons(), "Expected there to be a reason for the failure (but there was not)");
-      assertThat(e.getReasons())
-         .withFailMessage("Expected any of:\n%s\nTo match: [%s]", e.getReasons(), reason)
+      assertNotNull(actualReasons, "Expected there to be a reason for the failure (but there was not)");
+      assertThat(actualReasons)
+         .withFailMessage("Expected any of:\n%s\nTo match: [%s]", actualReasons, reason)
          .anyMatch(s -> s.contains(reason));
    }
 

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/validation/plugins/AlwaysFailsProcessValidatorPlugin.java

@@ -0,0 +1,45 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.instances.validation.plugins;
+
+
+import com.kingsrook.qqq.backend.core.instances.QInstanceValidator;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
+
+
+/*******************************************************************************
+ ** test instance of a validator plugin, that always adds an error
+ *******************************************************************************/
+public class AlwaysFailsProcessValidatorPlugin implements QInstanceValidatorPluginInterface<QProcessMetaData>
+{
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public void validate(QProcessMetaData object, QInstance qInstance, QInstanceValidator qInstanceValidator)
+   {
+      qInstanceValidator.getErrors().add("I always fail.");
+   }
+
+}

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/validation/plugins/BasepullExtractStepValidatorTest.java

@@ -0,0 +1,109 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.instances.validation.plugins;
+
+
+import com.kingsrook.qqq.backend.core.BaseTest;
+import com.kingsrook.qqq.backend.core.context.QContext;
+import com.kingsrook.qqq.backend.core.instances.QInstanceValidator;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.processes.implementations.basepull.BasepullConfiguration;
+import com.kingsrook.qqq.backend.core.processes.implementations.basepull.ExtractViaBasepullQueryStep;
+import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.ExtractViaQueryStep;
+import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.LoadViaInsertStep;
+import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess;
+import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcessTest;
+import com.kingsrook.qqq.backend.core.utils.TestUtils;
+import org.junit.jupiter.api.Test;
+import static com.kingsrook.qqq.backend.core.instances.QInstanceValidatorTest.assertValidationFailureReasons;
+import static org.assertj.core.api.Assertions.assertThat;
+
+
+/*******************************************************************************
+ ** Unit test for BasepullExtractStepValidator 
+ *******************************************************************************/
+class BasepullExtractStepValidatorTest extends BaseTest
+{
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testNoExtractStepAtAllFails()
+   {
+      QInstance          qInstance = QContext.getQInstance();
+      QInstanceValidator validator = new QInstanceValidator();
+
+      ///////////////////////////////////////////////////////////////////////////////////////////////////////////
+      // turns out, our "basepullTestProcess" doesn't have an extract step, so that makes this condition fire. //
+      ///////////////////////////////////////////////////////////////////////////////////////////////////////////
+      new BasepullExtractStepValidator().validate(qInstance.getProcess(TestUtils.PROCESS_NAME_BASEPULL), qInstance, validator);
+      assertValidationFailureReasons(false, validator.getErrors(), "does not have a field with a default value that is a BasepullExtractStepInterface CodeReference");
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testExtractViaQueryNotBasePull()
+   {
+      QInstance          qInstance = QContext.getQInstance();
+      QInstanceValidator validator = new QInstanceValidator();
+
+      ////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      // set up a streamed-etl process, but with an ExtractViaQueryStep instead of a basepull - it should fail! //
+      ////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      new BasepullExtractStepValidator().validate(StreamedETLWithFrontendProcess.defineProcessMetaData(
+         TestUtils.TABLE_NAME_SHAPE,
+         TestUtils.TABLE_NAME_PERSON_MEMORY,
+         ExtractViaQueryStep.class,
+         StreamedETLWithFrontendProcessTest.TestTransformShapeToPersonStep.class,
+         LoadViaInsertStep.class).withBasepullConfiguration(new BasepullConfiguration()), qInstance, validator);
+      assertValidationFailureReasons(false, validator.getErrors(), "does not have a field with a default value that is a BasepullExtractStepInterface CodeReference");
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testExtractViaBasepullQueryPasses()
+   {
+      QInstance          qInstance = QContext.getQInstance();
+      QInstanceValidator validator = new QInstanceValidator();
+
+      //////////////////////////////////////////////////////////////////////////////////////////////////
+      // set up a streamed-etl process, with an ExtractViaBasepullQueryStep as expected - should pass //
+      //////////////////////////////////////////////////////////////////////////////////////////////////
+      new BasepullExtractStepValidator().validate(StreamedETLWithFrontendProcess.defineProcessMetaData(
+         TestUtils.TABLE_NAME_SHAPE,
+         TestUtils.TABLE_NAME_PERSON_MEMORY,
+         ExtractViaBasepullQueryStep.class,
+         StreamedETLWithFrontendProcessTest.TestTransformShapeToPersonStep.class,
+         LoadViaInsertStep.class).withBasepullConfiguration(new BasepullConfiguration()), qInstance, validator);
+      assertThat(validator.getErrors()).isNullOrEmpty();
+   }
+
+}

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/logging/QCollectingLoggerTest.java

@@ -0,0 +1,92 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.logging;
+
+
+import com.kingsrook.qqq.backend.core.BaseTest;
+import org.apache.logging.log4j.Level;
+import org.junit.jupiter.api.Test;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+
+/*******************************************************************************
+ ** Unit test for QCollectingLogger 
+ *******************************************************************************/
+class QCollectingLoggerTest extends BaseTest
+{
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void test()
+   {
+      ClassThatLogsThings classThatLogsThings = new ClassThatLogsThings();
+      classThatLogsThings.logAnInfo("1");
+
+      QCollectingLogger collectingLogger = QLogger.activateCollectingLoggerForClass(ClassThatLogsThings.class);
+      classThatLogsThings.logAnInfo("2");
+      classThatLogsThings.logAWarn("3");
+      QLogger.deactivateCollectingLoggerForClass(ClassThatLogsThings.class);
+
+      classThatLogsThings.logAWarn("4");
+
+      assertEquals(2, collectingLogger.getCollectedMessages().size());
+
+      assertThat(collectingLogger.getCollectedMessages().get(0).getMessage()).contains("""
+         "message":"2",""");
+      assertEquals("2", collectingLogger.getCollectedMessages().get(0).getMessageAsJSONObject().getString("message"));
+      assertEquals(Level.INFO, collectingLogger.getCollectedMessages().get(0).getLevel());
+
+      assertThat(collectingLogger.getCollectedMessages().get(1).getMessage()).contains("""
+         "message":"3",""");
+      assertEquals(Level.WARN, collectingLogger.getCollectedMessages().get(1).getLevel());
+      assertEquals("3", collectingLogger.getCollectedMessages().get(1).getMessageAsJSONObject().getString("message"));
+   }
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public static class ClassThatLogsThings
+   {
+      private static final QLogger LOG = QLogger.getLogger(ClassThatLogsThings.class);
+
+      /*******************************************************************************
+       **
+       *******************************************************************************/
+      private void logAnInfo(String message)
+      {
+         LOG.info(message);
+      }
+
+      /*******************************************************************************
+       **
+       *******************************************************************************/
+      private void logAWarn(String message)
+      {
+         LOG.warn(message);
+      }
+   }
+
+}

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/actions/audits/AuditDetailAccumulatorTest.java

@@ -0,0 +1,81 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.model.actions.audits;
+
+
+import java.util.Collection;
+import com.kingsrook.qqq.backend.core.BaseTest;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.utils.TestUtils;
+import org.junit.jupiter.api.Test;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+
+/*******************************************************************************
+ ** Unit test for AuditDetailAccumulator 
+ *******************************************************************************/
+class AuditDetailAccumulatorTest extends BaseTest
+{
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void test()
+   {
+      AuditDetailAccumulator auditDetailAccumulator = new AuditDetailAccumulator("During test");
+      auditDetailAccumulator.addAuditDetail(TestUtils.TABLE_NAME_PERSON, new QRecord().withValue("id", 1701), "Something happened");
+      auditDetailAccumulator.addAuditDetail(TestUtils.TABLE_NAME_PERSON, new QRecord().withValue("id", 1701), "Something else happened");
+      auditDetailAccumulator.addAuditDetail(TestUtils.TABLE_NAME_PERSON, new QRecord().withValue("id", 74256), "Something happened here too");
+      auditDetailAccumulator.addAuditDetail(TestUtils.TABLE_NAME_ORDER, new QRecord().withValue("id", 74256), "Something happened to an order");
+
+      Collection<AuditSingleInput> auditSingleInputs = auditDetailAccumulator.getAccumulatedAuditSingleInputs();
+      assertEquals(3, auditSingleInputs.size());
+      assertThat(auditSingleInputs).anyMatch(asi -> asi.getAuditTableName().equals(TestUtils.TABLE_NAME_PERSON) && asi.getRecordId().equals(1701) && asi.getDetails().size() == 2);
+      assertThat(auditSingleInputs).anyMatch(asi -> asi.getAuditTableName().equals(TestUtils.TABLE_NAME_PERSON) && asi.getRecordId().equals(74256) && asi.getDetails().size() == 1);
+      assertThat(auditSingleInputs).anyMatch(asi -> asi.getAuditTableName().equals(TestUtils.TABLE_NAME_ORDER) && asi.getRecordId().equals(74256) && asi.getDetails().size() == 1);
+
+      auditDetailAccumulator.clear();;
+      auditSingleInputs = auditDetailAccumulator.getAccumulatedAuditSingleInputs();
+      assertEquals(0, auditSingleInputs.size());
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testContext()
+   {
+      AuditDetailAccumulator auditDetailAccumulator = new AuditDetailAccumulator("During test");
+      auditDetailAccumulator.setInContext();
+
+      AuditDetailAccumulator.getFromContext().ifPresent(ada -> ada.addAuditDetail(TestUtils.TABLE_NAME_PERSON, new QRecord().withValue("id", 1701), "Something happened"));
+
+      Collection<AuditSingleInput> auditSingleInputs = auditDetailAccumulator.getAccumulatedAuditSingleInputs();
+      assertEquals(1, auditSingleInputs.size());
+      assertThat(auditSingleInputs).anyMatch(asi -> asi.getAuditTableName().equals(TestUtils.TABLE_NAME_PERSON) && asi.getRecordId().equals(1701) && asi.getDetails().size() == 1);
+   }
+
+}

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/metadata/fields/DynamicDefaultValueBehaviorTest.java

@@ -24,6 +24,7 @@ package com.kingsrook.qqq.backend.core.model.metadata.fields;
 
 import java.time.LocalDate;
 import java.util.List;
+import java.util.Set;
 import com.kingsrook.qqq.backend.core.BaseTest;
 import com.kingsrook.qqq.backend.core.actions.values.ValueBehaviorApplier;
 import com.kingsrook.qqq.backend.core.context.QContext;
@@ -38,7 +39,7 @@ import static org.junit.jupiter.api.Assertions.assertNull;
 
 
 /*******************************************************************************
- ** Unit test for DynamicDefaultValueBehavior 
+ ** Unit test for DynamicDefaultValueBehavior
  *******************************************************************************/
 class DynamicDefaultValueBehaviorTest extends BaseTest
 {
@@ -53,7 +54,7 @@ class DynamicDefaultValueBehaviorTest extends BaseTest
       QTableMetaData table     = qInstance.getTable(TestUtils.TABLE_NAME_PERSON_MEMORY);
 
       QRecord record = new QRecord().withValue("id", 1);
-      ValueBehaviorApplier.applyFieldBehaviors(ValueBehaviorApplier.Action.INSERT, qInstance, table, List.of(record));
+      ValueBehaviorApplier.applyFieldBehaviors(ValueBehaviorApplier.Action.INSERT, qInstance, table, List.of(record), null);
 
       assertNotNull(record.getValue("createDate"));
       assertNotNull(record.getValue("modifyDate"));
@@ -71,7 +72,7 @@ class DynamicDefaultValueBehaviorTest extends BaseTest
       QTableMetaData table     = qInstance.getTable(TestUtils.TABLE_NAME_PERSON_MEMORY);
 
       QRecord record = new QRecord().withValue("id", 1);
-      ValueBehaviorApplier.applyFieldBehaviors(ValueBehaviorApplier.Action.UPDATE, qInstance, table, List.of(record));
+      ValueBehaviorApplier.applyFieldBehaviors(ValueBehaviorApplier.Action.UPDATE, qInstance, table, List.of(record), null);
 
       assertNull(record.getValue("createDate"));
       assertNotNull(record.getValue("modifyDate"));
@@ -79,6 +80,25 @@ class DynamicDefaultValueBehaviorTest extends BaseTest
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testOmitModifyDateUpdate()
+   {
+      QInstance      qInstance = QContext.getQInstance();
+      QTableMetaData table     = qInstance.getTable(TestUtils.TABLE_NAME_PERSON_MEMORY);
+
+      Set<FieldBehavior<?>> behaviorsToOmit = Set.of(DynamicDefaultValueBehavior.MODIFY_DATE);
+      QRecord               record          = new QRecord().withValue("id", 1);
+      ValueBehaviorApplier.applyFieldBehaviors(ValueBehaviorApplier.Action.UPDATE, qInstance, table, List.of(record), behaviorsToOmit);
+
+      assertNull(record.getValue("createDate"));
+      assertNull(record.getValue("modifyDate"));
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/
@@ -92,11 +112,11 @@ class DynamicDefaultValueBehaviorTest extends BaseTest
 
       QRecord record = new QRecord().withValue("id", 1);
 
-      ValueBehaviorApplier.applyFieldBehaviors(ValueBehaviorApplier.Action.INSERT, qInstance, table, List.of(record));
+      ValueBehaviorApplier.applyFieldBehaviors(ValueBehaviorApplier.Action.INSERT, qInstance, table, List.of(record), null);
       assertNull(record.getValue("createDate"));
       assertNull(record.getValue("modifyDate"));
 
-      ValueBehaviorApplier.applyFieldBehaviors(ValueBehaviorApplier.Action.UPDATE, qInstance, table, List.of(record));
+      ValueBehaviorApplier.applyFieldBehaviors(ValueBehaviorApplier.Action.UPDATE, qInstance, table, List.of(record), null);
       assertNull(record.getValue("createDate"));
       assertNull(record.getValue("modifyDate"));
    }
@@ -114,7 +134,7 @@ class DynamicDefaultValueBehaviorTest extends BaseTest
       table.getField("createDate").withType(QFieldType.DATE);
 
       QRecord record = new QRecord().withValue("id", 1);
-      ValueBehaviorApplier.applyFieldBehaviors(ValueBehaviorApplier.Action.INSERT, qInstance, table, List.of(record));
+      ValueBehaviorApplier.applyFieldBehaviors(ValueBehaviorApplier.Action.INSERT, qInstance, table, List.of(record), null);
       assertNotNull(record.getValue("createDate"));
       assertThat(record.getValue("createDate")).isInstanceOf(LocalDate.class);
    }
@@ -132,8 +152,8 @@ class DynamicDefaultValueBehaviorTest extends BaseTest
       table.getField("firstName").withBehavior(DynamicDefaultValueBehavior.CREATE_DATE);
 
       QRecord record = new QRecord().withValue("id", 1);
-      ValueBehaviorApplier.applyFieldBehaviors(ValueBehaviorApplier.Action.INSERT, qInstance, table, List.of(record));
+      ValueBehaviorApplier.applyFieldBehaviors(ValueBehaviorApplier.Action.INSERT, qInstance, table, List.of(record), null);
       assertNull(record.getValue("firstName"));
    }
 
-}
+}

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/automation/HealBadRecordAutomationStatusesProcessStepTest.java

@@ -29,13 +29,20 @@ import java.util.Map;
 import com.kingsrook.qqq.backend.core.BaseTest;
 import com.kingsrook.qqq.backend.core.actions.automation.AutomationStatus;
 import com.kingsrook.qqq.backend.core.actions.automation.RecordAutomationStatusUpdater;
+import com.kingsrook.qqq.backend.core.actions.processes.QProcessCallbackFactory;
+import com.kingsrook.qqq.backend.core.actions.processes.RunProcessAction;
 import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
 import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
 import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
 import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
+import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput;
+import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessOutput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.DynamicDefaultValueBehavior;
@@ -74,6 +81,43 @@ class HealBadRecordAutomationStatusesProcessStepTest extends BaseTest
 
 
 
+   /*******************************************************************************
+    ** at one point, when the review step go added, we were double-adding records
+    ** to the output/result screen.  This test verifies, if we run the full process
+    ** that that doesn't happen.
+    *******************************************************************************/
+   @Test
+   void testTwoFailedUpdatesFullProcess() throws QException
+   {
+      QContext.getQInstance().addProcess(new HealBadRecordAutomationStatusesProcessStep().produce(QContext.getQInstance()));
+
+      new InsertAction().execute(new InsertInput(tableName).withRecords(List.of(new QRecord(), new QRecord())));
+      List<QRecord> records = queryAllRecords();
+      RecordAutomationStatusUpdater.setAutomationStatusInRecordsAndUpdate(QContext.getQInstance().getTable(tableName), records, AutomationStatus.FAILED_UPDATE_AUTOMATIONS, null);
+
+      assertThat(queryAllRecords()).allMatch(r -> AutomationStatus.FAILED_UPDATE_AUTOMATIONS.getId().equals(getAutomationStatus(r)));
+
+      RunProcessInput input = new RunProcessInput();
+      input.setProcessName(HealBadRecordAutomationStatusesProcessStep.NAME);
+      input.setCallback(QProcessCallbackFactory.forFilter(new QQueryFilter(new QFilterCriteria("id", QCriteriaOperator.IN, records.stream().map(r -> r.getValue("id")).toList()))));
+      RunProcessAction runProcessAction = new RunProcessAction();
+      RunProcessOutput runProcessOutput = runProcessAction.execute(input);
+
+      input.setStartAfterStep(runProcessOutput.getProcessState().getNextStepName().get());
+      runProcessOutput = runProcessAction.execute(input);
+
+      input.setStartAfterStep(runProcessOutput.getProcessState().getNextStepName().get());
+      runProcessOutput = runProcessAction.execute(input);
+
+      List<QRecord> outputRecords = runProcessOutput.getProcessState().getRecords();
+      assertEquals(1, outputRecords.size());
+      assertEquals(2, outputRecords.get(0).getValueInteger("count"));
+
+      assertThat(queryAllRecords()).allMatch(r -> AutomationStatus.PENDING_UPDATE_AUTOMATIONS.getId().equals(getAutomationStatus(r)));
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/
@@ -103,7 +147,7 @@ class HealBadRecordAutomationStatusesProcessStepTest extends BaseTest
     **
     *******************************************************************************/
    @Test
-   void testOldRunning() throws QException
+   void testOldRunningUpdates() throws QException
    {
       /////////////////////////////////////////////////
       // temporarily remove the modify-date behavior //
@@ -160,6 +204,72 @@ class HealBadRecordAutomationStatusesProcessStepTest extends BaseTest
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testOldRunningInserts() throws QException
+   {
+      ///////////////////////////////////////////////////////////////
+      // temporarily remove the create-date & modify-date behavior //
+      ///////////////////////////////////////////////////////////////
+      QContext.getQInstance().getTable(tableName).getField("modifyDate").withBehavior(DynamicDefaultValueBehavior.NONE);
+      QContext.getQInstance().getTable(tableName).getField("createDate").withBehavior(DynamicDefaultValueBehavior.NONE);
+
+      //////////////////////////////////////////////////////////////////////////
+      // insert 2 records, one with an old createDate, one with 6 minutes ago //
+      // but set both with modifyDate very recent                             //
+      //////////////////////////////////////////////////////////////////////////
+      Instant old    = Instant.parse("2023-01-01T12:00:00Z");
+      Instant recent = Instant.now().minus(6, ChronoUnit.MINUTES);
+      new InsertAction().execute(new InsertInput(tableName).withRecords(List.of(
+         new QRecord().withValue("firstName", "Darin").withValue("createDate", old).withValue("modifyDate", recent),
+         new QRecord().withValue("firstName", "Tim").withValue("createDate", recent).withValue("modifyDate", recent)
+      )));
+      List<QRecord> records = queryAllRecords();
+
+      ///////////////////////////////////////////////////////
+      // put those records both in status: running-inserts //
+      ///////////////////////////////////////////////////////
+      RecordAutomationStatusUpdater.setAutomationStatusInRecordsAndUpdate(QContext.getQInstance().getTable(tableName), records, AutomationStatus.RUNNING_INSERT_AUTOMATIONS, null);
+
+      assertThat(queryAllRecords())
+         .allMatch(r -> AutomationStatus.RUNNING_INSERT_AUTOMATIONS.getId().equals(getAutomationStatus(r)));
+
+      //////////////////////////////////////////////////
+      // restore the createDate & modifyDate behavior //
+      //////////////////////////////////////////////////
+      QContext.getQInstance().getTable(tableName).getField("modifyDate").withBehavior(DynamicDefaultValueBehavior.MODIFY_DATE);
+      QContext.getQInstance().getTable(tableName).getField("createDate").withBehavior(DynamicDefaultValueBehavior.CREATE_DATE);
+
+      /////////////////////////
+      // run code under test //
+      /////////////////////////
+      RunBackendStepOutput output = runProcessStep();
+
+      /////////////////////////////////////////////////////////////////////////////////////////////
+      // assert we updated 1 (the old one) to pending-inserts, the other left as running-inserts //
+      /////////////////////////////////////////////////////////////////////////////////////////////
+      assertEquals(1, output.getValueInteger("totalRecordsUpdated"));
+      assertThat(queryAllRecords())
+         .anyMatch(r -> AutomationStatus.PENDING_INSERT_AUTOMATIONS.getId().equals(getAutomationStatus(r)))
+         .anyMatch(r -> AutomationStatus.RUNNING_INSERT_AUTOMATIONS.getId().equals(getAutomationStatus(r)));
+
+      /////////////////////////////////
+      // re-run, with 3-minute limit //
+      /////////////////////////////////
+      output = runProcessStep(new RunBackendStepInput().withValues(Map.of("minutesOldLimit", 3)));
+
+      /////////////////////////////////////////////////////////////////
+      // assert that one updated too, and all are now pending-insert //
+      /////////////////////////////////////////////////////////////////
+      assertEquals(1, output.getValueInteger("totalRecordsUpdated"));
+      assertThat(queryAllRecords())
+         .allMatch(r -> AutomationStatus.PENDING_INSERT_AUTOMATIONS.getId().equals(getAutomationStatus(r)));
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/

qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/importer/FilesystemImporterStep.java

@@ -362,7 +362,7 @@ public class FilesystemImporterStep implements BackendStep
          + "-" + sourceFileName.replaceAll(".*" + File.separator, "");
       path = AbstractBaseFilesystemAction.stripDuplicatedSlashes(path);
 
-      LOG.info("Archiving file", logPair("path", path));
+      LOG.info("Archiving file", logPair("path", path), logPair("archiveBackendName", archiveBackend.getName()), logPair("archiveTableName", archiveTable.getName()));
       archiveActionBase.writeFile(archiveBackend, path, bytes);
 
       return (path);

qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/AbstractS3Action.java

@@ -42,6 +42,7 @@ import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.AbstractF
 import com.kingsrook.qqq.backend.module.filesystem.exceptions.FilesystemException;
 import com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata.S3BackendMetaData;
 import com.kingsrook.qqq.backend.module.filesystem.s3.utils.S3Utils;
+import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
 
 
 /*******************************************************************************
@@ -162,9 +163,18 @@ public class AbstractS3Action extends AbstractBaseFilesystemAction<S3ObjectSumma
    @Override
    public void writeFile(QBackendMetaData backendMetaData, String path, byte[] contents) throws IOException
    {
-      path = stripLeadingSlash(stripDuplicatedSlashes(path));
       String bucketName = ((S3BackendMetaData) backendMetaData).getBucketName();
-      getS3Utils().writeFile(bucketName, path, contents);
+
+      try
+      {
+         path = stripLeadingSlash(stripDuplicatedSlashes(path));
+         getS3Utils().writeFile(bucketName, path, contents);
+      }
+      catch(Exception e)
+      {
+         LOG.warn("Error writing file", e, logPair("path", path), logPair("bucketName", bucketName));
+         throw (new IOException("Error writing file", e));
+      }
    }
 
 

qqq-dev-tools/bin/end-of-sprint-release.sh

@@ -60,7 +60,7 @@ cd $QQQ_RELEASE_DIR/qqq || exit
 
 MVN_VERIFY_LOG=/tmp/mvn-verify.log
 gumBanner "Doing clean build (logging to $MVN_VERIFY_LOG)"
-mvn clean verify > $MVN_VERIFY_LOG 2>&1
+mvn -Duser.timezone=UTC clean verify > $MVN_VERIFY_LOG 2>&1
 tail -30 $MVN_VERIFY_LOG
 gumConfirmProceed "Can we Proceed, or are there build errors to fix?" "Proceed" "There are build errors to fix"
 

qqq-middleware-api/src/main/java/com/kingsrook/qqq/api/actions/GenerateOpenApiSpecAction.java

@@ -1403,7 +1403,7 @@ public class GenerateOpenApiSpecAction extends AbstractQActionFunction<GenerateO
       {
          String           associatedTableName        = association.getAssociatedTableName();
          QTableMetaData   associatedTable            = QContext.getQInstance().getTable(associatedTableName);
-         ApiTableMetaData associatedApiTableMetaData = ObjectUtils.tryElse(() -> ApiTableMetaDataContainer.of(associatedTable).getApiTableMetaData(apiName), new ApiTableMetaData());
+         ApiTableMetaData associatedApiTableMetaData = ObjectUtils.tryAndRequireNonNullElse(() -> ApiTableMetaDataContainer.of(associatedTable).getApiTableMetaData(apiName), new ApiTableMetaData());
          String           associatedTableApiName     = StringUtils.hasContent(associatedApiTableMetaData.getApiTableName()) ? associatedApiTableMetaData.getApiTableName() : associatedTableName;
 
          ApiAssociationMetaData apiAssociationMetaData = thisApiTableMetaData.getApiAssociationMetaData().get(association.getName());

qqq-sample-project/pom.xml

@@ -68,7 +68,7 @@
       <dependency>
          <groupId>com.kingsrook.qqq</groupId>
          <artifactId>qqq-frontend-material-dashboard</artifactId>
-         <version>feature-CE-876-develop-missing-widget-types-20240221.011827-1</version>
+         <version>${revision}</version>
       </dependency>
       <dependency>
          <groupId>com.h2database</groupId>