Rename 'run' to 'runOnePage'

CE-1240: added multi table widget type

.circleci/config.yml

@@ -98,6 +98,31 @@ commands:
             - ~/.m2
           key: v1-dependencies-{{ checksum "pom.xml" }}
 
+  install_asciidoctor:
+    steps:
+      - checkout
+      - run:
+          name: Install asciidoctor
+          command: |
+            sudo apt-get update
+            sudo apt install -y asciidoctor
+
+  run_asciidoctor:
+    steps:
+      - run:
+          name: Run asciidoctor
+          command: |
+            cd docs
+            asciidoctor -a docinfo=shared index.adoc
+
+  upload_docs_site:
+    steps:
+      - run:
+          name: scp html to justinsgotskinnylegs.com
+          command: |
+            cd docs
+            scp index.html dkelkhoff@45.79.44.221:/mnt/first-volume/dkelkhoff/nginx/html/justinsgotskinnylegs.com/qqq-docs.html
+
 jobs:
   mvn_test:
     executor: localstack/default
@@ -114,6 +139,13 @@ jobs:
       - mvn_verify
       - mvn_jar_deploy
 
+  publish_asciidoc:
+    executor: localstack/default
+    steps:
+      - install_asciidoctor
+      - run_asciidoctor
+      - upload_docs_site
+
 workflows:
   test_only:
     jobs:
@@ -121,7 +153,7 @@ workflows:
           context: [ qqq-maven-registry-credentials, build-qqq-sample-app ]
           filters:
             branches:
-              ignore: /dev/
+              ignore: /(dev|integration.*)/
             tags:
               ignore: /(version|snapshot)-.*/
 
@@ -131,7 +163,10 @@ workflows:
           context: [ qqq-maven-registry-credentials, build-qqq-sample-app ]
           filters:
             branches:
-              only: /dev/
+              only: /(dev|integration.*)/
             tags:
               only: /(version|snapshot)-.*/
-
+      - publish_asciidoc:
+          filters:
+            branches:
+              only: /dev/

docs/.gitignore

@@ -0,0 +1 @@
+*.html

docs/actions/GetAction.adoc

@@ -0,0 +1,64 @@
+== GetAction
+include::../variables.adoc[]
+
+The `*GetAction*` is essentially a subset of the <<QueryAction>>, only specifically meant to get just a single record from a {link-table}.
+In SQL/RDBMS terms, it is analogous to a `SELECT` statement, where a single record may be found - or - it may not be found.
+
+For all tables, `GetAction` can do a lookup by primary key.
+In addition, for tables that define a `UniqueKey`, name/value pairs (in the form of a `Map`) can be used as input for `GetAction`.
+
+=== Examples
+[source,java]
+.Basic form - by primary key
+----
+GetInput input = new GetInput();
+input.setTableName("orders");
+input.setPrimaryKey(1);
+GetOutput output = new GetAction.execute(input);
+QRecord record = output.getRecord();
+----
+
+[source,java]
+.Secondary form - by Unique Key
+----
+GetInput input = new GetInput();
+input.setTableName("products");
+input.setUniqueKey(Map.of("storeId", 1701, "sku", "ABCD"));
+GetOutput output = new GetAction.execute(input);
+QRecord record = output.getRecord();
+----
+
+=== GetInput
+* `table` - *String, Required* - Name of the table being queried against.
+* `primaryKey` - *Serializable, Conditional* - Value for the primary key field of the table being queried.
+Type should match the table's primary key field's type.
+If a `primaryKey` is not given, then a `uniqueKey` must be given.
+* `uniqueKey` - *Map of String -> Serializable, Conditional* - Map of name-value pairs that define the record to be fetcheed.
+Keys in the map must be field names from the table being queried.
+Values in the map should should be of types that correspond to the fields.
+If a `primaryKey` is not given, then a `uniqueKey` must be given.
+If both `primaryKey` and `uniqueKey` are given, then `uniqueKey` is ignored.
+* `transaction` - *QBackendTransaction object* - Optional transaction object.
+** Behavior for this object is backend-dependant.
+In an RDBMS backend, this object is generally needed if you want your query to see data that may have been modified within the same transaction.
+
+* `shouldTranslatePossibleValues` - *boolean, default: false* - Controls whether any fields in the table with a *possibleValueSource* assigned to them should have those possible values looked up
+(e.g., to provide text translations in the generated records' `displayValues` map).
+** For example, if getting a record to present to a user, this would generally need to be *true*.
+But if getting a record as part of a process, then this can generally be left as *false*.
+* `shouldGenerateDisplayValues` - *boolean, default: false* - Controls whether field level *displayFormats* should be used to populate the generated records' `displayValues` map.
+** For example, if getting a record to present to a user, this would generally need to be *true*.
+But if getting a record as part of a process, then this can generally be left as *false*.
+* `shouldFetchHeavyFields` - *boolean, default: true* - Controls whether or not fields marked as `isHeavy` should be fetched & returned or not.
+* `shouldOmitHiddenFields` - *boolean, default: true* - Controls whether or not fields marked as `isHidden` should be included in the result or not.
+* `shouldMaskPassword` - *boolean, default: true* - Controls whether or not fields with `type` = `PASSWORD` should be masked, or if their actual values should be returned.
+* `queryJoins` - *List of <<QueryJoin>> objects* - Optional list of tables to be joined with the main table being queried.
+See QueryJoin under <<QueryAction>> for further details.
+* `includeAssociations` - *boolean, default: false* - Control whether or not records from tables defined as `associations` under the table being queried should be included in the result or not.
+* `associationNamesToInclude` - *Collection of String* - If `includeAssociations` is true, then this field can be used to limit which associated tables are included.
+If this field is null, then all associated tables are included.
+Otherwise, a table is only included if its name is in this collection.
+
+=== GetOutput
+* `record` - *QRecord* - The record that was specified by the input `primaryKey` or `uniqueKey`.
+Will be null if the record was not found.

docs/actions/InsertAction.adoc

@@ -0,0 +1,86 @@
+== InsertAction
+include::../variables.adoc[]
+
+To insert (add, create) new records into any {link-table}, the `*InsertAction*` is used.
+In SQL/RDBMS terms, it is analogous to a `INSERT` statement, where one or more records can be provided as input.
+
+=== Examples
+[source,java]
+.Canonical InsertAction invocation
+----
+InsertInput insertInput = new InsertInput();
+insertInput.setTableName("person");
+insertInput.setRecords(personRecordList);
+InsertOutput insertOutput = new InsertAction().execute(insertInput);
+List<QRecord> insertedPersonRecords = insertOutput.getRecords();
+----
+
+=== Details
+`InsertAction` does several things beyond just inserting records into the specified table.
+A high-level flow of its internal logic is:
+
+. For tables using an automation status field, set its value to `PENDING_INSERT_AUTOMATIONS` for all `records` that are going to be inserted.
+. Perform the following validations, which include running the table's `PRE_INSERT_CUSTOMIZER`, if one is defined, at the time that is specified by the customizer's `WhenToRun` property (default is `AFTER_ALL_VALIDATIONS`):
+.. Ensure that default values specified in the table's fields are present if needed.
+.. Apply per-field behaviors, as defined in {link-field} meta-data, such as truncating strings that are longer than their specified max.
+.. Check for unique key violations (done here instead of in the backend, to provide better error messaging, and to allow a subset of records to be stored while some fail).
+_We might want to make an input control in the future to specify that either the full input set should succeed or fail..._
+.. Validate that required fields (again, per {link-field} meta-data) are set, generating per-record errors if they are not.
+.. Validate any security fields in the records - e.g., ensure that the user has permission to insert records with the values they are attempting to insert.
+. Send the records to the table's backend module to actually insert them into the backend storage.
+. If the table has any associations defined, and if associated records are present, then recursively run `InsertAction` on the associated records.
+.. In particular, before these recursive `InsertAction` calls are made, values that were generated by the original insert may need to be propagated down into the associated records.
+*** For example, if inserting `order` and `lineItem` records, where a {link-join} exists between the two tables on `order.id` and `lineItem.orderId`, and `order.id` values were generated in the first `InsertAction`, then those values are propagated down into the associated `lineItem.orderId` fields.
+. If the {link-instance} has an `audit` table, then based on the {link-table}'s audit rules, audits about the inserted records are created.
+. If the table has a `POST_INSERT_CUSTOMIZER`, it is executed.
+
+=== Overloads
+
+`InsertAction` can be called in a few alternate forms, mostly just for convenience:
+
+[source,java]
+.If inserting a single record, get that record back instead of the InsertOutput:
+----
+InsertInput insertInput = new InsertInput();
+insertInput.setTableName("person");
+insertInput.setRecords(List.of(personRecord));
+QRecord insertedRecord = new InsertAction().executeForRecord(insertInput);
+
+// or more compactly, using InsertInput.withRecord (instead of withRecords)
+QRecord insertedRecord = new InsertAction()
+   .executeForRecord(new InsertInput("person").withRecord(personRecord));
+----
+
+[source,java]
+.Taking QRecordEntity objects as inputs instead of QRecords:
+----
+// insert a list of person entities:
+InsertInput insertInput = new InsertInput("person").withRecordEntities(personList);
+InsertOutput insertOutput = new InsertAction().execute(insertInput);
+
+// or for a single person entity (also mapping the output record back to an entity):
+Person insertedPerson = new Person(new InsertAction()
+   .executeForRecord(new InsertInput("person").withRecordEntity(person)));
+
+----
+
+=== InsertInput
+* `table` - *String, Required* - Name of the table that records are to be inserted into.
+* `records` - *List of QRecord, Required* - List of records to be inserted into the table.
+If the list is empty, the insert action does a `noop`.
+
+.Less common options
+* `inputSource` - *InputSource object, default: QInputSource.SYSTEM* - an indicator of what the source of the action is - generally, a `SYSTEM` triggered action, or a `USER` triggered action.
+** `InsertAction` will call the `shouldValidateRequiredFields()` method on this object to determine if it should validate that required fields on the records have values.
+Both `QInputSource.SYSTEM` and `QInputSource.USER` return `true` from this method, but application can define their own `InputSource` objects with different behavior.
+** In addition, this field can be used in pre- and post-insert customizers to drive further custom logic.
+* `skipUniqueKeyCheck` - *boolean, default: false* - control whether or not `InsertAction` should check for unique key violations before attempting to insert its records.
+** In a context where one has previously done this validation, or is okay letting the backend provide such checks, they may wish to avoid re-doing this work, and thus may set this property to `true`.
+* `omitDmlAudit` - *boolean, default: false* - control if the automatic DML audit that `InsertAction` generally performs should be omitted.
+* `auditContext` - *String* - optional message which can be included in DML audit messages, to give users more context about why the insert occurred.
+
+
+=== InsertOutput
+* `records` - *List of QRecord* - Copy of the input list of records, with details added based on the results of the input action.
+** If there were warnings or errors, the corresponding field (`warnings` or `errors`) will be set in the records.
+** If the insert action generated any values (such as a serial id or a default value), those values will be in the record's `fields` map.

docs/index.adoc

@@ -26,6 +26,16 @@ include::metaData/Reports.adoc[leveloffset=+1]
 include::metaData/Icons.adoc[leveloffset=+1]
 include::metaData/PermissionRules.adoc[leveloffset=+1]
 
+== Services
+
+include::misc/ScheduledJobs.adoc[leveloffset=+1]
+
+=== Web server (Javalin)
+#todo#
+
+=== API server (OpenAPI)
+#todo#
+
 == Custom Application Code
 include::misc/QContext.adoc[leveloffset=+1]
 include::misc/QRecords.adoc[leveloffset=+1]
@@ -63,4 +73,4 @@ include::implementations/TableSync.adoc[leveloffset=+1]
 // later... include::actions/RenderTemplateAction.adoc[leveloffset=+1]
 
 == QQQ Utility Classes
-include::utilities/RecordLookupHelper.adoc[leveloffset=+1]
+include::utilities/RecordLookupHelper.adoc[leveloffset=+1]

docs/justfile

@@ -0,0 +1,12 @@
+## https://github.com/casey/just
+
+default:
+   just --list
+
+build-index-html:
+   asciidoctor -a docinfo=shared index.adoc
+   line=$(grep 'Last updated' index.html) && sed -i "s/id=\"content\">/&$line/" index.html
+
+build-and-publish-index-html: build-index-html
+   scp index.html first-node:/mnt/first-volume/dkelkhoff/nginx/html/justinsgotskinnylegs.com/qqq-docs.html
+   @echo "Updated: https://justinsgotskinnylegs.com/qqq-docs.html"

docs/metaData/Fields.adoc

@@ -21,5 +21,99 @@ Used to set values in the `displayValues` map within a `QRecord`.
 * `possibleValueSourceName` - *String* - Reference to a {link-pvs} to be used for this field.
 Values in this field should correspond to ids from the referenced Possible Value Source.
 * `maxLength` - *Integer* - Maximum length (number of characters) allowed for values in this field.
-Only applicable for fields with `type=STRING`.
-* `
+Only applicable for fields with `type=STRING`.  Needs to be used with a `FieldBehavior` of type `ValueTooLongBehavior`.
+
+==== Field Behaviors
+Additional behaviors can be attached to fields through the use of the `behaviors` attribute,
+which is a `Set` of 0 or more instances of implementations of the `FieldBehavior` interface.
+Note that in some cases, these instances may be `enum` constants,
+but other times may be regular Objects.
+
+QQQ provides a set of common field behaviors.
+Applications can also define their own field behaviors by implementing the `FieldBehavior` interface,
+and attaching instances of their custom behavior classes to fields.
+
+===== ValueTooLongBehavior
+Used on String fields.  Requires the field to have a `maxLength` set.
+Depending on the chosen instance of this enum, before a record is Inserted or Updated,
+if the value in the field is longer than the `maxLength`, then one of the following actions can occur:
+
+* `TRUNCATE` - The value will be simply truncated to the `maxLength`.
+* `TRUNCATE_ELLIPSIS` - The value will be truncated to 3 characters less than the `maxLength`, and three periods (an ellipsis) will be placed at the end.
+* `ERROR` - An error will be reported, and the record will not be inserted or updated.
+* `PASS_THROUGH` - Nothing will happen.  This is the same as not having a `ValueTooLongBehavior` on the field.
+
+[source,java]
+.Examples of using ValueTooLongBehavior
+----
+   new QFieldMetaData("sku", QFieldType.STRING)
+      .withMaxLength(40),
+      .withBehavior(ValueTooLongBehavior.ERROR),
+
+   new QFieldMetaData("reason", QFieldType.STRING)
+      .withMaxLength(250),
+      .withBehavior(ValueTooLongBehavior.TRUNCATE_ELLIPSIS),
+
+----
+
+===== DynamicDefaultValueBehavior
+Used to set a dynamic default value to a field when it is being inserted or updated.
+For example, instead of having a hard-coded `defaultValue` specified in the field meta-data,
+and instead of having to add, for example, a pre-insert custom action.
+
+* `CREATE_DATE` - On inserts, sets the field's value to the current time.
+* `MODIFY_DATE` - On inserts and updates, sets the field's value to the current time.
+* `USER_ID` - On inserts and updates, sets the field's value to the current user's id (but only if the value is currently null).
+
+_Note that the `QInstanceEnricher` will, by default, add the `CREATE_DATE` and `MODIFY_DATE` `DynamicDefaultValueBehavior`
+options to any fields named `"createDate"` or `"modifyDate"`.
+This behavior can be disabled by setting the `configAddDynamicDefaultValuesToFieldsNamedCreateDateAndModifyDate` property
+on the `QInstanceEnricher` instance used by the application to `false`._
+
+[source,java]
+.Examples of using DynamicDefaultValueBehavior
+----
+   new QFieldMetaData("createDate", QFieldType.DATE_TIME)
+      .withBehavior(DynamicDefaultValueBehavior.CREATE_DATE),
+
+   new QFieldMetaData("modifyDate", QFieldType.DATE_TIME)
+      .withBehavior(DynamicDefaultValueBehavior.MODIFY_DATE),
+
+   new QFieldMetaData("createdByUserId", QFieldType.STRING)
+      .withBehavior(DynamicDefaultValueBehavior.USER_ID),
+----
+
+===== DateTimeDisplayValueBehavior
+By default, in QQQ, fields of type `DATE_TIME` are stored in UTC,
+and their values in a QRecord is a java `Instant` instance, which is always UTC.
+However, frontends will prefer to display date-time values in the user's local Time Zone whenever possible.
+
+Using `DateTimeDisplayValueBehavior` allows a `DATE_TIME` field to be displayed in a different Time Zone.
+An example use-case for this would be displaying airplane flight times,
+where you would want a flight from California to New York to display Pacific Time for its departure time,
+and Eastern Time for its arrival.
+
+An instance of `DateTimeDisplayValueBehavior` can be configured to either use a hard-coded time `ZoneId`
+(for example, to always show users UTC, or a business's home-office time zone).
+Or, it can be set up to get the time zone to use from another field in the table.
+
+[source,java]
+.Examples of using DateTimeDisplayValueBehavior
+----
+new QTableMetaData().withName("flights").withFields(List.of(
+   ...
+   new QFieldMetaData("departureTimeZoneId", QFieldType.STRING),
+   new QFieldMetaData("arrivaTimeZoneId", QFieldType.STRING),
+
+   new QFieldMetaData("departureTime", QFieldType.DATE_TIME)
+      .withBehavior(new DateTimeDisplayValueBehavior()
+         .withZoneIdFromFieldName("departureTimeZoneId")),
+
+   new QFieldMetaData("arrivalTime", QFieldType.DATE_TIME)
+      .withBehavior(new DateTimeDisplayValueBehavior()
+         .withZoneIdFromFieldName("arrivalTimeZoneId"))
+
+   new QFieldMetaData("ticketSaleStartDateTime", QFieldType.DATE_TIME)
+      .withBehavior(new DateTimeDisplayValueBehavior()
+         .withDefaultZoneId("UTC"))
+----

docs/metaData/Icons.adoc

@@ -0,0 +1,20 @@
+[#Icons]
+== Icons
+include::../variables.adoc[]
+
+#TODO#
+
+=== QIcon
+Icons are defined in a QQQ Instance in a `*QIcon*` object.
+
+#TODO#
+
+*QIcon Properties:*
+
+* `name` - *String* - Name of an icon from the https://mui.com/material-ui/material-icons/[Material UI Icon set]
+** Note that icon names from the link above need to be converted from _CamelCase_ to _underscore_case_...
+* `path` - *String* - Path to a file served under the application's web root, to be used as the icon.
+
+_Either `name` or `path` must be specified. If both are given, then name is used._
+
+

docs/metaData/Joins.adoc

@@ -2,16 +2,57 @@
 == Joins
 include::../variables.adoc[]
 
-#TODO#
+A `QJoinMetaData` is a meta-data object that tells QQQ, essentially “it is possible for these 2 tables to join, here’s how to do it”.
+
+Joins can be used then, in an application, in a number of possible ways:
+
+* In a {link-table}, we can specify joins to be “exposed”, e.g., made available to users on a query screen
+* Also in a Table, as part of an “Association”, which sets up one table as a “parent” of another,
+such that you can store (and fetch) the child-records at the same time as the parent
+** A common use-case here may be an order & lineItem table -
+such that QQQ can generate an API uses to allow you to post an order and its lines in a single request, and they get stored all together
+* In defining the security field (record lock) on a table,
+sometimes, it isn’t a field directly on the table, but instead comes from a joined table (possibly even more than just 1 table away).
+** For example, maybe a lineItem table, doesn't have a clientId, but needs secured by that field
+- so its recordLock can specify a “joinNameChain” that describes how to get from lineItem to order.clientId
+* The `QueryAction` can take (through its QueryInput object) zero or more QueryJoin objects,
+which must make a reference (implicitly or explicitly) to a QJoinMetaData.
+See the section on <<QueryJoin,QueryJoins>> for more details.
 
 === QJoinMetaData
 Joins are defined in a QQQ Instance in a `*QJoinMetaData*` object.
 
-#TODO#
+In this object, we have the concept of a "leftTable" and a "rightTable".
+There isn't generally anything special about which table is on the "left" and which is on the "right".
+But the remaining pieces of the QJoinMetaData do all need to line-up with these sides.
+
+For example:
+
+* The Type (one-to-one, one-to-many, many-to-one) - where the leftTable comes first, and rightTable comes second
+(e.g., a one-to-many means 1-row in leftTable has many-rows in rightTable associated with it)
+* In a JoinOn object, the 1st field name given is from the leftTable;
+the second fieldName from the rightTable.
 
 *QJoinMetaData Properties:*
 
-* `name` - *String, Required* - Unique name for the join within the QQQ Instance. #todo infererences or conventions?#
-
-#TODO#
+* `name` - *String, Required* - Unique name for the join within the QQQ Instance.
+** One convention is to name joins based on (leftTable + "Join" + rightTable).
+** If you do not wish to define join names yourself, the method `withInferredName()`
+can be called (which defers to
+`public static String makeInferredJoinName(String leftTable, String rightTable)`),
+to create a name for the join following the (leftTable + "Join" + rightTable) convention.
+* `leftTable` - *String, Required* - Name of a {link-table} in the {link-instance}.
+* `rightTable` - *String, Required* - Name of a {link-table} in the {link-instance}.
+* `type` - *enum, Required* - cardinality between the two tables in the join.
+** e.g., `ONE_TO_ONE`, `ONE_TO_MANY` (indicating 1 record in the left table may join
+to many records in the right table), or `MANY_TO_ONE` (vice-versa).
+** Note that there is no MANY_TO_MANY option, as a many-to-many is built as multiple QJoinMetaData's
+going through the intermediary (intersection) table.
+* `joinOns` - *List<JoinOn>, Required* - fields used to join the tables.
+Note: In the 2-arg JoinOn constructor, the leftTable's field comes first.
+Alternatively, the no-arg constructor can be used along with `.withLeftField().withRightField()`
+* `orderBys` - *List<QFilterOrderBy>* - Optional list of order-by objects,
+used in some framework-generated queries using the join.
+The field names are assumed to come from the rightTable.
 
+#TODO# what else do we need here?

docs/metaData/PermissionRules.adoc

@@ -0,0 +1,13 @@
+[#PermissionRules]
+== Permission Rules
+include::../variables.adoc[]
+
+#TODO#
+
+=== PermissionRule
+#TODO#
+
+*PermissionRule Properties:*
+
+#TODO#
+

docs/metaData/Processes.adoc

@@ -16,6 +16,7 @@ Processes are defined in a QQQ Instance in a `*QProcessMetaData*` object.
 In addition to directly building a `QProcessMetaData` object setting its properties directly, there are a few common process patterns that provide *Builder* objects for ease-of-use.
 See StreamedETLWithFrontendProcess below for a common example
 
+[#_QProcessMetaData_Properties]
 *QProcessMetaData Properties:*
 
 * `name` - *String, Required* - Unique name for the process within the QQQ Instance.
@@ -30,12 +31,13 @@ See below for details.
 * `permissionRules` - *QPermissionRules object* - define the permission/access rules for the process.
 See {link-permissionRules} for details.
 * `steps` and `stepList` - *Map of String → <<QStepMetaData>>* and *List of QStepMetaData* - Defines the <<QFrontendStepMetaData,screens>> and <<QBackendStepMetaData,backend code>> that makes up the process.
-** `stepList` is the list of steps in the order that they will by default be executed.
-** `steps` is a map, including all steps from `stepList`, but which may also include steps which can used by the process if its backend steps make the decision to do so, at run-time.
+** `stepList` is the list of steps in the order that they will be executed
+(that is to say - this is the _default_ order of execution - but it can be customized - see <<_custom_process_flow>> for details).
+** `steps` is a map, including all steps from `stepList`, but which may also include steps which can used by the process if its backend steps make the decision to do so, at run-time (e.g., using <<_custom_process_flow>>).
 ** A process's steps are normally defined in one of two was:
 *** 1) by a single call to `.withStepList(List<QStepMetaData>)`, which internally adds each step into the `steps` map.
 *** 2) by multiple calls to `.addStep(QStepMetaData)`, which adds a step to both the `stepList` and `steps` map.
-** If a process also needs optional steps, they should be added by a call to `.addOptionalStep(QStepMetaData)`, which only places them in the `steps` map.
+** If a process also needs optional steps (for a <<_custom_process_flow>>), they should be added by a call to `.addOptionalStep(QStepMetaData)`, which only places them in the `steps` map.
 * `schedule` - *<<QScheduleMetaData>>* - set up the process to run automatically on the specified schedule.
 See below for details.
 * `minInputRecords` - *Integer* - #not used...#
@@ -214,3 +216,112 @@ But for some cases, doing page-level transactions can reduce long-transactions a
 * `withFields(List<QFieldMetaData> fieldList)` - Adds additional input fields to the preview step of the process.
 * `withBasepullConfiguration(BasepullConfiguration basepullConfiguration)` - Add a <<BasepullConfiguration>> to the process.
 * `withSchedule(QScheduleMetaData schedule)` - Add a <<QScheduleMetaData>> to the process.
+
+[#_custom_process_flow]
+==== Custom Process Flow
+As referenced in the definition of the <<_QProcessMetaData_Properties,QProcessMetaData Properties>>, by default, a process
+will execute each of its steps in-order, as defined in the `stepList` property.
+However, a Backend Step can customize this flow #todo - write more clearly here...
+
+There are generally 2 method to call (in a `BackendStep`) to do a dynamic flow:
+
+* `RunBackendStepOutput.setOverrideLastStepName(String stepName)`
+** QQQ's `RunProcessAction` keeps track of which step it "last" ran, e.g., to tell it which one to run next.
+However, if a step sets the `OverrideLastStepName` property in its output object,
+then the step named in that property becomes the effective "last" step,
+thus determining which step comes next.
+
+* `RunBackendStepOutput.updateStepList(List<String> stepNameList)`
+** Calling this method changes the process's runtime definition of steps to be executed.
+Thus allowing a completely custom flow.
+It should be noted, that the "last" step name (as tracked by QQQ within `RunProcessAction`)
+does need to be found in the new `stepNameList` - otherwise, the framework will not know where you were,
+for figuring out where to go next.
+
+[source,java]
+.Example of a defining process that can use a flexible flow:
+----
+// for a case like this, it would be recommended to define all step names in constants:
+public final static String STEP_START = "start";
+public final static String STEP_A     = "a";
+public final static String STEP_B     = "b";
+public final static String STEP_C     = "c";
+public final static String STEP_1     = "1";
+public final static String STEP_2     = "2";
+public final static String STEP_3     = "3";
+public final static String STEP_END   = "end";
+
+// also, to define the possible flows (lists of steps) in constants as well:
+public final static List<String> LETTERS_STEP_LIST = List.of(
+  STEP_START, STEP_A, STEP_B, STEP_C, STEP_END);
+
+public final static List<String> NUMBERS_STEP_LIST = List.of(
+  STEP_START, STEP_1, STEP_2, STEP_3, STEP_END);
+
+// when we define the process's meta-data, we only give a "skeleton" stepList -
+// we must at least have our starting step, and we may want at least one frontend step
+// for the UI to show some placeholder(s):
+QProcessMetaData process = new QProcessMetaData()
+   .withName(PROCESS_NAME)
+   .withStepList(List.of(
+      new QBackendStepMetaData().withName(STEP_START)
+         .withCode(new QCodeReference(/*...*/)),
+      new QFrontendStepMetaData()
+         .withName(STEP_END)
+   ));
+
+// the additional steps get added via `addOptionalStep`, which only puts them in
+// the process's stepMap, not its stepList!
+process.addOptionalStep(new QFrontendStepMetaData().withName(STEP_A));
+process.addOptionalStep(new QBackendStepMetaData().withName(STEP_B)
+   .withCode(new QCodeReference(/*...*/)));
+process.addOptionalStep(new QFrontendStepMetaData().withName(STEP_C));
+
+process.addOptionalStep(new QBackendStepMetaData().withName(STEP_1)
+   .withCode(new QCodeReference(/*...*/)));
+process.addOptionalStep(new QFrontendStepMetaData().withName(STEP_2));
+process.addOptionalStep(new QBackendStepMetaData().withName(STEP_3)
+   .withCode(new QCodeReference(/*...*/)));
+
+----
+
+[source,java]
+.Example of a process backend step adjusting the process's runtime flow:
+----
+/***************************************************************************
+** look at the value named "which". if it's "letters", then make the process
+** go through the stepList consisting of letters; else, update the step list
+** to be the "numbers" steps.
+**
+** Also - if the "skipSomeSteps" value is give as true, then set the
+** overrideLastStepName to skip again (in the letters case, skip past A, B
+** and C; in the numbers case, skip past 1 and 2).
+***************************************************************************/
+public static class StartStep implements BackendStep
+{
+  @Override
+  public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException
+  {
+     Boolean skipSomeSteps = runBackendStepInput.getValueBoolean("skipSomeSteps");
+
+     if(runBackendStepInput.getValueString("which").equals("letters"))
+     {
+        runBackendStepOutput.updateStepList(LETTERS_STEP_LIST);
+        if(BooleanUtils.isTrue(skipSomeSteps))
+        {
+           runBackendStepOutput.setOverrideLastStepName(STEP_C);
+        }
+     }
+     else
+     {
+        runBackendStepOutput.updateStepList(NUMBERS_STEP_LIST);
+        if(BooleanUtils.isTrue(skipSomeSteps))
+        {
+           runBackendStepOutput.setOverrideLastStepName(STEP_2);
+        }
+     }
+  }
+}
+----
+
+

docs/metaData/QInstance.adoc

@@ -0,0 +1,39 @@
+[#QInstance]
+== QInstance
+include::../variables.adoc[]
+
+An application in QQQ is defined as a set of Meta Data objects.  These objects are all stored together in an object called a `*QInstance*`.
+
+Currently, a `QInstance` must be programmatically constructed in java code - e.g., by constructing objects which get added to the QInstance, for example:
+
+[source,java]
+.Adding meta-data for two tables and one process to a QInstance
+----
+QInstance qInstance = new QInstance();
+qInstance.addTable(definePersonTable());
+qInstance.addTable(defineHomeTable());
+qInstance.addProcess(defineSendPersonHomeProcess());
+----
+
+It is on the QQQ roadmap to allow meta-data to be defined in a non-programmatic way, for example, in YAML or JSON files, or even from a dynamic data source (e.g. a document or relational database).
+
+The middleware and/or frontends being used in an application will drive how the `QInstance` is connected to the running server/process.
+For example, using the `qqq-middleware-javalin` module, a the `QJavalinImplementation` class () has a constructor which takes a `QInstance` as an argument:
+
+[source,java]
+.Starting a QQQ Javalin middleware server - passing a QInstance as a parameter to the new QJavalinImplementation
+----
+QJavalinImplementation qJavalinImplementation = new QJavalinImplementation(qInstance);
+Javalin service = Javalin.create();
+service.routes(qJavalinImplementation.getRoutes());
+service.start();
+----
+
+*QBackendMetaData Setup Methods:*
+
+These are the methods that one is most likely to use when setting up (defining) a `QInstance` object:
+
+* asdf
+
+*QBackendMetaData Usage Methods:*
+

docs/misc/ProcessBackendSteps.adoc

@@ -0,0 +1,155 @@
+== Process Backend Steps
+include::../variables.adoc[]
+
+In many QQQ applications, much of the code that engineers write will take the form of Backend Steps for {link-processes}.
+Such code is defined in classes which implement the interface `BackendStep`.  
+This interface defines only a single method:
+[source,java]
+.BackendStep.java
+----
+public interface BackendStep
+{
+   /*******************************************************************************
+    ** Execute the backend step - using the request as input, and the result as output.
+    **
+    *******************************************************************************/
+   void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException;
+}
+
+----
+
+Process backend steps have access to state information - specifically, a list of records, and a map of name=value pairs - in the input & output objects.
+This state data is persisted by QQQ between steps (e.g., if a frontend step is presented to a user between backend steps).
+
+
+=== RunBackendStepInput
+All input data to the step is available in the `RunBackendStepInput` object.
+Key methods in this class are:
+
+* `getRecords()` - Returns the List of <<QRecords>> that are currently being acted on in the process.
+* `getValues()` - Returns a Map of String -> Serializable; name=value pairs that are the state-data of the process.
+** Values can be added to this state from a process's meta-data, from a screen, or from another backend step.
+* `getValue(String fieldName)` - Returns a specific value, by name, from the process's state.
+** This method has several variations that return the value as a specific type, such as `getValueString`, `getValueInteger`, `getValueBoolean`...
+* `getAsyncJobCallback()` - Accessor for an `AsyncJobCallback` object, which provides a way for the process backend step to communicate about its status or progress with a user running the process in a frontend.
+Provides methods:
+** `updateStatus(String message)` - Where general status messages can be given.
+For example, `"Loading census data"`
+** `updateStatus(int current, int total)` - For updating a progress meter.
+e.g., "47 of 1701" would be display by calling `.updateStatus(47, 1701)`
+* `getFrontendStepBehavior()` - Enum, indicating what should happen when a frontend step is encountered as the process's next step to run.
+Possible values are:
+** `BREAK` - Indicates that the process's execution should be suspended, so that the screen represented by the frontend step can be presented to a user.
+This would be the expected behavior if a process is being run by a user from a UI.
+** `SKIP` - Indicates that frontend steps should be skipped.
+This would be the expected behavior if a process is running from a scheduled job (without a user present to drive it), for example.
+** `FAIL` - Indicates that the process should end with an exception if a frontend step is encountered.
+** A backend step may want to act differently based on its frontendStepBehavior.
+For example, additional data may be looked up for displaying to a user if the behavior is `BREAK`.
+* `getBasepullLastRunTime()` - For <<BasepullConfiguration,Basepull>> processes, this is the `Instant` stored in the basepull table as the process's last run time.
+
+=== RunBackendStepOutput
+All output from a process step should be placed in its `RunBackendStepOutput` object (and/or stored to a backend, as appropriate).
+Key methods in this class are:
+
+* `addValue(String fieldName, Serializable value)` - Adds a single named value to the process's state, overwriting it the value if it already exists.
+* `addRecord(QRecord record)` - Add a `<<QRecord>>` to the process's output.
+* `addAuditSingleInput(AuditSingleInput auditSingleInput)` - Add a new entry to the process's list of audit inputs, to be stored at the completion of the process.
+** An `AuditSingleInput` object can most easily be built with the constructor: `AuditSingleInput(String tableName, QRecord record, String auditMessage)`.
+** Additional audit details messages (sub-bullets that accompany the primary `auditMessage`) can be added to an `AuditSingleInput` via the `addDetail(String message)` method.
+** _Note that at this time, the automatic storing of these audits is only provided by the execute step of a StreamedETLWithFrontendProcesses._
+
+=== Example
+
+[source,java]
+.Example of a BackendStep
+----
+/*******************************************************************************
+ ** For the "person" table's "Add Age" process -
+ ** For each input person record, add the specified yearsToAdd to their age.
+ *******************************************************************************/
+public class AddAge implements BackendStep
+{
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput)
+   {
+      /////////////////////////////////////////////////////////////////
+      // get the yearsToAdd input field value from the process input //
+      /////////////////////////////////////////////////////////////////
+      Integer yearsToAdd = runBackendStepInput.getValueInteger("yearsToAdd");
+      int totalYearsAdded = 0;
+
+      ///////////////////////////////////////////////////
+      // loop over the records passed into the process //
+      ///////////////////////////////////////////////////
+      for(QRecord record : runBackendStepInput.getRecords())
+      {
+         Integer age = record.getValueInteger("age");
+         age += yearsToAdd;
+         totalYearsAdded += yearsToAdd;
+
+         ////////////////////////////////////////////////////////////////////////////////////////////
+         // update the record with the new "age" value.                                            //
+         // note that this update record object will implicitly be available to the process's next //
+         // backend step, via the sharing of the processState object.                              //
+         ////////////////////////////////////////////////////////////////////////////////////////////
+         record.setValue("age", age);
+      }
+
+      /////////////////////////////////////////
+      // set an output value for the process //
+      /////////////////////////////////////////
+      runBackendStepOutput.addValue("totalYearsAdded", totalYearsAdded);
+   }
+}
+
+----
+
+
+=== Backend Steps for StreamedETLWithFrontendProcesses
+
+For <<StreamedETLWithFrontendProcess>> type processes, backend steps are defined a little bit differently than they are for other process types.
+In this type of process, the process meta-data defines 3 backend steps which are built-in to QQQ, and which do not have any custom application logic.
+These steps are:
+
+* `StreamedETLPreviewStep`
+* `StreamedETLValidateStep`
+* `StreamedETLExecuteStep`
+
+For custom application logic to be implemented in a StreamedETLWithFrontendProcesses, an application engineer must define (up to) 3 backend step classes which are loaded by the steps listed above.
+These application-defined steps must extend specific classes (which themselves implement the `BackendStep` interface), to provide the needed logic of this style of process.
+These steps are:
+
+* *Extract* - a subclass of `AbstractExtractStep` - is responsible for Extracting records from the source table.
+** For this step, we can often use the QQQ-provided class `ExtractViaQueryStep`, or sometimes a subclass of it.
+** The Extract step is called before the Preview, Validate, and Result screens, though for the Preview screen, it is set to only extract a small number of records (10).
+* *Transform* - a subclass of `AbstractTransformStep` - is responsible for applying the majority of the business logic of the process.
+In ETL terminology, this is the "Transform" action - which means applying some type of logical transformation an input record (found by the Extract step) to generate an output record (stored by the Load step).
+** A Transform step's `run` method will be called, potentially, multiple times, each time with a page of records in the `runBackendStepInput` parameter.
+** This method is responsible for adding records to the `runBackendStepOutput`, which will then be passed to the *Load* step.
+** This class is also responsible for implementing the method `getProcessSummary`, which provides the data to the *Validate* screen.
+** The run method will generally update ProcessSummaryLine objects to facilitate this functionality.
+** The Transform step is called before the Preview, Validate, and Result screens, consuming all records selected by the Extract step.
+* *Load* - a subclass of `AbstractLoadStep` - is responsible for the Load function of the ETL job.
+_A quick word on terminology - this step is actually doing what we are more likely to think of as storing data - which feels like the opposite of “loading” - but we use the name Load to keep in line with the ETL naming convention…_
+** The Load step is ONLY called before the Result screen is presented (possibly after Preview, if the user chose to skip validation, otherwise, after validation).
+** Similar to the Transform step, the Load step's `run` method will be called potentially multiple times, with pages of records in its input.
+** As such, the Load step is generally the only step where data writes should occur.
+*** e.g., a Transform step should not do any writes, as it will be called when the user is going to the Preview & Validate screens - e.g., before the user confirmed that they want to execute the action!
+** A common pattern is that the Load step just needs to insert or update the list of records output by the Transform step, in which case the QQQ-provided `LoadViaInsertStep` or `LoadViaUpdateStep` can be used, but custom use-cases can be built as well.
+
+Another distinction between StreamedELTWithFrontendProcess steps and general QQQ process backend steps, is that the list of records in the input & output objects is NOT shared for StreamedELTWithFrontendProcess steps.
+The direct implication of this is, that a Transform step MUST explicitly call `output.addRecord()` for any records that it wants to pass along to the Load step.
+
+==== Example
+
+[source,java]
+.Examples of a Transform and Load step for a StreamedELTWithFrontendProcess
+----
+   // todo!
+----
+
+#todo: more details on these 3 specialized types of process steps (e.g., method to overload, when stuff like pre-action is called; how summaries work).#

docs/misc/QContext.adoc

@@ -0,0 +1,30 @@
+== QContext
+include::../variables.adoc[]
+
+The class `QContext` contains a collection of thread-local variables, to define the current context of the QQQ code that is currently running.
+For example, what `QInstance` (meta-data container) is being used, what `QSession` (user attributes) is active, etc.
+
+Most of the time, main-line application code does not need to worry about setting up the `QContext` - although unit-test code can is a common exception to that rule.
+This is because all of QQQ's entry-points into execution (e.g., web context handlers, CLI startup methods, schedulers, and multi-threaded executors) take care of initializing the context.
+
+It is more common though, for application code to need to get data from the context, such as the current session or any piece of meta-data from the QInstance.
+The methods to access data from the `QContext` are fairly straightforward:
+
+=== Examples
+==== Get a QTableMetaData from the active QInstance
+[source,java]
+----
+QTableMeataData myTable = QContext.getQInstance().getTable("myTable");
+for(QFieldMeataData field : myTable.getFields().values())
+{
+   // ...
+}
+----
+
+==== Get a security key value for the current user session
+[source,java]
+----
+QSession           session   = QContext.getQSession();
+List<Serializable> clientIds = session.getSecurityKeyValues("clientId");
+----
+

docs/misc/QRecordEntities.adoc

@@ -0,0 +1,116 @@
+== QRecordEntities
+include::../variables.adoc[]
+
+While `<<QRecords>>` provide a flexible mechanism for passing around record data in a QQQ Application, they have one big disadvantage from the point-of-view of a Java Application:
+They do not provide a mechanism to ensure compile-time checks of field names or field types.
+
+As such, an alternative mechanism exists, which allows records in a QQQ application to be worked with following a more familiar Java Bean (Entity Bean) like pattern.
+This mechanism is known as a `QRecordEntity`.
+
+Specifically speaking, `QRecordEntity` is an abstract base class, whose purpose is to be the base class for entity-bean classes.
+Using reflection, `QRecordEntity` is able to convert objects back and forth from `QRecord` to specific entity-bean subtypes.
+
+For example, the method `QRecordEntity::toQRecord()` converts a record entity object into a `QRecord`.
+
+Inversely, `QRecordEntity::populateFromQRecord(QRecord record)` sets fields in a record entity object, based on the values in the supplied `QRecord`.
+It is conventional for a subclass of `QRecordEntity` to have both a no-arg (default) constructor, and a constructor that takes a `QRecord` as a parameter, and calls `populateFromQRecord`.
+
+In addition to these constructors, a `QRecordEntity` subclass will generally contain:
+
+* A `public static final String TABLE_NAME`, used throughout the application as a constant reference to the name for the {link-table}.
+* A series of `private` fields, corresponding to the fields in the table that the entity represents.
+** If these fields are annotated as `@QField()`, then the {link-table} meta-data for the table that the entity represents can in part be inferred by QQQ, by using the method `QTableMetaData::withFieldsFromEntity`.
+* `getX()`, `setX()`, and `withX()` methods for all of the entity's fields.
+
+=== Examples
+[source,java]
+.Example Definition of a QRecordEntity subclass:  Person.java
+----
+/*******************************************************************************
+ ** QRecordEntity for the person table.
+ *******************************************************************************/
+public class Person extends QRecordEntity
+{
+   public static final String TABLE_NAME = "person";
+
+   @QField(isEditable = false)
+   private Integer id;
+
+   @QField()
+   private String firstName;
+
+   @QField()
+   private String lastName;
+
+   @QField()
+   private Integer age;
+
+   // all other fields
+
+   /*******************************************************************************
+    ** Default constructor
+    *******************************************************************************/
+   public Person()
+   {
+   }
+
+
+   /*******************************************************************************
+    ** Constructor that takes a QRecord
+    *******************************************************************************/
+   public Person(QRecord record)
+   {
+      populateFromQRecord(record);
+   }
+
+
+   /*******************************************************************************
+    ** Custom method to concatenate firstName and lastName
+    *******************************************************************************/
+   public String getFullName()
+   {
+      //////////////////////////////////////////////////////////////////////
+      // if there were more than an example, we could do some null-checks //
+      // here to avoid silly output like "Bobby null" :)                  //
+      //////////////////////////////////////////////////////////////////////
+      return (firstName + " " + lastName);
+   }
+
+   // all getter, setter, and fluent setter (wither) methods
+}
+----
+
+The core ORM actions and process-execution actions of QQQ work with `QRecords`.
+However, application engineers may want to apply patterns like the following example, to gain the compile-time safety of `QRecordEntities`:
+
+[source,java]
+.Example Usage of a QRecordEntity
+----
+//////////////////////////////////////////////////////////////////////
+// assume we're working with the "person" table & entity from above //
+//////////////////////////////////////////////////////////////////////
+List<QRecord> recordsToUpdate = new ArrayList<>();
+for(QRecord record : inputRecordList)
+{
+   /////////////////////////////////////////////////////////////////////////////
+   // call the constructor that copies values from the record into the entity //
+   /////////////////////////////////////////////////////////////////////////////
+   Person person = new Person(record);
+
+   ////////////////////////////////////////////////
+   // call a custom method defined in the entity //
+   ////////////////////////////////////////////////
+   LOG.info("Processing: " + person.getFullName());
+
+   /////////////////////////////////////////////////////////////
+   // age is an Integer, so no type-conversion is needed here //
+   /////////////////////////////////////////////////////////////
+   person.setAge(person.getAge() + 1);
+
+   ///////////////////////////////////////////////////////////////////////////
+   // to pass the updated records to UpdateAction, convert them to QRecords //
+   ///////////////////////////////////////////////////////////////////////////
+   recordsToUpdate.add(person.toQRecord());
+}
+----
+

docs/misc/QRecords.adoc

@@ -0,0 +1,68 @@
+== QRecords
+include::../variables.adoc[]
+
+Almost all code inside a QQQ application will be dealing with *Records* (aka Tuples or Rows).
+That is: a collection of named values, representing a single Entity, Fact, or, Row from a {link-table}.
+
+The class that QQQ uses to work with records is called:  `QRecord`.
+
+=== Values
+At its core, a `QRecord` is a wrapper around a `Map<String, Serializable> values`.
+These are the *actual* values for the fields in the table for the record.
+That is, direct representations of the values as they are stored in the {link-backend}.
+
+The keys in the `values` map are names from the {link-fields} in the {link-table}.
+
+The values in `values` map are declared as `Serializable` (to help ensure the serializability of the `QRecord` as a whole).
+In practice, their types will be based on the `QFieldType` of the {link-field} that they correspond to.
+That will typically be one of: `String`, `Integer`, `Boolean`, `BigDecimal`, `Instant`, `LocalDate`, `LocalTime`, or `byte[]`.
+Be aware that `null` values may be in the `values` map, especially/per if the backend/table support `null`.
+
+To work with the `values` map, the following methods are provided:
+
+* `setValue(String fieldName, Serializable value)` - Sets a value for the specified field in the record.
+** Overloaded as `setValue(String fieldName, Object value)` - For some cases where the value may not be known to be `Serializable`.
+In this overload, if the value is `null` or `Serializable`, the primary version of `setValue` is called.
+Otherwise, the `value` is passed through `String::valueOf`, and the result is stored.
+** Overloaded as `setValue(QFieldMetaData field, Serializable value)` - which simply defers to the primary version of `setValue`, passing `field.getName()` as the first parameter.
+* `removeValue(String fieldName)` - Remove the given field from the `values` map.
+** Note that in some situations this is an important distinction from having a `null` value in the map (See <<UpdateAction)>>).
+* `setValues(Map<String, Serializable> values)` - Sets the full map of `values`.
+* `getValues()` - Returns the full map of `values`.
+* `getValue(String fieldName)` - Returns the value for the named field - possibly `null` - as a `Serializable`.
+* Several type-specific variations of `getValueX(String fieldName)`, where internally, values will be not exactly type-cast, but effectively converted (if possible) to the requested type.
+These conversions are done using the `ValueUtils.getValueAsX(Object)` methods.
+These methods are generally the preferred/cleanest way to get record values in application code, when it is needed in a type-specific way .
+** `getValueString(String fieldName)`
+** `getValueInteger(String fieldName)`
+** `getValueBoolean(String fieldName)`
+** `getValueBigDecimal(String fieldName)`
+** `getValueInstant(String fieldName)`
+** `getValueLocalDate(String fieldName)`
+** `getValueLocalTime(String fieldName)`
+** `getValueByteArray(String fieldName)`
+
+=== Display Values
+In addition to the `values` map, a `QRecord` contains another map called `displayValues`, which only stores `String` values.
+That is to say, values for other types are stringified, based on their {link-field}'s type and `displayFormat` property.
+In addition, fields which have a `possibleValueSource` property will have their translated values set in the `displayValues` map.
+
+By default, a `QRecord` will not have its `displayValues` populated.
+To populate `displayValues`, the <<QueryAction>> and <<GetAction>> classes take a property in their inputs called `shouldGenerateDisplayValues`, which must be set to `true` to generate `displayValues`.
+In addition, these two actions also have a property `shouldTranslatePossibleValues` in their inputs, which needs to be set to `true` if possible-value lookups are to be performed.
+
+As an alternative to the `shouldGenerateDisplayValues` and `shouldTranslatePossibleValues` inputs to <<QueryAction>> and <<GetAction>>, one can directly call the `QValueFormatter.setDisplayValuesInRecords` and/or `qPossibleValueTranslator.translatePossibleValuesInRecords` methods.
+Or, for special cases, `setDisplayValue(String fieldName, String displayValue)` or `setDisplayValues(Map<String, String> displayValues)` can be called directly.
+
+=== Backend Details
+Sometimes a backend may want to place additional data in a `QRecord` that doesn't exactly correspond to a field.
+To do this, the `Map<String, Serializable> backendDetails` member is used.
+
+For example, an API backend may store the full JSON `String` that came from the API as a backend detail in a `QRecord`.
+Or fields that are marked as `isHeavy`, if the full (heavy) value of the field hasn't been fetched, then the lengths of any such heavy fields may be stored in `backendDetails`.
+
+=== Errors and Warnings
+#todo#
+
+=== Associated Records
+#todo#

docs/misc/ScheduledJobs.adoc

@@ -0,0 +1,141 @@
+== Schedulers and Scheduled Jobs
+include::../variables.adoc[]
+
+QQQ has the ability to automatically run various types of jobs on schedules,
+either defined in your instance's meta-data,
+or optionally via data in your application, in a `scheduledJob` table.
+
+=== Schedulers and QSchedulerMetaData
+2 types of schedulers are included in QQQ by default (though an application can define its own schedulers):
+
+* `SimpleScheduler` - is (as its name suggests) a simple class which uses java's `ScheduledExecutorService`
+to run jobs on repeating intervals.
+** Cannot run cron schedules - only repeating intervals.
+** If multiple servers are running, each will potentially run the same job concurrently
+** Has no configurations, e.g., to limit the number of threads.
+
+* `QuartzScheduler` - uses the 3rd party https://www.quartz-scheduler.org/[Quartz Scheduler] library to provide
+a much more capable, though admittedly more complex, scheduling solution.
+** Can run both cron schedules and repeating intervals.
+** By default, will not allow concurrent executions of the same job.
+** Supports multiple configurations, e.g., to limit the number of threads.
+
+An application can define its own scheduler by providing a class which implements the `QSchedulerInterface`.
+
+A `QInstance` can work with 0 or more schedulers, as defined by adding `QSchedulerMetaData` objects
+to the instance.
+
+This meta-data class is `abstract`, and is extended by the 2 built-in schedulers
+(e.g., `SimpleSchedulerMetaData` and `QuartzSchedulerMetaData`).  As such,
+these concrete subclasses are what you need to instantiate and add to your instance.
+
+To configure a QuartzScheduler, you can add a `Properties` object to the `QuartzSchedulerMetaData` object.
+See https://www.quartz-scheduler.org/documentation/[Quartz's documentation] for available configuration properties.
+
+[source,java]
+.Defining SchedulerMetaData
+----
+qInstance.addScheduler(new SimpleSchedulerMetaData().withName("mySimpleScheduler"));
+
+qInstance.addScheduler(new QuartzSchedulerMetaData()
+   .withName("myQuartzScheduler")
+   .withProperties(myQuartzProperties);
+----
+
+=== SchedulableTypes
+The types of jobs which can be scheduled in a QQQ application are defined in the `QInstance` by
+instances of the `SchedulableType` meta-data class.
+These objects contain a name, along with a `QCodeReference` to the `runner`,
+which must be a class that implements the `SchedulableRunner` interface.
+
+By default, (in the `QInstanceEnricher`), QQQ will make 3 `SchedulableType` options available:
+
+* `PROCESS` - Any Process defined in the `QInstance` can be scheduled.
+* `QUEUE_PROCESSOR` - A Queue defined in the `QInstance`, which requires polling (e.g., SQS), can be scheduled.
+* `TABLE_AUTOMATIONS` - A Table in the `QInstance`, with `AutomationDetails` referring to an
+AutomationProvider which requires polling, can be scheduled.
+
+If an application only wants to use a subset of these `SchedulableType` options,
+or to add custom `SchedulableType` options,
+the `QInstance` will need to have 1 or more `SchedulableType` objects in it before the `QInstanceEnricher` runs.
+
+=== User-defined Scheduled Jobs
+To allow users to schedule jobs (rather than using programmer-defined schedules (in meta-data)),
+you can add a set of tables to your `QInstance`, using the `ScheduledJobsMetaDataProvider` class:
+
+[source,java]
+.Adding the ScheduledJob tables and related meta-data to a QInstance
+----
+new ScheduledJobsMetaDataProvider().defineAll(
+   qInstance, backendName, table -> tableEnricher(table));
+----
+
+This meta-data provider adds a "scheduledJob" and "scheduledJobParameter" table, along with
+some PossibleValueSources.
+These tables include post-action customizers, which manage (re-, un-) scheduling jobs based on
+changes made to records in this these tables.
+
+Also, when `QScheduleManager` is started, it will query these tables,and will schedule jobs as defined therein.
+
+_You can use a mix of user-defined and meta-data defined scheduled jobs in your instance.
+However, if a ScheduledJob record references a process, queue, or table automation with a
+meta-data defined schedule, the ScheduledJob will NOT be started by ScheduleManager --
+rather, the meta-data definition will "win"._
+
+[source,sql]
+.Example of inserting scheduled jobs records directly into an SQL backend
+----
+-- A process:
+INSERT INTO scheduled_job (label, scheduler_name, cron_expression, cron_time_zone_id, repeat_seconds, type, is_active) VALUES
+   ('myProcess', 'QuartzScheduler', null, null, 300, 'PROCESS', 1);
+INSERT INTO scheduled_job_parameter (scheduled_job_id, `key`, value) VALUES
+   ((SELECT id FROM scheduled_job WHERE label = 'myProcess'), 'processName', 'myProcess');
+
+-- A table's insert & update automations:
+INSERT INTO scheduled_job (label, scheduler_name, cron_expression, cron_time_zone_id, repeat_seconds, type, is_active) VALUES
+   ('myTable.PENDING_INSERT_AUTOMATIONS', 'QuartzScheduler', null, null, 15, 'TABLE_AUTOMATIONS', 1),
+   ('myTable.PENDING_UPDATE_AUTOMATIONS', 'QuartzScheduler', null, null, 15, 'TABLE_AUTOMATIONS', 1);
+INSERT INTO scheduled_job_parameter (scheduled_job_id, `key`, value) VALUES
+   ((SELECT id FROM scheduled_job WHERE label = 'myTable.PENDING_INSERT_AUTOMATIONS'), 'tableName', 'myTable'),
+   ((SELECT id FROM scheduled_job WHERE label = 'myTable.PENDING_INSERT_AUTOMATIONS'), 'automationStatus', 'PENDING_INSERT_AUTOMATIONS'),
+   ((SELECT id FROM scheduled_job WHERE label = 'myTable.PENDING_UPDATE_AUTOMATIONS'), 'tableName', 'myTable'),
+   ((SELECT id FROM scheduled_job WHERE label = 'myTable.PENDING_UPDATE_AUTOMATIONS'), 'automationStatus', 'PENDING_UPDATE_AUTOMATIONS');
+
+-- A queue processor:
+INSERT INTO scheduled_job (label, scheduler_name, cron_expression, cron_time_zone_id, repeat_seconds, type, is_active) VALUES
+   ('mySqsQueue', 'QuartzScheduler', null, null, 60, 'QUEUE_PROCESSOR', 1);
+INSERT INTO scheduled_job_parameter (scheduled_job_id, `key`, value) VALUES
+   ((SELECT id FROM scheduled_job WHERE label = 'mySqsQueue'), 'queueName', 'mySqsQueue');
+----
+
+=== Running Scheduled Jobs
+In a server running QQQ, if you wish to start running scheduled jobs, you need to initialize
+the `QScheduleManger` singleton class, then call its `start()` method.
+
+Note that internally, this class will check for a system property of `qqq.scheduleManager.enabled`
+or an environment variable of `QQQ_SCHEDULE_MANAGER_ENABLED`, and if the first value found is
+`"false"`, then the scheduler will not actually run its jobs (but, in the case of the `QuartzSchdeuler`,
+it will be available for managing scheduled jobs).
+
+The method `QScheduleManager.initInstance` requires 2 parameters:  Your `QInstance`, and a
+`Supplier<QSession>` lambda, which returns the session that will be used for scheduled jobs when they
+are executed.
+
+[source,java]
+.Starting the Schedule Manager service
+----
+QScheduleManager.initInstance(qInstance, () -> systemUserSession).start();
+----
+
+=== Examples
+[source,java]
+.Attach a schedule in meta-data to a Process
+----
+QProcessMetaData myProcess = new QProcessMetaData()
+   // ...
+   .withSchedule(new QScheduleMetaData()
+      .withSchedulerName("myScheduler")
+      .withDescription("Run myProcess every five minutes")
+      .withRepeatSeconds(300))
+----
+

pom.xml

@@ -29,6 +29,7 @@
    <packaging>pom</packaging>
 
    <modules>
+      <module>qqq-bom</module>
       <module>qqq-backend-core</module>
       <module>qqq-backend-module-api</module>
       <module>qqq-backend-module-filesystem</module>
@@ -54,7 +55,7 @@
       <maven.compiler.showDeprecation>true</maven.compiler.showDeprecation>
       <maven.compiler.showWarnings>true</maven.compiler.showWarnings>
       <coverage.haltOnFailure>true</coverage.haltOnFailure>
-      <coverage.instructionCoveredRatioMinimum>0.80</coverage.instructionCoveredRatioMinimum>
+      <coverage.instructionCoveredRatioMinimum>0.75</coverage.instructionCoveredRatioMinimum>
       <coverage.classCoveredRatioMinimum>0.95</coverage.classCoveredRatioMinimum>
       <plugin.shade.phase>none</plugin.shade.phase>
    </properties>
@@ -245,8 +246,7 @@ echo "   See also target/site/jacoco/index.html"
 echo "   and https://www.jacoco.org/jacoco/trunk/doc/counters.html"
 echo "------------------------------------------------------------"
 
-which xpath > /dev/null 2>&1
-if [ "$?" == "0" ]; then
+if which xpath > /dev/null 2>&1; then
    echo "Element\nInstructions Missed\nInstruction Coverage\nBranches Missed\nBranch Coverage\nComplexity Missed\nComplexity Hit\nLines Missed\nLines Hit\nMethods Missed\nMethods Hit\nClasses Missed\nClasses Hit\n" > /tmp/$$.headers
    xpath -n -q -e '/html/body/table/tfoot/tr[1]/td/text()' target/site/jacoco/index.html > /tmp/$$.values
    paste /tmp/$$.headers /tmp/$$.values | tail +2 | awk -v FS='\t' '{printf("%-20s %s\n",$1,$2)}'
@@ -255,8 +255,7 @@ else
    echo "xpath is not installed.  Jacoco coverage summary will not be produced here...";
 fi
 
-which xpath > /dev/null 2>&1
-if [ "$?" == "0" ]; then
+if which html2text > /dev/null 2>&1; then
    echo "Untested classes, per Jacoco:"
    echo "-----------------------------"
    for i in target/site/jacoco/*/index.html; do
@@ -330,28 +329,28 @@ fi
 
    <!-- mvn javadoc:aggregate -->
    <reporting>
-    <plugins>
-      <plugin>
-        <groupId>org.apache.maven.plugins</groupId>
-        <artifactId>maven-javadoc-plugin</artifactId>
-        <version>3.6.2</version>
-        <reportSets>
-          <reportSet>
-            <id>aggregate</id>
-            <inherited>false</inherited>
-            <reports>
-              <report>aggregate</report>
-            </reports>
-          </reportSet>
-          <reportSet>
-            <id>default</id>
-            <reports>
-              <report>javadoc</report>
-            </reports>
-          </reportSet>
-        </reportSets>
-      </plugin>
-    </plugins>
+      <plugins>
+         <plugin>
+            <groupId>org.apache.maven.plugins</groupId>
+            <artifactId>maven-javadoc-plugin</artifactId>
+            <version>3.6.2</version>
+            <reportSets>
+               <reportSet>
+                  <id>aggregate</id>
+                  <inherited>false</inherited>
+                  <reports>
+                     <report>aggregate</report>
+                  </reports>
+               </reportSet>
+               <reportSet>
+                  <id>default</id>
+                  <reports>
+                     <report>javadoc</report>
+                  </reports>
+               </reportSet>
+            </reportSets>
+         </plugin>
+      </plugins>
    </reporting>
 
    <repositories>

qqq-backend-core/pom.xml

@@ -102,6 +102,16 @@
          <artifactId>fastexcel</artifactId>
          <version>0.12.15</version>
       </dependency>
+      <dependency>
+         <groupId>org.apache.poi</groupId>
+         <artifactId>poi</artifactId>
+         <version>5.2.5</version>
+      </dependency>
+      <dependency>
+         <groupId>org.apache.poi</groupId>
+         <artifactId>poi-ooxml</artifactId>
+         <version>5.2.5</version>
+      </dependency>
       <dependency>
          <groupId>com.auth0</groupId>
          <artifactId>auth0</artifactId>
@@ -163,6 +173,38 @@
          <version>1.12.321</version>
       </dependency>
 
+      <dependency>
+         <groupId>com.amazonaws</groupId>
+         <artifactId>aws-java-sdk-ses</artifactId>
+         <version>1.12.705</version>
+      </dependency>
+
+      <dependency>
+         <groupId>cloud.localstack</groupId>
+         <artifactId>localstack-utils</artifactId>
+         <version>0.2.20</version>
+         <scope>test</scope>
+      </dependency>
+
+      <dependency>
+         <groupId>org.quartz-scheduler</groupId>
+         <artifactId>quartz</artifactId>
+         <version>2.3.2</version>
+      </dependency>
+
+      <!-- Many of the deps we bring in use slf4j.  This dep maps slf4j to our logger, log4j -->
+      <dependency>
+         <groupId>org.apache.logging.log4j</groupId>
+         <artifactId>log4j-slf4j-impl</artifactId>
+         <version>2.23.0</version>
+      </dependency>
+
+      <dependency>
+         <groupId>com.sun.mail</groupId>
+         <artifactId>jakarta.mail</artifactId>
+         <version>2.0.1</version>
+      </dependency>
+
       <!-- Common deps for all qqq modules -->
       <dependency>
          <groupId>org.apache.maven.plugins</groupId>

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/async/AsyncJobManager.java

@@ -159,7 +159,7 @@ public class AsyncJobManager
    private <T extends Serializable> T runAsyncJob(String jobName, AsyncJob<T> asyncJob, UUIDAndTypeStateKey uuidAndTypeStateKey, AsyncJobStatus asyncJobStatus)
    {
       String originalThreadName = Thread.currentThread().getName();
-      Thread.currentThread().setName("Job:" + jobName + ":" + uuidAndTypeStateKey.getUuid().toString().substring(0, 8));
+      Thread.currentThread().setName("Job:" + jobName);
       try
       {
          LOG.debug("Starting job " + uuidAndTypeStateKey.getUuid());
@@ -169,17 +169,24 @@ public class AsyncJobManager
          LOG.debug("Completed job " + uuidAndTypeStateKey.getUuid());
          return (result);
       }
-      catch(Exception e)
+      catch(Throwable t)
       {
          asyncJobStatus.setState(AsyncJobState.ERROR);
-         asyncJobStatus.setCaughtException(e);
+         if(t instanceof Exception e)
+         {
+            asyncJobStatus.setCaughtException(e);
+         }
+         else
+         {
+            asyncJobStatus.setCaughtException(new QException("Caught throwable", t));
+         }
          getStateProvider().put(uuidAndTypeStateKey, asyncJobStatus);
 
          //////////////////////////////////////////////////////
          // if user facing, just log an info, warn otherwise //
          //////////////////////////////////////////////////////
-         LOG.log((e instanceof QUserFacingException) ? Level.INFO : Level.WARN, "Job ended with an exception", e, logPair("jobId", uuidAndTypeStateKey.getUuid()));
-         throw (new CompletionException(e));
+         LOG.log((t instanceof QUserFacingException) ? Level.INFO : Level.WARN, "Job ended with an exception", t, logPair("jobId", uuidAndTypeStateKey.getUuid()));
+         throw (new CompletionException(t));
       }
       finally
       {

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/async/AsyncJobStatus.java

@@ -31,6 +31,7 @@ import java.io.Serializable;
  *******************************************************************************/
 public class AsyncJobStatus implements Serializable
 {
+   private String        jobName;
    private AsyncJobState state;
    private String        message;
    private Integer       current;
@@ -187,4 +188,36 @@ public class AsyncJobStatus implements Serializable
    {
       this.cancelRequested = cancelRequested;
    }
+
+
+
+   /*******************************************************************************
+    ** Getter for jobName
+    *******************************************************************************/
+   public String getJobName()
+   {
+      return (this.jobName);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for jobName
+    *******************************************************************************/
+   public void setJobName(String jobName)
+   {
+      this.jobName = jobName;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for jobName
+    *******************************************************************************/
+   public AsyncJobStatus withJobName(String jobName)
+   {
+      this.jobName = jobName;
+      return (this);
+   }
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/async/NonPersistedAsyncJobCallback.java

@@ -0,0 +1,62 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.actions.async;
+
+
+import java.util.UUID;
+
+
+/*******************************************************************************
+ ** subclass designed to be used when we want there to be an instance (so code
+ ** doesn't have to all be null-tolerant), but there's no one who will ever be
+ ** reading the status data, so we don't need to store the object in a
+ ** state provider.
+ *******************************************************************************/
+public class NonPersistedAsyncJobCallback extends AsyncJobCallback
+{
+   private final AsyncJobStatus asyncJobStatus;
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public NonPersistedAsyncJobCallback(UUID jobUUID, AsyncJobStatus asyncJobStatus)
+   {
+      super(jobUUID, asyncJobStatus);
+      this.asyncJobStatus = asyncJobStatus;
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   protected void storeUpdatedStatus()
+   {
+      ///////////////////////////////////////////////////////////////////////////////////////
+      // noop - cf. base class, which writes to persistence here (our point is, we do not) //
+      ///////////////////////////////////////////////////////////////////////////////////////
+   }
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/audits/AuditAction.java

@@ -295,7 +295,7 @@ public class AuditAction extends AbstractQActionFunction<AuditInput, AuditOutput
          }
          catch(Exception e)
          {
-            LOG.error("Error performing an audit", e);
+            LOG.warn("Error performing an audit", e);
          }
       }
 

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/audits/DMLAuditAction.java

@@ -149,7 +149,7 @@ public class DMLAuditAction extends AbstractQActionFunction<DMLAuditInput, DMLAu
             // sort the field names by their labels //
             //////////////////////////////////////////
             List<String> sortedFieldNames = table.getFields().keySet().stream()
-               .sorted(Comparator.comparing(fieldName -> table.getFields().get(fieldName).getLabel()))
+               .sorted(Comparator.comparing(fieldName -> Objects.requireNonNullElse(table.getFields().get(fieldName).getLabel(), fieldName)))
                .toList();
 
             QFieldMetaData primaryKeyField = table.getField(table.getPrimaryKeyField());
@@ -187,7 +187,7 @@ public class DMLAuditAction extends AbstractQActionFunction<DMLAuditInput, DMLAu
       }
       catch(Exception e)
       {
-         LOG.error("Error performing DML audit", e, logPair("type", String.valueOf(dmlType)), logPair("table", table.getName()));
+         LOG.warn("Error performing DML audit", e, logPair("type", String.valueOf(dmlType)), logPair("table", table.getName()));
       }
 
       return (output);
@@ -303,7 +303,7 @@ public class DMLAuditAction extends AbstractQActionFunction<DMLAuditInput, DMLAu
          }
          else
          {
-            String formattedValue = getFormattedValueForAuditDetail(record, fieldName, field, value);
+            String formattedValue = getFormattedValueForAuditDetail(table, record, fieldName, field, value);
             detailRecord = new QRecord().withValue("message", "Set " + field.getLabel() + " to " + formattedValue);
             detailRecord.withValue("newValue", formattedValue);
          }
@@ -329,8 +329,8 @@ public class DMLAuditAction extends AbstractQActionFunction<DMLAuditInput, DMLAu
             }
             else
             {
-               String formattedValue    = getFormattedValueForAuditDetail(record, fieldName, field, value);
-               String formattedOldValue = getFormattedValueForAuditDetail(oldRecord, fieldName, field, oldValue);
+               String formattedValue    = getFormattedValueForAuditDetail(table, record, fieldName, field, value);
+               String formattedOldValue = getFormattedValueForAuditDetail(table, oldRecord, fieldName, field, oldValue);
 
                if(oldValue == null)
                {
@@ -464,7 +464,7 @@ public class DMLAuditAction extends AbstractQActionFunction<DMLAuditInput, DMLAu
    /*******************************************************************************
     **
     *******************************************************************************/
-   private static String getFormattedValueForAuditDetail(QRecord record, String fieldName, QFieldMetaData field, Serializable value)
+   private static String getFormattedValueForAuditDetail(QTableMetaData table, QRecord record, String fieldName, QFieldMetaData field, Serializable value)
    {
       String formattedValue = null;
       if(value != null)
@@ -479,7 +479,8 @@ public class DMLAuditAction extends AbstractQActionFunction<DMLAuditInput, DMLAu
          }
          else
          {
-            formattedValue = QValueFormatter.formatValue(field, value);
+            QValueFormatter.setDisplayValuesInRecord(table, table.getFields(), record);
+            formattedValue = record.getDisplayValue(fieldName);
          }
       }
 

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/automation/polling/PollingAutomationPerTableRunner.java

@@ -132,6 +132,11 @@ public class PollingAutomationPerTableRunner implements Runnable
        *******************************************************************************/
       String tableName();
 
+      /*******************************************************************************
+       **
+       *******************************************************************************/
+      QTableAutomationDetails tableAutomationDetails();
+
       /*******************************************************************************
        **
        *******************************************************************************/
@@ -143,7 +148,7 @@ public class PollingAutomationPerTableRunner implements Runnable
    /*******************************************************************************
     ** Wrapper for a pair of (tableName, automationStatus)
     *******************************************************************************/
-   public record TableActions(String tableName, AutomationStatus status) implements TableActionsInterface
+   public record TableActions(String tableName, QTableAutomationDetails tableAutomationDetails, AutomationStatus status) implements TableActionsInterface
    {
       /*******************************************************************************
        **
@@ -159,7 +164,7 @@ public class PollingAutomationPerTableRunner implements Runnable
     ** extended version of TableAction, for sharding use-case - adds the shard
     ** details.
     *******************************************************************************/
-   public record ShardedTableActions(String tableName, AutomationStatus status, String shardByFieldName, Serializable shardValue, String shardLabel) implements TableActionsInterface
+   public record ShardedTableActions(String tableName, QTableAutomationDetails tableAutomationDetails, AutomationStatus status, String shardByFieldName, Serializable shardValue, String shardLabel) implements TableActionsInterface
    {
       /*******************************************************************************
        **
@@ -198,8 +203,8 @@ public class PollingAutomationPerTableRunner implements Runnable
                   {
                      Serializable shardId = record.getValue(automationDetails.getShardIdFieldName());
                      String       label   = record.getValueString(automationDetails.getShardLabelFieldName());
-                     tableActionList.add(new ShardedTableActions(table.getName(), AutomationStatus.PENDING_INSERT_AUTOMATIONS, automationDetails.getShardByFieldName(), shardId, label));
-                     tableActionList.add(new ShardedTableActions(table.getName(), AutomationStatus.PENDING_UPDATE_AUTOMATIONS, automationDetails.getShardByFieldName(), shardId, label));
+                     tableActionList.add(new ShardedTableActions(table.getName(), automationDetails, AutomationStatus.PENDING_INSERT_AUTOMATIONS, automationDetails.getShardByFieldName(), shardId, label));
+                     tableActionList.add(new ShardedTableActions(table.getName(), automationDetails, AutomationStatus.PENDING_UPDATE_AUTOMATIONS, automationDetails.getShardByFieldName(), shardId, label));
                   }
                }
                catch(Exception e)
@@ -209,11 +214,11 @@ public class PollingAutomationPerTableRunner implements Runnable
             }
             else
             {
-               ///////////////////////////////////////////////////////////////////
-               // for non-sharded, we just need tabler name & automation status //
-               ///////////////////////////////////////////////////////////////////
-               tableActionList.add(new TableActions(table.getName(), AutomationStatus.PENDING_INSERT_AUTOMATIONS));
-               tableActionList.add(new TableActions(table.getName(), AutomationStatus.PENDING_UPDATE_AUTOMATIONS));
+               //////////////////////////////////////////////////////////////////
+               // for non-sharded, we just need table name & automation status //
+               //////////////////////////////////////////////////////////////////
+               tableActionList.add(new TableActions(table.getName(), automationDetails, AutomationStatus.PENDING_INSERT_AUTOMATIONS));
+               tableActionList.add(new TableActions(table.getName(), automationDetails, AutomationStatus.PENDING_UPDATE_AUTOMATIONS));
             }
          }
       }
@@ -521,7 +526,7 @@ public class PollingAutomationPerTableRunner implements Runnable
          // note - this method - will re-query the objects, so we should have confidence that their data is fresh... //
          //////////////////////////////////////////////////////////////////////////////////////////////////////////////
          List<QRecord> matchingQRecords = getRecordsMatchingActionFilter(table, records, action);
-         LOG.debug("Of the {} records that were pending automations, {} of them match the filter on the action {}", records.size(), matchingQRecords.size(), action);
+         LOG.debug("Of the [" + records.size() + "] records that were pending automations, [" + matchingQRecords.size() + "] of them match the filter on the action:" +  action);
          if(CollectionUtils.nullSafeHasContents(matchingQRecords))
          {
             LOG.debug("  Processing " + matchingQRecords.size() + " records in " + table + " for action " + action);
@@ -596,7 +601,7 @@ public class PollingAutomationPerTableRunner implements Runnable
 
    /*******************************************************************************
     ** Finally, actually run action code against a list of known matching records.
-    ** todo not commit - move to somewhere genericer
+    **
     *******************************************************************************/
    public static void applyActionToMatchingRecords(QTableMetaData table, List<QRecord> records, TableAutomationAction action) throws Exception
    {

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/AbstractPostDeleteCustomizer.java

@@ -47,12 +47,24 @@ import com.kingsrook.qqq.backend.core.model.data.QRecord;
  ** records that the delete action marked in error - the user might want to do
  ** something special with them (idk, try some other way to delete them?)
  *******************************************************************************/
-public abstract class AbstractPostDeleteCustomizer
+public abstract class AbstractPostDeleteCustomizer implements TableCustomizerInterface
 {
    protected DeleteInput deleteInput;
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public List<QRecord> postDelete(DeleteInput deleteInput, List<QRecord> records) throws QException
+   {
+      this.deleteInput = deleteInput;
+      return apply(records);
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/AbstractPostInsertCustomizer.java

@@ -42,12 +42,24 @@ import com.kingsrook.qqq.backend.core.model.data.QRecord;
  **
  ** Note that the full insertInput is available as a field in this class.
  *******************************************************************************/
-public abstract class AbstractPostInsertCustomizer
+public abstract class AbstractPostInsertCustomizer implements TableCustomizerInterface
 {
    protected InsertInput insertInput;
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public List<QRecord> postInsert(InsertInput insertInput, List<QRecord> records) throws QException
+   {
+      this.insertInput = insertInput;
+      return (apply(records));
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/AbstractPostQueryCustomizer.java

@@ -23,16 +23,29 @@ package com.kingsrook.qqq.backend.core.actions.customizers;
 
 
 import java.util.List;
-import com.kingsrook.qqq.backend.core.model.actions.AbstractTableActionInput;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.actions.tables.QueryOrGetInputInterface;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
 
 
 /*******************************************************************************
  **
  *******************************************************************************/
-public abstract class AbstractPostQueryCustomizer
+public abstract class AbstractPostQueryCustomizer implements TableCustomizerInterface
 {
-   protected AbstractTableActionInput input;
+   protected QueryOrGetInputInterface input;
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public List<QRecord> postQuery(QueryOrGetInputInterface queryInput, List<QRecord> records) throws QException
+   {
+      input = queryInput;
+      return apply(records);
+   }
 
 
 
@@ -47,7 +60,7 @@ public abstract class AbstractPostQueryCustomizer
     ** Getter for input
     **
     *******************************************************************************/
-   public AbstractTableActionInput getInput()
+   public QueryOrGetInputInterface getInput()
    {
       return (input);
    }
@@ -58,7 +71,7 @@ public abstract class AbstractPostQueryCustomizer
     ** Setter for input
     **
     *******************************************************************************/
-   public void setInput(AbstractTableActionInput input)
+   public void setInput(QueryOrGetInputInterface input)
    {
       this.input = input;
    }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/AbstractPostUpdateCustomizer.java

@@ -26,6 +26,7 @@ import java.io.Serializable;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Optional;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
@@ -48,7 +49,7 @@ import com.kingsrook.qqq.backend.core.model.data.QRecord;
  ** available (if the backend supports it) - both as a list (`getOldRecordList`)
  ** and as a memoized (by this class) map of primaryKey to record (`getOldRecordMap`).
  *******************************************************************************/
-public abstract class AbstractPostUpdateCustomizer
+public abstract class AbstractPostUpdateCustomizer implements TableCustomizerInterface
 {
    protected UpdateInput   updateInput;
    protected List<QRecord> oldRecordList;
@@ -57,6 +58,19 @@ public abstract class AbstractPostUpdateCustomizer
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public List<QRecord> postUpdate(UpdateInput updateInput, List<QRecord> records, Optional<List<QRecord>> oldRecordList) throws QException
+   {
+      this.updateInput = updateInput;
+      this.oldRecordList = oldRecordList.orElse(null);
+      return apply(records);
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/AbstractPreDeleteCustomizer.java

@@ -50,7 +50,7 @@ import com.kingsrook.qqq.backend.core.model.data.QRecord;
  ** Note that the full deleteInput is available as a field in this class.
  **
  *******************************************************************************/
-public abstract class AbstractPreDeleteCustomizer
+public abstract class AbstractPreDeleteCustomizer implements TableCustomizerInterface
 {
    protected DeleteInput deleteInput;
 
@@ -58,6 +58,19 @@ public abstract class AbstractPreDeleteCustomizer
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public List<QRecord> preDelete(DeleteInput deleteInput, List<QRecord> records, boolean isPreview) throws QException
+   {
+      this.deleteInput = deleteInput;
+      this.isPreview = isPreview;
+      return apply(records);
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/AbstractPreInsertCustomizer.java

@@ -47,7 +47,7 @@ import com.kingsrook.qqq.backend.core.model.data.QRecord;
  **
  ** Note that the full insertInput is available as a field in this class.
  *******************************************************************************/
-public abstract class AbstractPreInsertCustomizer
+public abstract class AbstractPreInsertCustomizer implements TableCustomizerInterface
 {
    protected InsertInput insertInput;
 
@@ -70,6 +70,30 @@ public abstract class AbstractPreInsertCustomizer
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public List<QRecord> preInsert(InsertInput insertInput, List<QRecord> records, boolean isPreview) throws QException
+   {
+      this.insertInput = insertInput;
+      this.isPreview = isPreview;
+      return (apply(records));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public WhenToRun whenToRunPreInsert(InsertInput insertInput, boolean isPreview)
+   {
+      return getWhenToRun();
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/AbstractPreUpdateCustomizer.java

@@ -26,6 +26,7 @@ import java.io.Serializable;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Optional;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
@@ -53,7 +54,7 @@ import com.kingsrook.qqq.backend.core.model.data.QRecord;
  ** available (if the backend supports it) - both as a list (`getOldRecordList`)
  ** and as a memoized (by this class) map of primaryKey to record (`getOldRecordMap`).
  *******************************************************************************/
-public abstract class AbstractPreUpdateCustomizer
+public abstract class AbstractPreUpdateCustomizer implements TableCustomizerInterface
 {
    protected UpdateInput   updateInput;
    protected List<QRecord> oldRecordList;
@@ -63,6 +64,20 @@ public abstract class AbstractPreUpdateCustomizer
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public List<QRecord> preUpdate(UpdateInput updateInput, List<QRecord> records, boolean isPreview, Optional<List<QRecord>> oldRecordList) throws QException
+   {
+      this.updateInput = updateInput;
+      this.isPreview = isPreview;
+      this.oldRecordList = oldRecordList.orElse(null);
+      return apply(records);
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/QCodeLoader.java

@@ -22,6 +22,7 @@
 package com.kingsrook.qqq.backend.core.actions.customizers;
 
 
+import java.lang.reflect.Constructor;
 import java.util.Optional;
 import java.util.function.Function;
 import com.kingsrook.qqq.backend.core.actions.automation.RecordAutomationHandler;
@@ -34,42 +35,35 @@ import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeType;
 import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSource;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.TableAutomationAction;
+import com.kingsrook.qqq.backend.core.utils.lambdas.UnsafeFunction;
+import com.kingsrook.qqq.backend.core.utils.memoization.Memoization;
 import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
 
 
 /*******************************************************************************
  ** Utility to load code for running QQQ customizers.
+ **
+ ** TODO - redo all to go through method that memoizes class & constructor
+ ** lookup.  That memoziation causes 1,000,000 such calls to go from ~500ms
+ ** to ~100ms.
  *******************************************************************************/
 public class QCodeLoader
 {
    private static final QLogger LOG = QLogger.getLogger(QCodeLoader.class);
 
-
-
-   /*******************************************************************************
-    **
-    *******************************************************************************/
-   public static <T, R> Optional<Function<T, R>> getTableCustomizerFunction(QTableMetaData table, String customizerName)
-   {
-      Optional<QCodeReference> codeReference = table.getCustomizer(customizerName);
-      if(codeReference.isPresent())
-      {
-         return (Optional.ofNullable(QCodeLoader.getFunction(codeReference.get())));
-      }
-      return (Optional.empty());
-   }
+   private static Memoization<String, Constructor<?>> constructorMemoization = new Memoization<>();
 
 
 
    /*******************************************************************************
     **
     *******************************************************************************/
-   public static <T> Optional<T> getTableCustomizer(Class<T> expectedClass, QTableMetaData table, String customizerName)
+   public static Optional<TableCustomizerInterface> getTableCustomizer(QTableMetaData table, String customizerName)
    {
       Optional<QCodeReference> codeReference = table.getCustomizer(customizerName);
       if(codeReference.isPresent())
       {
-         return (Optional.ofNullable(QCodeLoader.getAdHoc(expectedClass, codeReference.get())));
+         return (Optional.ofNullable(QCodeLoader.getAdHoc(TableCustomizerInterface.class, codeReference.get())));
       }
       return (Optional.empty());
    }
@@ -102,7 +96,7 @@ public class QCodeLoader
       }
       catch(Exception e)
       {
-         LOG.error("Error initializing customizer", logPair("codeReference", codeReference), e);
+         LOG.error("Error initializing customizer", e, logPair("codeReference", codeReference));
 
          //////////////////////////////////////////////////////////////////////////////////////////////////////////
          // return null here - under the assumption that during normal run-time operations, we'll never hit here //
@@ -141,7 +135,7 @@ public class QCodeLoader
       }
       catch(Exception e)
       {
-         LOG.error("Error initializing customizer", logPair("codeReference", codeReference), e);
+         LOG.error("Error initializing customizer", e, logPair("codeReference", codeReference));
 
          //////////////////////////////////////////////////////////////////////////////////////////////////////////
          // return null here - under the assumption that during normal run-time operations, we'll never hit here //
@@ -175,12 +169,25 @@ public class QCodeLoader
 
       try
       {
-         Class<?> customizerClass = Class.forName(codeReference.getName());
-         return ((T) customizerClass.getConstructor().newInstance());
+         Optional<Constructor<?>> constructor = constructorMemoization.getResultThrowing(codeReference.getName(), (UnsafeFunction<String, Constructor<?>, Exception>) s ->
+         {
+            Class<?> customizerClass = Class.forName(codeReference.getName());
+            return customizerClass.getConstructor();
+         });
+
+         if(constructor.isPresent())
+         {
+            return ((T) constructor.get().newInstance());
+         }
+         else
+         {
+            LOG.error("Could not get constructor for code reference", logPair("codeReference", codeReference));
+            return (null);
+         }
       }
       catch(Exception e)
       {
-         LOG.error("Error initializing customizer", logPair("codeReference", codeReference), e);
+         LOG.error("Error initializing customizer", e, logPair("codeReference", codeReference));
 
          //////////////////////////////////////////////////////////////////////////////////////////////////////////
          // return null here - under the assumption that during normal run-time operations, we'll never hit here //

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/TableCustomizerInterface.java

@@ -0,0 +1,202 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.actions.customizers;
+
+
+import java.util.List;
+import java.util.Optional;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.logging.QLogger;
+import com.kingsrook.qqq.backend.core.model.actions.tables.QueryOrGetInputInterface;
+import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
+
+
+/*******************************************************************************
+ ** Common interface used by all (core) TableCustomizer types (e.g., post-query,
+ ** and {pre,post}-{insert,update,delete}.
+ **
+ ** Note that the abstract-base classes for each action still exist, though have
+ ** been back-ported to be implementors of this interface.  The action classes
+ ** will now expect this type, and call this type's methods.
+ **
+ *******************************************************************************/
+public interface TableCustomizerInterface
+{
+   QLogger LOG = QLogger.getLogger(TableCustomizerInterface.class);
+
+
+   /*******************************************************************************
+    ** custom actions to run after a query (or get!) takes place.
+    **
+    *******************************************************************************/
+   default List<QRecord> postQuery(QueryOrGetInputInterface queryInput, List<QRecord> records) throws QException
+   {
+      LOG.info("A default implementation of postQuery is running...  Probably not expected!", logPair("tableName", queryInput.getTableName()));
+      return (records);
+   }
+
+
+   /*******************************************************************************
+    ** custom actions before an insert takes place.
+    **
+    ** It's important for implementations to be aware of the isPreview field, which
+    ** is set to true when the code is running to give users advice, e.g., on a review
+    ** screen - vs. being false when the action is ACTUALLY happening.  So, if you're doing
+    ** things like storing data, you don't want to do that if isPreview is true!!
+    **
+    ** General implementation would be, to iterate over the records (the inputs to
+    ** the insert action), and look at their values:
+    ** - possibly adding Errors (`addError`) or Warnings (`addWarning`) to the records
+    ** - possibly manipulating values (`setValue`)
+    ** - possibly throwing an exception - if you really don't want the insert operation to continue.
+    ** - doing "whatever else" you may want to do.
+    ** - returning the list of records (can be the input list) that you want to go on to the backend implementation class.
+    *******************************************************************************/
+   default List<QRecord> preInsert(InsertInput insertInput, List<QRecord> records, boolean isPreview) throws QException
+   {
+      LOG.info("A default implementation of preInsert is running...  Probably not expected!", logPair("tableName", insertInput.getTableName()));
+      return (records);
+   }
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   default AbstractPreInsertCustomizer.WhenToRun whenToRunPreInsert(InsertInput insertInput, boolean isPreview)
+   {
+      return (AbstractPreInsertCustomizer.WhenToRun.AFTER_ALL_VALIDATIONS);
+   }
+
+
+   /*******************************************************************************
+    ** custom actions after an insert takes place.
+    **
+    ** General implementation would be, to iterate over the records (the outputs of
+    ** the insert action), and look at their values:
+    ** - possibly adding Errors (`addError`) or Warnings (`addWarning`) to the records
+    ** - possibly throwing an exception - though doing so won't stop the update, and instead
+    **   will just set a warning on all of the updated records...
+    ** - doing "whatever else" you may want to do.
+    ** - returning the list of records (can be the input list) that you want to go back to the caller.
+    *******************************************************************************/
+   default List<QRecord> postInsert(InsertInput insertInput, List<QRecord> records) throws QException
+   {
+      LOG.info("A default implementation of postInsert is running...  Probably not expected!", logPair("tableName", insertInput.getTableName()));
+      return (records);
+   }
+
+
+   /*******************************************************************************
+    ** custom actions before an update takes place.
+    **
+    ** It's important for implementations to be aware of the isPreview field, which
+    ** is set to true when the code is running to give users advice, e.g., on a review
+    ** screen - vs. being false when the action is ACTUALLY happening.  So, if you're doing
+    ** things like storing data, you don't want to do that if isPreview is true!!
+    **
+    ** General implementation would be, to iterate over the records (the inputs to
+    ** the update action), and look at their values:
+    ** - possibly adding Errors (`addError`) or Warnings (`addWarning`) to the records
+    ** - possibly manipulating values (`setValue`)
+    ** - possibly throwing an exception - if you really don't want the update operation to continue.
+    ** - doing "whatever else" you may want to do.
+    ** - returning the list of records (can be the input list) that you want to go on to the backend implementation class.
+    **
+    ** Note, "old records" (e.g., with values freshly fetched from the backend) will be
+    ** available (if the backend supports it)
+    *******************************************************************************/
+   default List<QRecord> preUpdate(UpdateInput updateInput, List<QRecord> records, boolean isPreview, Optional<List<QRecord>> oldRecordList) throws QException
+   {
+      LOG.info("A default implementation of preUpdate is running...  Probably not expected!", logPair("tableName", updateInput.getTableName()));
+      return (records);
+   }
+
+
+   /*******************************************************************************
+    ** custom actions after an update takes place.
+    **
+    ** General implementation would be, to iterate over the records (the outputs of
+    ** the update action), and look at their values:
+    ** - possibly adding Errors (`addError`) or Warnings (`addWarning`) to the records?
+    ** - possibly throwing an exception - though doing so won't stop the update, and instead
+    **   will just set a warning on all of the updated records...
+    ** - doing "whatever else" you may want to do.
+    ** - returning the list of records (can be the input list) that you want to go back to the caller.
+    **
+    ** Note, "old records" (e.g., with values freshly fetched from the backend) will be
+    ** available (if the backend supports it).
+    *******************************************************************************/
+   default List<QRecord> postUpdate(UpdateInput updateInput, List<QRecord> records, Optional<List<QRecord>> oldRecordList) throws QException
+   {
+      LOG.info("A default implementation of postUpdate is running...  Probably not expected!", logPair("tableName", updateInput.getTableName()));
+      return (records);
+   }
+
+
+   /*******************************************************************************
+    ** Custom actions before a delete takes place.
+    **
+    ** It's important for implementations to be aware of the isPreview param, which
+    ** is set to true when the code is running to give users advice, e.g., on a review
+    ** screen - vs. being false when the action is ACTUALLY happening.  So, if you're doing
+    ** things like storing data, you don't want to do that if isPreview is true!!
+    **
+    ** General implementation would be, to iterate over the records (which the DeleteAction
+    ** would look up based on the inputs to the delete action), and look at their values:
+    ** - possibly adding Errors (`addError`) or Warnings (`addWarning`) to the records
+    ** - possibly throwing an exception - if you really don't want the delete operation to continue.
+    ** - doing "whatever else" you may want to do.
+    ** - returning the list of records (can be the input list) - this is how errors
+    **   and warnings are propagated to the DeleteAction.  Note that any records with
+    **   an error will NOT proceed to the backend's delete interface - but those with
+    **   warnings will.
+    *******************************************************************************/
+   default List<QRecord> preDelete(DeleteInput deleteInput, List<QRecord> records, boolean isPreview) throws QException
+   {
+      LOG.info("A default implementation of preDelete is running...  Probably not expected!", logPair("tableName", deleteInput.getTableName()));
+      return (records);
+   }
+
+
+   /*******************************************************************************
+    ** Custom actions after a delete takes place.
+    **
+    ** General implementation would be, to iterate over the records (ones which didn't
+    ** have a delete error), and look at their values:
+    ** - possibly adding Errors (`addError`) or Warnings (`addWarning`) to the records?
+    ** - possibly throwing an exception - though doing so won't stop the delete, and instead
+    **   will just set a warning on all of the deleted records...
+    ** - doing "whatever else" you may want to do.
+    ** - returning the list of records (can be the input list) that you want to go back
+    **   to the caller - this is how errors and warnings are propagated .
+    *******************************************************************************/
+   default List<QRecord> postDelete(DeleteInput deleteInput, List<QRecord> records) throws QException
+   {
+      LOG.info("A default implementation of postDelete is running...  Probably not expected!", logPair("tableName", deleteInput.getTableName()));
+      return (records);
+   }
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/TableCustomizers.java

@@ -29,13 +29,13 @@ package com.kingsrook.qqq.backend.core.actions.customizers;
  *******************************************************************************/
 public enum TableCustomizers
 {
-   POST_QUERY_RECORD("postQueryRecord", AbstractPostQueryCustomizer.class),
-   PRE_INSERT_RECORD("preInsertRecord", AbstractPreInsertCustomizer.class),
-   POST_INSERT_RECORD("postInsertRecord", AbstractPostInsertCustomizer.class),
-   PRE_UPDATE_RECORD("preUpdateRecord", AbstractPreUpdateCustomizer.class),
-   POST_UPDATE_RECORD("postUpdateRecord", AbstractPostUpdateCustomizer.class),
-   PRE_DELETE_RECORD("preDeleteRecord", AbstractPreDeleteCustomizer.class),
-   POST_DELETE_RECORD("postDeleteRecord", AbstractPostDeleteCustomizer.class);
+   POST_QUERY_RECORD("postQueryRecord", TableCustomizerInterface.class),
+   PRE_INSERT_RECORD("preInsertRecord", TableCustomizerInterface.class),
+   POST_INSERT_RECORD("postInsertRecord", TableCustomizerInterface.class),
+   PRE_UPDATE_RECORD("preUpdateRecord", TableCustomizerInterface.class),
+   POST_UPDATE_RECORD("postUpdateRecord", TableCustomizerInterface.class),
+   PRE_DELETE_RECORD("preDeleteRecord", TableCustomizerInterface.class),
+   POST_DELETE_RECORD("postDeleteRecord", TableCustomizerInterface.class);
 
 
    private final String   role;

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/dashboard/widgets/AbstractWidgetRenderer.java

@@ -42,6 +42,7 @@ import com.kingsrook.qqq.backend.core.model.actions.widgets.RenderWidgetOutput;
 import com.kingsrook.qqq.backend.core.model.dashboard.widgets.QWidgetData;
 import com.kingsrook.qqq.backend.core.model.metadata.dashboard.QWidgetMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.dashboard.WidgetDropdownData;
+import com.kingsrook.qqq.backend.core.model.metadata.dashboard.WidgetDropdownType;
 import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValue;
 import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSource;
 import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
@@ -72,80 +73,104 @@ public abstract class AbstractWidgetRenderer
     *******************************************************************************/
    protected boolean setupDropdowns(RenderWidgetInput input, QWidgetMetaData metaData, QWidgetData widgetData) throws QException
    {
-      List<List<Map<String, String>>> pvsData                   = new ArrayList<>();
-      List<String>                    pvsLabels                 = new ArrayList<>();
-      List<String>                    pvsNames                  = new ArrayList<>();
+      List<List<Map<String, String>>> dataList                  = new ArrayList<>();
+      List<String>                    labelList                 = new ArrayList<>();
+      List<String>                    nameList                  = new ArrayList<>();
       List<String>                    missingRequiredSelections = new ArrayList<>();
       for(WidgetDropdownData dropdownData : CollectionUtils.nonNullList(metaData.getDropdowns()))
       {
-         String               possibleValueSourceName = dropdownData.getPossibleValueSourceName();
-         QPossibleValueSource possibleValueSource     = input.getInstance().getPossibleValueSource(possibleValueSourceName);
-
-         /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-         // this looks complicated, but is just look for a label in the dropdown data and if found use it,                                                                                  //
-         // otherwise look for label in PVS and if found use that, otherwise just use the PVS name                                                                                          //
-         /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-         String pvsLabel = dropdownData.getLabel() != null ? dropdownData.getLabel() : (possibleValueSource.getLabel() != null ? possibleValueSource.getLabel() : possibleValueSourceName);
-         pvsLabels.add(pvsLabel);
-         pvsNames.add(possibleValueSourceName);
-
-         SearchPossibleValueSourceInput pvsInput = new SearchPossibleValueSourceInput();
-         pvsInput.setPossibleValueSourceName(possibleValueSourceName);
-
-         if(dropdownData.getForeignKeyFieldName() != null)
+         if(WidgetDropdownType.DATE_PICKER.equals(dropdownData.getType()))
          {
-            ////////////////////////////////////////
-            // look for an id in the query params //
-            ////////////////////////////////////////
-            Integer id = null;
-            if(input.getQueryParams() != null && input.getQueryParams().containsKey("id") && StringUtils.hasContent(input.getQueryParams().get("id")))
+            String name = dropdownData.getName();
+            nameList.add(name);
+            labelList.add(dropdownData.getLabel());
+            dataList.add(new ArrayList<>());
+
+            ////////////////////////////////////////////////////////////////////////////////////////////////////////////
+            // sure that something has been selected, and if not, display a message that a selection needs made       //
+            ////////////////////////////////////////////////////////////////////////////////////////////////////////////
+            if(dropdownData.getIsRequired())
             {
-               id = Integer.parseInt(input.getQueryParams().get("id"));
-            }
-            if(id != null)
-            {
-               pvsInput.setDefaultQueryFilter(new QQueryFilter().withCriteria(
-                  new QFilterCriteria(
-                     dropdownData.getForeignKeyFieldName(),
-                     QCriteriaOperator.EQUALS,
-                     id)));
+               if(!input.getQueryParams().containsKey(name) || !StringUtils.hasContent(input.getQueryParams().get(name)))
+               {
+                  missingRequiredSelections.add(dropdownData.getLabel());
+               }
             }
          }
-
-         SearchPossibleValueSourceOutput output = new SearchPossibleValueSourceAction().execute(pvsInput);
-
-         List<Map<String, String>> dropdownOptionList = new ArrayList<>();
-         pvsData.add(dropdownOptionList);
-
-         //////////////////////////////////////////
-         // sort results, dedupe, and add to map //
-         //////////////////////////////////////////
-         Set<String> exists = new HashSet<>();
-         output.getResults().removeIf(pvs -> !exists.add(pvs.getLabel()));
-         for(QPossibleValue<?> possibleValue : output.getResults())
+         else
          {
-            dropdownOptionList.add(MapBuilder.of(
-               "id", String.valueOf(possibleValue.getId()),
-               "label", possibleValue.getLabel()
-            ));
-         }
-
-         ////////////////////////////////////////////////////////////////////////////////////////////////////////////
-         // because we know the dropdowns and what the field names will be when something is selected, we can make //
-         // sure that something has been selected, and if not, display a message that a selection needs made       //
-         ////////////////////////////////////////////////////////////////////////////////////////////////////////////
-         if(dropdownData.getIsRequired())
-         {
-            if(!input.getQueryParams().containsKey(possibleValueSourceName) || !StringUtils.hasContent(input.getQueryParams().get(possibleValueSourceName)))
+            String possibleValueSourceName = dropdownData.getPossibleValueSourceName();
+            if(possibleValueSourceName != null)
             {
-               missingRequiredSelections.add(pvsLabel);
+               QPossibleValueSource possibleValueSource = input.getInstance().getPossibleValueSource(possibleValueSourceName);
+
+               /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+               // this looks complicated, but is just look for a label in the dropdown data and if found use it,                                                                                  //
+               // otherwise look for label in PVS and if found use that, otherwise just use the PVS name                                                                                          //
+               /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+               String pvsLabel = dropdownData.getLabel() != null ? dropdownData.getLabel() : (possibleValueSource.getLabel() != null ? possibleValueSource.getLabel() : possibleValueSourceName);
+               labelList.add(pvsLabel);
+               nameList.add(possibleValueSourceName);
+
+               SearchPossibleValueSourceInput pvsInput = new SearchPossibleValueSourceInput();
+               pvsInput.setPossibleValueSourceName(possibleValueSourceName);
+
+               if(dropdownData.getForeignKeyFieldName() != null)
+               {
+                  ////////////////////////////////////////
+                  // look for an id in the query params //
+                  ////////////////////////////////////////
+                  Integer id = null;
+                  if(input.getQueryParams() != null && input.getQueryParams().containsKey("id") && StringUtils.hasContent(input.getQueryParams().get("id")))
+                  {
+                     id = Integer.parseInt(input.getQueryParams().get("id"));
+                  }
+                  if(id != null)
+                  {
+                     pvsInput.setDefaultQueryFilter(new QQueryFilter().withCriteria(
+                        new QFilterCriteria(
+                           dropdownData.getForeignKeyFieldName(),
+                           QCriteriaOperator.EQUALS,
+                           id)));
+                  }
+               }
+
+               SearchPossibleValueSourceOutput output = new SearchPossibleValueSourceAction().execute(pvsInput);
+
+               List<Map<String, String>> dropdownOptionList = new ArrayList<>();
+               dataList.add(dropdownOptionList);
+
+               //////////////////////////////////////////
+               // sort results, dedupe, and add to map //
+               //////////////////////////////////////////
+               Set<String> exists = new HashSet<>();
+               output.getResults().removeIf(pvs -> !exists.add(pvs.getLabel()));
+               for(QPossibleValue<?> possibleValue : output.getResults())
+               {
+                  dropdownOptionList.add(MapBuilder.of(
+                     "id", String.valueOf(possibleValue.getId()),
+                     "label", possibleValue.getLabel()
+                  ));
+               }
+
+               ////////////////////////////////////////////////////////////////////////////////////////////////////////////
+               // because we know the dropdowns and what the field names will be when something is selected, we can make //
+               // sure that something has been selected, and if not, display a message that a selection needs made       //
+               ////////////////////////////////////////////////////////////////////////////////////////////////////////////
+               if(dropdownData.getIsRequired())
+               {
+                  if(!input.getQueryParams().containsKey(possibleValueSourceName) || !StringUtils.hasContent(input.getQueryParams().get(possibleValueSourceName)))
+                  {
+                     missingRequiredSelections.add(pvsLabel);
+                  }
+               }
             }
          }
       }
 
-      widgetData.setDropdownNameList(pvsNames);
-      widgetData.setDropdownLabelList(pvsLabels);
-      widgetData.setDropdownDataList(pvsData);
+      widgetData.setDropdownNameList(nameList);
+      widgetData.setDropdownLabelList(labelList);
+      widgetData.setDropdownDataList(dataList);
 
       ////////////////////////////////////////////////////////////////////////////////
       // if there are any missing required dropdowns, build up a message to display //

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/dashboard/widgets/Aggregate2DTableWidgetRenderer.java

@@ -0,0 +1,202 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.actions.dashboard.widgets;
+
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.LinkedHashMap;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import com.kingsrook.qqq.backend.core.actions.tables.AggregateAction;
+import com.kingsrook.qqq.backend.core.context.QContext;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.logging.QLogger;
+import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.Aggregate;
+import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateOperator;
+import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateOutput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateResult;
+import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.GroupBy;
+import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.QFilterOrderByAggregate;
+import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.QFilterOrderByGroupBy;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
+import com.kingsrook.qqq.backend.core.model.actions.widgets.RenderWidgetInput;
+import com.kingsrook.qqq.backend.core.model.actions.widgets.RenderWidgetOutput;
+import com.kingsrook.qqq.backend.core.model.dashboard.widgets.TableData;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+import com.kingsrook.qqq.backend.core.utils.StringUtils;
+import com.kingsrook.qqq.backend.core.utils.ValueUtils;
+
+
+/*******************************************************************************
+ ** Generic widget that does an aggregate query, and presents its results
+ ** as a table, using group-by values as both row & column labels.
+ *******************************************************************************/
+public class Aggregate2DTableWidgetRenderer extends AbstractWidgetRenderer
+{
+   private static final QLogger LOG = QLogger.getLogger(Aggregate2DTableWidgetRenderer.class);
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public RenderWidgetOutput render(RenderWidgetInput input) throws QException
+   {
+      Map<String, Serializable> values = input.getWidgetMetaData().getDefaultValues();
+
+      String         tableName   = ValueUtils.getValueAsString(values.get("tableName"));
+      String         valueField  = ValueUtils.getValueAsString(values.get("valueField"));
+      String         rowField    = ValueUtils.getValueAsString(values.get("rowField"));
+      String         columnField = ValueUtils.getValueAsString(values.get("columnField"));
+      QTableMetaData table       = QContext.getQInstance().getTable(tableName);
+
+      AggregateInput aggregateInput = new AggregateInput();
+      aggregateInput.setTableName(tableName);
+
+      // todo - allow input of "list of columns" (e.g., in case some miss sometimes, or as a version of filter)
+      // todo - max rows, max cols?
+
+      // todo - from input map
+      QQueryFilter filter = new QQueryFilter();
+      aggregateInput.setFilter(filter);
+
+      Aggregate aggregate = new Aggregate(valueField, AggregateOperator.COUNT);
+      aggregateInput.withAggregate(aggregate);
+
+      GroupBy rowGroupBy    = new GroupBy(table.getField(rowField));
+      GroupBy columnGroupBy = new GroupBy(table.getField(columnField));
+      aggregateInput.withGroupBy(rowGroupBy);
+      aggregateInput.withGroupBy(columnGroupBy);
+
+      String orderBys = ValueUtils.getValueAsString(values.get("orderBys"));
+      if(StringUtils.hasContent(orderBys))
+      {
+         for(String orderBy : orderBys.split(","))
+         {
+            switch(orderBy)
+            {
+               case "row" -> filter.addOrderBy(new QFilterOrderByGroupBy(rowGroupBy));
+               case "column" -> filter.addOrderBy(new QFilterOrderByGroupBy(columnGroupBy));
+               case "value" -> filter.addOrderBy(new QFilterOrderByAggregate(aggregate));
+               default -> LOG.warn("Unrecognized orderBy: " + orderBy);
+            }
+         }
+      }
+
+      AggregateOutput aggregateOutput = new AggregateAction().execute(aggregateInput);
+
+      Map<Serializable, Map<Serializable, Serializable>> data       = new LinkedHashMap<>();
+      Set<Serializable>                                  columnsSet = new LinkedHashSet<>();
+
+      for(AggregateResult result : aggregateOutput.getResults())
+      {
+         Serializable column = result.getGroupByValue(columnGroupBy);
+         Serializable row    = result.getGroupByValue(rowGroupBy);
+         Serializable value  = result.getAggregateValue(aggregate);
+
+         Map<Serializable, Serializable> rowMap = data.computeIfAbsent(row, (k) -> new LinkedHashMap<>());
+         rowMap.put(column, value);
+         columnsSet.add(column);
+      }
+
+      // todo - possible values from rows, cols
+
+      ////////////////////////////////////
+      // setup datastructures for table //
+      ////////////////////////////////////
+      List<Map<String, Object>> tableRows    = new ArrayList<>();
+      List<TableData.Column>    tableColumns = new ArrayList<>();
+      tableColumns.add(new TableData.Column("default", table.getField(rowField).getLabel(), "_row", "2fr", "left"));
+
+      for(Serializable column : columnsSet)
+      {
+         tableColumns.add(new TableData.Column("default", String.valueOf(column) /* todo display value */, String.valueOf(column), "1fr", "right"));
+      }
+
+      tableColumns.add(new TableData.Column("default", "Total", "_total", "1fr", "right"));
+
+      TableData tableData = new TableData(null, tableColumns, tableRows)
+         .withRowsPerPage(100)
+         .withFixedStickyLastRow(false)
+         .withHidePaginationDropdown(true);
+
+      Map<Serializable, Integer> columnSums = new HashMap<>();
+      int                        grandTotal = 0;
+      for(Map.Entry<Serializable, Map<Serializable, Serializable>> rowEntry : data.entrySet())
+      {
+         Map<String, Object> rowMap = new HashMap<>();
+         tableRows.add(rowMap);
+
+         rowMap.put("_row", rowEntry.getKey() /* todo display value */);
+         int rowTotal = 0;
+         for(Serializable column : columnsSet)
+         {
+            Serializable value = rowEntry.getValue().get(column);
+            if(value == null)
+            {
+               value = 0; // todo?
+            }
+
+            Integer valueAsInteger = Objects.requireNonNullElse(ValueUtils.getValueAsInteger(value), 0);
+            rowTotal += valueAsInteger;
+            columnSums.putIfAbsent(column, 0);
+            columnSums.put(column, columnSums.get(column) + valueAsInteger);
+
+            rowMap.put(String.valueOf(column), value); // todo format commas?
+         }
+
+         rowMap.put("_total", rowTotal);
+         grandTotal += rowTotal;
+      }
+
+      ///////////////
+      // total row //
+      ///////////////
+      Map<String, Object> totalRowMap = new HashMap<>();
+      tableRows.add(totalRowMap);
+
+      totalRowMap.put("_row", "Total");
+      int rowTotal = 0;
+      for(Serializable column : columnsSet)
+      {
+         Serializable value = columnSums.get(column);
+         if(value == null)
+         {
+            value = 0; // todo?
+         }
+
+         totalRowMap.put(String.valueOf(column), value); // todo format commas?
+      }
+
+      totalRowMap.put("_total", grandTotal);
+
+      return (new RenderWidgetOutput(tableData));
+   }
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/dashboard/widgets/ChildRecordListRenderer.java

@@ -36,6 +36,7 @@ import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
 import com.kingsrook.qqq.backend.core.actions.values.QValueFormatter;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.exceptions.QNotFoundException;
+import com.kingsrook.qqq.backend.core.logging.QLogger;
 import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetOutput;
@@ -59,6 +60,7 @@ import com.kingsrook.qqq.backend.core.utils.JsonUtils;
 import com.kingsrook.qqq.backend.core.utils.StringUtils;
 import com.kingsrook.qqq.backend.core.utils.ValueUtils;
 import org.apache.commons.lang.BooleanUtils;
+import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
 
 
 /*******************************************************************************
@@ -66,6 +68,9 @@ import org.apache.commons.lang.BooleanUtils;
  *******************************************************************************/
 public class ChildRecordListRenderer extends AbstractWidgetRenderer
 {
+   private static final QLogger LOG = QLogger.getLogger(ChildRecordListRenderer.class);
+
+
 
    /*******************************************************************************
     **
@@ -137,7 +142,7 @@ public class ChildRecordListRenderer extends AbstractWidgetRenderer
        *******************************************************************************/
       public Builder withCanAddChildRecord(boolean b)
       {
-         widgetMetaData.withDefaultValue("canAddChildRecord", true);
+         widgetMetaData.withDefaultValue("canAddChildRecord", b);
          return (this);
       }
 
@@ -151,6 +156,17 @@ public class ChildRecordListRenderer extends AbstractWidgetRenderer
          widgetMetaData.withDefaultValue("disabledFieldsForNewChildRecords", new HashSet<>(disabledFieldsForNewChildRecords));
          return (this);
       }
+
+
+
+      /*******************************************************************************
+       **
+       *******************************************************************************/
+      public Builder withManageAssociationName(String manageAssociationName)
+      {
+         widgetMetaData.withDefaultValue("manageAssociationName", manageAssociationName);
+         return (this);
+      }
    }
 
 
@@ -161,98 +177,134 @@ public class ChildRecordListRenderer extends AbstractWidgetRenderer
    @Override
    public RenderWidgetOutput render(RenderWidgetInput input) throws QException
    {
-      String         widgetLabel = input.getQueryParams().get("widgetLabel");
-      String         joinName    = input.getQueryParams().get("joinName");
-      QJoinMetaData  join        = input.getInstance().getJoin(joinName);
-      String         id          = input.getQueryParams().get("id");
-      QTableMetaData leftTable   = input.getInstance().getTable(join.getLeftTable());
-      QTableMetaData rightTable  = input.getInstance().getTable(join.getRightTable());
-
-      Integer maxRows = null;
-      if(StringUtils.hasContent(input.getQueryParams().get("maxRows")))
+      try
       {
-         maxRows = ValueUtils.getValueAsInteger(input.getQueryParams().get("maxRows"));
-      }
-      else if(input.getWidgetMetaData().getDefaultValues().containsKey("maxRows"))
-      {
-         maxRows = ValueUtils.getValueAsInteger(input.getWidgetMetaData().getDefaultValues().containsKey("maxRows"));
-      }
+         String         widgetLabel = input.getQueryParams().get("widgetLabel");
+         String         joinName    = input.getQueryParams().get("joinName");
+         QJoinMetaData  join        = input.getInstance().getJoin(joinName);
+         String         id          = input.getQueryParams().get("id");
+         QTableMetaData leftTable   = input.getInstance().getTable(join.getLeftTable());
+         QTableMetaData rightTable  = input.getInstance().getTable(join.getRightTable());
 
-      ////////////////////////////////////////////////////////
-      // fetch the record that we're getting children for.  //
-      // e.g., the left-side of the join, with the input id //
-      ////////////////////////////////////////////////////////
-      GetInput getInput = new GetInput();
-      getInput.setTableName(join.getLeftTable());
-      getInput.setPrimaryKey(id);
-      GetOutput getOutput = new GetAction().execute(getInput);
-      QRecord   record    = getOutput.getRecord();
-
-      if(record == null)
-      {
-         throw (new QNotFoundException("Could not find " + (leftTable == null ? "" : leftTable.getLabel()) + " with primary key " + id));
-      }
-
-      ////////////////////////////////////////////////////////////////////
-      // set up the query - for the table on the right side of the join //
-      ////////////////////////////////////////////////////////////////////
-      QQueryFilter filter = new QQueryFilter();
-      for(JoinOn joinOn : join.getJoinOns())
-      {
-         filter.addCriteria(new QFilterCriteria(joinOn.getRightField(), QCriteriaOperator.EQUALS, List.of(record.getValue(joinOn.getLeftField()))));
-      }
-      filter.setOrderBys(join.getOrderBys());
-      filter.setLimit(maxRows);
-
-      QueryInput queryInput = new QueryInput();
-      queryInput.setTableName(join.getRightTable());
-      queryInput.setShouldTranslatePossibleValues(true);
-      queryInput.setShouldGenerateDisplayValues(true);
-      queryInput.setFilter(filter);
-      QueryOutput queryOutput = new QueryAction().execute(queryInput);
-
-      QValueFormatter.setBlobValuesToDownloadUrls(rightTable, queryOutput.getRecords());
-
-      int totalRows = queryOutput.getRecords().size();
-      if(maxRows != null && (queryOutput.getRecords().size() == maxRows))
-      {
-         /////////////////////////////////////////////////////////////////////////////////////
-         // if the input said to only do some max, and the # of results we got is that max, //
-         // then do a count query, for displaying 1-n of <count>                            //
-         /////////////////////////////////////////////////////////////////////////////////////
-         CountInput countInput = new CountInput();
-         countInput.setTableName(join.getRightTable());
-         countInput.setFilter(filter);
-         totalRows = new CountAction().execute(countInput).getCount();
-      }
-
-      String tablePath   = input.getInstance().getTablePath(rightTable.getName());
-      String viewAllLink = tablePath == null ? null : (tablePath + "?filter=" + URLEncoder.encode(JsonUtils.toJson(filter), Charset.defaultCharset()));
-
-      ChildRecordListData widgetData = new ChildRecordListData(widgetLabel, queryOutput, rightTable, tablePath, viewAllLink, totalRows);
-
-      if(BooleanUtils.isTrue(ValueUtils.getValueAsBoolean(input.getQueryParams().get("canAddChildRecord"))))
-      {
-         widgetData.setCanAddChildRecord(true);
-
-         //////////////////////////////////////////////////////////
-         // new child records must have values from the join-ons //
-         //////////////////////////////////////////////////////////
-         Map<String, Serializable> defaultValuesForNewChildRecords = new HashMap<>();
-         for(JoinOn joinOn : join.getJoinOns())
+         Integer maxRows = null;
+         if(StringUtils.hasContent(input.getQueryParams().get("maxRows")))
          {
-            defaultValuesForNewChildRecords.put(joinOn.getRightField(), record.getValue(joinOn.getLeftField()));
+            maxRows = ValueUtils.getValueAsInteger(input.getQueryParams().get("maxRows"));
          }
-         widgetData.setDefaultValuesForNewChildRecords(defaultValuesForNewChildRecords);
-
-         Map<String, Serializable> widgetValues = input.getWidgetMetaData().getDefaultValues();
-         if(widgetValues.containsKey("disabledFieldsForNewChildRecords"))
+         else if(input.getWidgetMetaData().getDefaultValues().containsKey("maxRows"))
          {
-            widgetData.setDisabledFieldsForNewChildRecords((Set<String>) widgetValues.get("disabledFieldsForNewChildRecords"));
+            maxRows = ValueUtils.getValueAsInteger(input.getWidgetMetaData().getDefaultValues().containsKey("maxRows"));
          }
-      }
 
-      return (new RenderWidgetOutput(widgetData));
+         //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+         // fetch the record that we're getting children for. e.g., the left-side of the join, with the input id                                     //
+         // but - only try this if we were given an id.  note, this widget could be called for on an INSERT screen, where we don't have a record yet //
+         // but we still want to be able to return all the other data in here that otherwise comes from the widget meta data, join, etc.             //
+         //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+         int          totalRows     = 0;
+         QRecord      primaryRecord = null;
+         QQueryFilter filter        = null;
+         QueryOutput  queryOutput   = new QueryOutput(new QueryInput());
+         if(StringUtils.hasContent(id))
+         {
+            GetInput getInput = new GetInput();
+            getInput.setTableName(join.getLeftTable());
+            getInput.setPrimaryKey(id);
+            GetOutput getOutput = new GetAction().execute(getInput);
+            primaryRecord = getOutput.getRecord();
+
+            if(primaryRecord == null)
+            {
+               throw (new QNotFoundException("Could not find " + (leftTable == null ? "" : leftTable.getLabel()) + " with primary key " + id));
+            }
+
+            ////////////////////////////////////////////////////////////////////
+            // set up the query - for the table on the right side of the join //
+            ////////////////////////////////////////////////////////////////////
+            filter = new QQueryFilter();
+            for(JoinOn joinOn : join.getJoinOns())
+            {
+               filter.addCriteria(new QFilterCriteria(joinOn.getRightField(), QCriteriaOperator.EQUALS, List.of(primaryRecord.getValue(joinOn.getLeftField()))));
+            }
+            filter.setOrderBys(join.getOrderBys());
+            filter.setLimit(maxRows);
+
+            QueryInput queryInput = new QueryInput();
+            queryInput.setTableName(join.getRightTable());
+            queryInput.setShouldTranslatePossibleValues(true);
+            queryInput.setShouldGenerateDisplayValues(true);
+            queryInput.setFilter(filter);
+            queryOutput = new QueryAction().execute(queryInput);
+
+            QValueFormatter.setBlobValuesToDownloadUrls(rightTable, queryOutput.getRecords());
+
+            totalRows = queryOutput.getRecords().size();
+            if(maxRows != null && (queryOutput.getRecords().size() == maxRows))
+            {
+               /////////////////////////////////////////////////////////////////////////////////////
+               // if the input said to only do some max, and the # of results we got is that max, //
+               // then do a count query, for displaying 1-n of <count>                            //
+               /////////////////////////////////////////////////////////////////////////////////////
+               CountInput countInput = new CountInput();
+               countInput.setTableName(join.getRightTable());
+               countInput.setFilter(filter);
+               totalRows = new CountAction().execute(countInput).getCount();
+            }
+         }
+
+         String tablePath   = input.getInstance().getTablePath(rightTable.getName());
+         String viewAllLink = tablePath == null ? null : (tablePath + "?filter=" + URLEncoder.encode(JsonUtils.toJson(filter), Charset.defaultCharset()));
+
+         ChildRecordListData widgetData = new ChildRecordListData(widgetLabel, queryOutput, rightTable, tablePath, viewAllLink, totalRows);
+
+         if(BooleanUtils.isTrue(ValueUtils.getValueAsBoolean(input.getQueryParams().get("canAddChildRecord"))))
+         {
+            widgetData.setCanAddChildRecord(true);
+
+            //////////////////////////////////////////////////////////
+            // new child records must have values from the join-ons //
+            //////////////////////////////////////////////////////////
+            Map<String, Serializable> defaultValuesForNewChildRecords = new HashMap<>();
+            if(primaryRecord != null)
+            {
+               for(JoinOn joinOn : join.getJoinOns())
+               {
+                  defaultValuesForNewChildRecords.put(joinOn.getRightField(), primaryRecord.getValue(joinOn.getLeftField()));
+               }
+            }
+
+            widgetData.setDefaultValuesForNewChildRecords(defaultValuesForNewChildRecords);
+
+            Map<String, Serializable> widgetValues = input.getWidgetMetaData().getDefaultValues();
+            if(widgetValues.containsKey("disabledFieldsForNewChildRecords"))
+            {
+               widgetData.setDisabledFieldsForNewChildRecords((Set<String>) widgetValues.get("disabledFieldsForNewChildRecords"));
+            }
+            else
+            {
+               /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+               // if there are no disabled fields specified - then normally any fields w/ a default value get implicitly disabled //
+               // but - if we didn't look-up the primary record, then we'll want to explicit disable fields from joins            //
+               /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+               if(primaryRecord == null)
+               {
+                  Set<String> implicitlyDisabledFields = new HashSet<>();
+                  widgetData.setDisabledFieldsForNewChildRecords(implicitlyDisabledFields);
+                  for(JoinOn joinOn : join.getJoinOns())
+                  {
+                     implicitlyDisabledFields.add(joinOn.getRightField());
+                  }
+               }
+            }
+         }
+
+         return (new RenderWidgetOutput(widgetData));
+      }
+      catch(Exception e)
+      {
+         LOG.warn("Error rendering child record list", e, logPair("widgetName", () -> input.getWidgetMetaData().getName()));
+         throw (e);
+      }
    }
 
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/interfaces/QStorageInterface.java

@@ -0,0 +1,69 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.actions.interfaces;
+
+
+import java.io.InputStream;
+import java.io.OutputStream;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.actions.tables.storage.StorageInput;
+
+
+/*******************************************************************************
+ ** Interface for actions that a backend can perform, based on streaming data
+ ** into the backend's storage.
+ *******************************************************************************/
+public interface QStorageInterface
+{
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   OutputStream createOutputStream(StorageInput storageInput) throws QException;
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   InputStream getInputStream(StorageInput storageInput) throws QException;
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   default void makePublic(StorageInput storageInput) throws QException
+   {
+      //////////
+      // noop //
+      //////////
+   }
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   default String getDownloadURL(StorageInput storageInput) throws QException
+   {
+      return (null);
+   }
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/messaging/SendMessageAction.java

@@ -0,0 +1,64 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.actions.messaging;
+
+
+import com.kingsrook.qqq.backend.core.actions.AbstractQActionFunction;
+import com.kingsrook.qqq.backend.core.context.QContext;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.actions.messaging.SendMessageInput;
+import com.kingsrook.qqq.backend.core.model.actions.messaging.SendMessageOutput;
+import com.kingsrook.qqq.backend.core.model.metadata.messaging.QMessagingProviderMetaData;
+import com.kingsrook.qqq.backend.core.modules.messaging.MessagingProviderInterface;
+import com.kingsrook.qqq.backend.core.modules.messaging.QMessagingProviderDispatcher;
+import com.kingsrook.qqq.backend.core.utils.StringUtils;
+
+
+/*******************************************************************************
+ **
+ *******************************************************************************/
+public class SendMessageAction extends AbstractQActionFunction<SendMessageInput, SendMessageOutput>
+{
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public SendMessageOutput execute(SendMessageInput input) throws QException
+   {
+      if(!StringUtils.hasContent(input.getMessagingProviderName()))
+      {
+         throw (new QException("Messaging provider name was not given in SendMessageInput."));
+      }
+
+      QMessagingProviderMetaData messagingProvider = QContext.getQInstance().getMessagingProvider(input.getMessagingProviderName());
+      if(messagingProvider == null)
+      {
+         throw (new QException("Messaging provider named [" + input.getMessagingProviderName() + "] was not found in this QInstance."));
+      }
+
+      MessagingProviderInterface messagingProviderInterface = new QMessagingProviderDispatcher().getMessagingProviderInterface(messagingProvider.getType());
+
+      return (messagingProviderInterface.sendMessage(input));
+   }
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/metadata/JoinGraph.java

@@ -29,6 +29,7 @@ import java.util.List;
 import java.util.Objects;
 import java.util.Set;
 import java.util.TreeSet;
+import com.kingsrook.qqq.backend.core.instances.QMetaDataVariableInterpreter;
 import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
 import com.kingsrook.qqq.backend.core.model.metadata.joins.QJoinMetaData;
 import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
@@ -41,9 +42,19 @@ import com.kingsrook.qqq.backend.core.utils.StringUtils;
  *******************************************************************************/
 public class JoinGraph
 {
-
    private Set<Edge> edges = new HashSet<>();
 
+   ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+   // as an instance grows, with the number of joins (say, more than 50?), especially as they may have a lot of connections, //
+   // it can become very very slow to process a full join graph (e.g., 10 seconds, maybe much worse, per Big-O...)           //
+   // also, it's not frequently useful to look at a join path that's more than a handful of tables long.                     //
+   // thus - this property exists - to limit the max length of a join path.  Keeping it small keeps instance enrichment      //
+   // and validation reasonably performant, at the possible cost of, some join-path that's longer than this limit may not    //
+   // be found - but - chances are, you don't want some 12-element join path to be used anyway, thus, this makes sense.      //
+   // but - it can be adjusted, per system property or ENV var.                                                              //
+   ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+   private int maxPathLength = new QMetaDataVariableInterpreter().getIntegerFromPropertyOrEnvironment("qqq.instance.joinGraph.maxPathLength", "QQQ_INSTANCE_JOIN_GRAPH_MAX_PATH_LENGTH", 3);
+
 
 
    /*******************************************************************************
@@ -303,6 +314,13 @@ public class JoinGraph
 
                if(otherTableName != null)
                {
+                  if(newPath.size() > maxPathLength)
+                  {
+                     ////////////////////////////////////////////////////////////////
+                     // performance hack.  see comment at maxPathLength definition //
+                     ////////////////////////////////////////////////////////////////
+                     continue;
+                  }
 
                   JoinConnectionList newConnectionList = connectionList.copy();
                   JoinConnection     joinConnection    = new JoinConnection(otherTableName, edge.joinName);

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/permissions/PermissionsHelper.java

@@ -34,15 +34,18 @@ import com.kingsrook.qqq.backend.core.exceptions.QPermissionDeniedException;
 import com.kingsrook.qqq.backend.core.logging.QLogger;
 import com.kingsrook.qqq.backend.core.model.actions.AbstractActionInput;
 import com.kingsrook.qqq.backend.core.model.actions.AbstractTableActionInput;
+import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
 import com.kingsrook.qqq.backend.core.model.metadata.dashboard.QWidgetMetaDataInterface;
 import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.permissions.DenyBehavior;
 import com.kingsrook.qqq.backend.core.model.metadata.permissions.MetaDataWithName;
 import com.kingsrook.qqq.backend.core.model.metadata.permissions.MetaDataWithPermissionRules;
+import com.kingsrook.qqq.backend.core.model.metadata.permissions.PermissionLevel;
 import com.kingsrook.qqq.backend.core.model.metadata.permissions.QPermissionRules;
 import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.Capability;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 import com.kingsrook.qqq.backend.core.model.session.QSession;
 import com.kingsrook.qqq.backend.core.utils.StringUtils;
@@ -333,9 +336,25 @@ public class PermissionsHelper
          QPermissionRules rules    = getEffectivePermissionRules(tableMetaData, instance);
          String           baseName = getEffectivePermissionBaseName(rules, tableMetaData.getName());
 
-         for(TablePermissionSubType permissionSubType : TablePermissionSubType.values())
+         QBackendMetaData backend = instance.getBackend(tableMetaData.getBackendName());
+         if(tableMetaData.isCapabilityEnabled(backend, Capability.TABLE_INSERT))
          {
-            addEffectiveAvailablePermission(rules, permissionSubType, rs, baseName, tableMetaData, "Table");
+            addEffectiveAvailablePermission(rules, TablePermissionSubType.INSERT, rs, baseName, tableMetaData, "Table");
+         }
+
+         if(tableMetaData.isCapabilityEnabled(backend, Capability.TABLE_UPDATE))
+         {
+            addEffectiveAvailablePermission(rules, TablePermissionSubType.EDIT, rs, baseName, tableMetaData, "Table");
+         }
+
+         if(tableMetaData.isCapabilityEnabled(backend, Capability.TABLE_DELETE))
+         {
+            addEffectiveAvailablePermission(rules, TablePermissionSubType.DELETE, rs, baseName, tableMetaData, "Table");
+         }
+
+         if(tableMetaData.isCapabilityEnabled(backend, Capability.TABLE_QUERY) || tableMetaData.isCapabilityEnabled(backend, Capability.TABLE_GET))
+         {
+            addEffectiveAvailablePermission(rules, TablePermissionSubType.READ, rs, baseName, tableMetaData, "Table");
          }
       }
 
@@ -369,7 +388,10 @@ public class PermissionsHelper
       {
          QPermissionRules rules    = getEffectivePermissionRules(widgetMetaData, instance);
          String           baseName = getEffectivePermissionBaseName(rules, widgetMetaData.getName());
-         addEffectiveAvailablePermission(rules, PrivatePermissionSubType.HAS_ACCESS, rs, baseName, widgetMetaData, "Widget");
+         if(!rules.getLevel().equals(PermissionLevel.NOT_PROTECTED))
+         {
+            addEffectiveAvailablePermission(rules, PrivatePermissionSubType.HAS_ACCESS, rs, baseName, widgetMetaData, "Widget");
+         }
       }
 
       return (rs);

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/processes/QProcessCallbackFactory.java

@@ -23,11 +23,19 @@ package com.kingsrook.qqq.backend.core.actions.processes;
 
 
 import java.io.Serializable;
+import java.util.Collection;
 import java.util.Collections;
 import java.util.List;
 import java.util.Map;
+import com.kingsrook.qqq.backend.core.context.QContext;
+import com.kingsrook.qqq.backend.core.exceptions.QRuntimeException;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.data.QRecordEntity;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
+import com.kingsrook.qqq.backend.core.utils.StringUtils;
 
 
 /*******************************************************************************
@@ -65,4 +73,56 @@ public class QProcessCallbackFactory
       };
    }
 
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public static QProcessCallback forRecordEntity(QRecordEntity entity)
+   {
+      return forRecord(entity.toQRecord());
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public static QProcessCallback forRecord(QRecord record)
+   {
+      String primaryKeyField = "id";
+      if(StringUtils.hasContent(record.getTableName()))
+      {
+         primaryKeyField = QContext.getQInstance().getTable(record.getTableName()).getPrimaryKeyField();
+      }
+
+      Serializable primaryKeyValue = record.getValue(primaryKeyField);
+      if(primaryKeyValue == null)
+      {
+         throw (new QRuntimeException("Record did not have value in its primary key field [" + primaryKeyField + "]"));
+      }
+
+      return (forPrimaryKey(primaryKeyField, primaryKeyValue));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public static QProcessCallback forPrimaryKey(String fieldName, Serializable value)
+   {
+      return (forFilter(new QQueryFilter().withCriteria(new QFilterCriteria(fieldName, QCriteriaOperator.EQUALS, value))));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public static QProcessCallback forPrimaryKeys(String fieldName, Collection<? extends Serializable> values)
+   {
+      return (forFilter(new QQueryFilter().withCriteria(new QFilterCriteria(fieldName, QCriteriaOperator.IN, values))));
+   }
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/processes/RunProcessAction.java

@@ -190,7 +190,25 @@ public class RunProcessAction
                // Run backend steps //
                ///////////////////////
                LOG.debug("Running backend step [" + step.getName() + "] in process [" + process.getName() + "]");
-               runBackendStep(runProcessInput, process, runProcessOutput, stateKey, backendStepMetaData, process, processState);
+               RunBackendStepOutput runBackendStepOutput = runBackendStep(runProcessInput, process, runProcessOutput, stateKey, backendStepMetaData, process, processState);
+
+               /////////////////////////////////////////////////////////////////////////////////////////
+               // if the step returned an override lastStepName, use that to determine how we proceed //
+               /////////////////////////////////////////////////////////////////////////////////////////
+               if(runBackendStepOutput.getOverrideLastStepName() != null)
+               {
+                  LOG.debug("Process step [" + lastStepName + "] returned an overrideLastStepName [" + runBackendStepOutput.getOverrideLastStepName() + "]!");
+                  lastStepName = runBackendStepOutput.getOverrideLastStepName();
+               }
+
+               /////////////////////////////////////////////////////////////////////////////////////////////
+               // similarly, if the step produced an updatedFrontendStepList, propagate that data outward //
+               /////////////////////////////////////////////////////////////////////////////////////////////
+               if(runBackendStepOutput.getUpdatedFrontendStepList() != null)
+               {
+                  LOG.debug("Process step [" + lastStepName + "] generated an updatedFrontendStepList [" + runBackendStepOutput.getUpdatedFrontendStepList().stream().map(s -> s.getName()).toList() + "]!");
+                  runProcessOutput.setUpdatedFrontendStepList(runBackendStepOutput.getUpdatedFrontendStepList());
+               }
             }
             else
             {
@@ -339,7 +357,7 @@ public class RunProcessAction
    /*******************************************************************************
     ** Run a single backend step.
     *******************************************************************************/
-   private void runBackendStep(RunProcessInput runProcessInput, QProcessMetaData process, RunProcessOutput runProcessOutput, UUIDAndTypeStateKey stateKey, QBackendStepMetaData backendStep, QProcessMetaData qProcessMetaData, ProcessState processState) throws Exception
+   private RunBackendStepOutput runBackendStep(RunProcessInput runProcessInput, QProcessMetaData process, RunProcessOutput runProcessOutput, UUIDAndTypeStateKey stateKey, QBackendStepMetaData backendStep, QProcessMetaData qProcessMetaData, ProcessState processState) throws Exception
    {
       RunBackendStepInput runBackendStepInput = new RunBackendStepInput(processState);
       runBackendStepInput.setProcessName(process.getName());
@@ -368,14 +386,16 @@ public class RunProcessAction
          runBackendStepInput.setBasepullLastRunTime((Instant) runProcessInput.getValues().get(BASEPULL_LAST_RUNTIME_KEY));
       }
 
-      RunBackendStepOutput lastFunctionResult = new RunBackendStepAction().execute(runBackendStepInput);
-      storeState(stateKey, lastFunctionResult.getProcessState());
+      RunBackendStepOutput runBackendStepOutput = new RunBackendStepAction().execute(runBackendStepInput);
+      storeState(stateKey, runBackendStepOutput.getProcessState());
 
-      if(lastFunctionResult.getException() != null)
+      if(runBackendStepOutput.getException() != null)
       {
-         runProcessOutput.setException(lastFunctionResult.getException());
-         throw (lastFunctionResult.getException());
+         runProcessOutput.setException(runBackendStepOutput.getException());
+         throw (runBackendStepOutput.getException());
       }
+
+      return (runBackendStepOutput);
    }
 
 
@@ -495,15 +515,15 @@ public class RunProcessAction
       String basepullKeyValue = (basepullConfiguration.getKeyValue() != null) ? basepullConfiguration.getKeyValue() : process.getName();
 
       //////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-      // if backend specifies that it uses variants, look for that data in the session and append to our basepull key //
+      // if process specifies that it uses variants, look for that data in the session and append to our basepull key //
       //////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-      if(process.getSchedule() != null && process.getSchedule().getVariantBackend() != null)
+      if(process.getVariantBackend() != null)
       {
          QSession         session         = QContext.getQSession();
-         QBackendMetaData backendMetaData = QContext.getQInstance().getBackend(process.getSchedule().getVariantBackend());
+         QBackendMetaData backendMetaData = QContext.getQInstance().getBackend(process.getVariantBackend());
          if(session.getBackendVariants() == null || !session.getBackendVariants().containsKey(backendMetaData.getVariantOptionsTableTypeValue()))
          {
-            LOG.info("Could not find Backend Variant information for Backend '" + backendMetaData.getName() + "'");
+            LOG.warn("Could not find Backend Variant information for Backend '" + backendMetaData.getName() + "'");
          }
          else
          {

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/CsvExportStreamer.java

@@ -31,6 +31,7 @@ import com.kingsrook.qqq.backend.core.logging.QLogger;
 import com.kingsrook.qqq.backend.core.model.actions.reporting.ExportInput;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportView;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 import com.kingsrook.qqq.backend.core.utils.StringUtils;
 
@@ -65,12 +66,12 @@ public class CsvExportStreamer implements ExportStreamerInterface
     **
     *******************************************************************************/
    @Override
-   public void start(ExportInput exportInput, List<QFieldMetaData> fields, String label) throws QReportingException
+   public void start(ExportInput exportInput, List<QFieldMetaData> fields, String label, QReportView view) throws QReportingException
    {
       this.exportInput = exportInput;
       this.fields = fields;
       table = exportInput.getTable();
-      outputStream = this.exportInput.getReportOutputStream();
+      outputStream = this.exportInput.getReportDestination().getReportOutputStream();
 
       writeTitleAndHeader();
    }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/DistinctFilteringRecordPipe.java

@@ -0,0 +1,146 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2023.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.actions.reporting;
+
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.UniqueKey;
+
+
+/*******************************************************************************
+ ** Subclass of record pipe that ony allows through distinct records, based on
+ ** the set of fields specified in the constructor as a uniqueKey.
+ *******************************************************************************/
+public class DistinctFilteringRecordPipe extends RecordPipe
+{
+   private UniqueKey         uniqueKey;
+   private Set<Serializable> seenValues = new HashSet<>();
+
+
+
+   /*******************************************************************************
+    ** Constructor
+    **
+    *******************************************************************************/
+   public DistinctFilteringRecordPipe(UniqueKey uniqueKey)
+   {
+      this.uniqueKey = uniqueKey;
+   }
+
+
+
+   /*******************************************************************************
+    ** Constructor that accepts pipe's overrideCapacity (allowed to be null)
+    **
+    *******************************************************************************/
+   public DistinctFilteringRecordPipe(UniqueKey uniqueKey, Integer overrideCapacity)
+   {
+      super(overrideCapacity);
+      this.uniqueKey = uniqueKey;
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public void addRecords(List<QRecord> records) throws QException
+   {
+      List<QRecord> recordsToAdd = new ArrayList<>();
+      for(QRecord record : records)
+      {
+         if(!seenBefore(record))
+         {
+            recordsToAdd.add(record);
+         }
+      }
+
+      if(recordsToAdd.isEmpty())
+      {
+         return;
+      }
+
+      super.addRecords(recordsToAdd);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public void addRecord(QRecord record) throws QException
+   {
+      if(seenBefore(record))
+      {
+         return;
+      }
+
+      super.addRecord(record);
+   }
+
+
+
+   /*******************************************************************************
+    ** return true if we've seen this record before (based on the unique key) -
+    ** also - update the set of seen values!
+    *******************************************************************************/
+   private boolean seenBefore(QRecord record)
+   {
+      Serializable ukValues = extractUKValues(record);
+      if(seenValues.contains(ukValues))
+      {
+         return true;
+      }
+      seenValues.add(ukValues);
+      return false;
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private Serializable extractUKValues(QRecord record)
+   {
+      if(uniqueKey.getFieldNames().size() == 1)
+      {
+         return (record.getValue(uniqueKey.getFieldNames().get(0)));
+      }
+      else
+      {
+         ArrayList<Serializable> rs = new ArrayList<>();
+         for(String fieldName : uniqueKey.getFieldNames())
+         {
+            rs.add(record.getValue(fieldName));
+         }
+         return (rs);
+      }
+   }
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/ExportAction.java

@@ -52,6 +52,7 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryJoin;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
+import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportView;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.ExposedJoin;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleDispatcher;
@@ -138,7 +139,7 @@ public class ExportAction
       ///////////////////////////////////////////////////////////////////////////////////////////////////////////
       // check if this report format has a max-rows limit -- if so, do a count to verify we're under the limit //
       ///////////////////////////////////////////////////////////////////////////////////////////////////////////
-      ReportFormat reportFormat = exportInput.getReportFormat();
+      ReportFormat reportFormat = exportInput.getReportDestination().getReportFormat();
       verifyCountUnderMax(exportInput, backendModule, reportFormat);
 
       preExecuteRan = true;
@@ -189,6 +190,9 @@ public class ExportAction
       Set<String>     addedJoinNames = new HashSet<>();
       if(CollectionUtils.nullSafeHasContents(exportInput.getFieldNames()))
       {
+         /////////////////////////////////////////////////////////////////////////////////////////////
+         // make sure that any tables being selected from are included as (LEFT) joins in the query //
+         /////////////////////////////////////////////////////////////////////////////////////////////
          for(String fieldName : exportInput.getFieldNames())
          {
             if(fieldName.contains("."))
@@ -197,27 +201,7 @@ public class ExportAction
                String   joinTableName = parts[0];
                if(!addedJoinNames.contains(joinTableName))
                {
-                  QueryJoin queryJoin = new QueryJoin(joinTableName).withType(QueryJoin.Type.LEFT).withSelect(true);
-                  queryJoins.add(queryJoin);
-
-                  /////////////////////////////////////////////////////////////////////////////////////////////
-                  // in at least some cases, we need to let the queryJoin know what join-meta-data to use... //
-                  // This code basically mirrors what QFMD is doing right now, so it's better -              //
-                  // but shouldn't all of this just be in JoinsContext?  it does some of this...             //
-                  /////////////////////////////////////////////////////////////////////////////////////////////
-                  QTableMetaData        table               = exportInput.getTable();
-                  Optional<ExposedJoin> exposedJoinOptional = CollectionUtils.nonNullList(table.getExposedJoins()).stream().filter(ej -> ej.getJoinTable().equals(joinTableName)).findFirst();
-                  if(exposedJoinOptional.isEmpty())
-                  {
-                     throw (new QException("Could not find exposed join between base table " + table.getName() + " and requested join table " + joinTableName));
-                  }
-                  ExposedJoin exposedJoin = exposedJoinOptional.get();
-
-                  if(exposedJoin.getJoinPath().size() == 1)
-                  {
-                     queryJoin.setJoinMetaData(QContext.getQInstance().getJoin(exposedJoin.getJoinPath().get(exposedJoin.getJoinPath().size() - 1)));
-                  }
-
+                  queryJoins.add(new QueryJoin(joinTableName).withType(QueryJoin.Type.LEFT).withSelect(true));
                   addedJoinNames.add(joinTableName);
                }
             }
@@ -232,6 +216,7 @@ public class ExportAction
       }
       queryInput.getFilter().setLimit(exportInput.getLimit());
       queryInput.setShouldTranslatePossibleValues(true);
+      queryInput.withQueryHint(QueryInput.QueryHint.POTENTIALLY_LARGE_NUMBER_OF_RESULTS);
 
       /////////////////////////////////////////////////////////////////
       // tell this query that it needs to put its output into a pipe //
@@ -242,10 +227,19 @@ public class ExportAction
       ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
       // set up a report streamer, which will read rows from the pipe, and write formatted report rows to the output stream //
       ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-      ReportFormat            reportFormat   = exportInput.getReportFormat();
+      ReportFormat            reportFormat   = exportInput.getReportDestination().getReportFormat();
       ExportStreamerInterface reportStreamer = reportFormat.newReportStreamer();
       List<QFieldMetaData>    fields         = getFields(exportInput);
-      reportStreamer.start(exportInput, fields, "Sheet 1");
+
+      //////////////////////////////////////////////////////////
+      // it seems we can pass a view with just a name in here //
+      //////////////////////////////////////////////////////////
+      List<QReportView> views = new ArrayList<>();
+      views.add(new QReportView()
+         .withName("export"));
+
+      reportStreamer.preRun(exportInput.getReportDestination(), views);
+      reportStreamer.start(exportInput, fields, "Sheet 1", views.get(0));
 
       //////////////////////////////////////////
       // run the query action as an async job //
@@ -334,7 +328,7 @@ public class ExportAction
 
       try
       {
-         exportInput.getReportOutputStream().close();
+         exportInput.getReportDestination().getReportOutputStream().close();
       }
       catch(Exception e)
       {

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/ExportStreamerInterface.java

@@ -26,8 +26,10 @@ import java.util.List;
 import java.util.Map;
 import com.kingsrook.qqq.backend.core.exceptions.QReportingException;
 import com.kingsrook.qqq.backend.core.model.actions.reporting.ExportInput;
+import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportDestination;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportView;
 
 
 /*******************************************************************************
@@ -35,20 +37,14 @@ import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
  *******************************************************************************/
 public interface ExportStreamerInterface
 {
-   /*******************************************************************************
-    ** Called once, before any rows are available.  Meant to write a header, for example.
-    *******************************************************************************/
-   void start(ExportInput exportInput, List<QFieldMetaData> fields, String label) throws QReportingException;
 
    /*******************************************************************************
-    ** Called as records flow into the pipe.
-    ******************************************************************************/
-   void addRecords(List<QRecord> recordList) throws QReportingException;
-
-   /*******************************************************************************
-    ** Called once, after all rows are available.  Meant to write a footer, or close resources, for example.
+    ** Called once, before any sheets are actually being produced.
     *******************************************************************************/
-   void finish() throws QReportingException;
+   default void preRun(ReportDestination reportDestination, List<QReportView> views) throws QReportingException
+   {
+      // noop in base class
+   }
 
    /*******************************************************************************
     **
@@ -58,6 +54,20 @@ public interface ExportStreamerInterface
       // noop in base class
    }
 
+   /*******************************************************************************
+    ** Called once per sheet, before any rows are available.  Meant to write a
+    ** header, for example.
+    **
+    ** If multiple sheets are being created, there is no separate end-sheet call.
+    ** Rather, a new one will just get started...
+    *******************************************************************************/
+   void start(ExportInput exportInput, List<QFieldMetaData> fields, String label, QReportView view) throws QReportingException;
+
+   /*******************************************************************************
+    ** Called as records flow into the pipe.
+    ******************************************************************************/
+   void addRecords(List<QRecord> recordList) throws QReportingException;
+
    /*******************************************************************************
     **
     *******************************************************************************/
@@ -65,4 +75,11 @@ public interface ExportStreamerInterface
    {
       addRecords(List.of(record));
    }
+
+   /*******************************************************************************
+    ** Called after all sheets are complete.  Meant to do a final write, or close
+    ** resources, for example.
+    *******************************************************************************/
+   void finish() throws QReportingException;
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/GenerateReportAction.java

@@ -24,6 +24,8 @@ package com.kingsrook.qqq.backend.core.actions.reporting;
 
 import java.io.Serializable;
 import java.math.BigDecimal;
+import java.time.Instant;
+import java.time.LocalDate;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashMap;
@@ -32,18 +34,23 @@ import java.util.List;
 import java.util.Map;
 import java.util.Objects;
 import java.util.Set;
+import java.util.concurrent.atomic.AtomicInteger;
 import java.util.function.Function;
 import java.util.function.Supplier;
 import java.util.stream.Collectors;
+import com.kingsrook.qqq.backend.core.actions.AbstractQActionFunction;
 import com.kingsrook.qqq.backend.core.actions.async.AsyncRecordPipeLoop;
 import com.kingsrook.qqq.backend.core.actions.customizers.QCodeLoader;
 import com.kingsrook.qqq.backend.core.actions.reporting.customizers.DataSourceQueryInputCustomizer;
 import com.kingsrook.qqq.backend.core.actions.reporting.customizers.ReportViewCustomizer;
+import com.kingsrook.qqq.backend.core.actions.tables.CountAction;
 import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
 import com.kingsrook.qqq.backend.core.actions.values.QValueFormatter;
+import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.exceptions.QFormulaException;
 import com.kingsrook.qqq.backend.core.exceptions.QReportingException;
+import com.kingsrook.qqq.backend.core.exceptions.QUserFacingException;
 import com.kingsrook.qqq.backend.core.instances.QMetaDataVariableInterpreter;
 import com.kingsrook.qqq.backend.core.logging.QLogger;
 import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
@@ -51,6 +58,9 @@ import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutp
 import com.kingsrook.qqq.backend.core.model.actions.reporting.ExportInput;
 import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportFormat;
 import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportInput;
+import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportOutput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountOutput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.JoinsContext;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
@@ -68,11 +78,17 @@ import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwith
 import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.BackendStepPostRunInput;
 import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.BackendStepPostRunOutput;
 import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
+import com.kingsrook.qqq.backend.core.utils.ObjectUtils;
 import com.kingsrook.qqq.backend.core.utils.Pair;
 import com.kingsrook.qqq.backend.core.utils.StringUtils;
 import com.kingsrook.qqq.backend.core.utils.aggregates.AggregatesInterface;
 import com.kingsrook.qqq.backend.core.utils.aggregates.BigDecimalAggregates;
+import com.kingsrook.qqq.backend.core.utils.aggregates.InstantAggregates;
 import com.kingsrook.qqq.backend.core.utils.aggregates.IntegerAggregates;
+import com.kingsrook.qqq.backend.core.utils.aggregates.LocalDateAggregates;
+import com.kingsrook.qqq.backend.core.utils.aggregates.LongAggregates;
+import com.kingsrook.qqq.backend.core.utils.aggregates.StringAggregates;
+import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
 
 
 /*******************************************************************************
@@ -87,7 +103,7 @@ import com.kingsrook.qqq.backend.core.utils.aggregates.IntegerAggregates;
  ** - summaries (like pivot tables, but called summary to avoid confusion with "native" pivot tables),
  ** - native pivot tables (not initially supported, due to lack of support in fastexcel...).
  *******************************************************************************/
-public class GenerateReportAction
+public class GenerateReportAction extends AbstractQActionFunction<ReportInput, ReportOutput>
 {
    private static final QLogger LOG = QLogger.getLogger(GenerateReportAction.class);
 
@@ -101,50 +117,67 @@ public class GenerateReportAction
    // Aggregates: (count:47;sum:10,000;max:2,000;min:15)                                                      //
    // salesSummaryReport > [(state:MO),(city:St.Louis)] > salePrice  > (count:47;sum:10,000;max:2,000;min:15) //
    /////////////////////////////////////////////////////////////////////////////////////////////////////////////
-   Map<String, Map<SummaryKey, Map<String, AggregatesInterface<?>>>> summaryAggregates  = new HashMap<>();
-   Map<String, Map<SummaryKey, Map<String, AggregatesInterface<?>>>> varianceAggregates = new HashMap<>();
+   Map<String, Map<SummaryKey, Map<String, AggregatesInterface<?, ?>>>> summaryAggregates  = new HashMap<>();
+   Map<String, Map<SummaryKey, Map<String, AggregatesInterface<?, ?>>>> varianceAggregates = new HashMap<>();
 
-   Map<String, AggregatesInterface<?>> totalAggregates         = new HashMap<>();
-   Map<String, AggregatesInterface<?>> varianceTotalAggregates = new HashMap<>();
+   Map<String, AggregatesInterface<?, ?>> totalAggregates         = new HashMap<>();
+   Map<String, AggregatesInterface<?, ?>> varianceTotalAggregates = new HashMap<>();
 
-   private QReportMetaData         report;
-   private ReportFormat            reportFormat;
    private ExportStreamerInterface reportStreamer;
+   private List<QReportDataSource> dataSources;
+   private List<QReportView>       views;
+
+   private Map<String, Integer> countByDataSource = new HashMap<>();
 
 
 
    /*******************************************************************************
     **
     *******************************************************************************/
-   public void execute(ReportInput reportInput) throws QException
+   public ReportOutput execute(ReportInput reportInput) throws QException
    {
-      report = reportInput.getInstance().getReport(reportInput.getReportName());
-      reportFormat = reportInput.getReportFormat();
+      ReportOutput    reportOutput = new ReportOutput();
+      QReportMetaData report       = getReportMetaData(reportInput);
+
+      this.views = report.getViews();
+      this.dataSources = report.getDataSources();
+
+      ReportFormat reportFormat = reportInput.getReportDestination().getReportFormat();
       if(reportFormat == null)
       {
          throw new QException("Report format was not specified.");
       }
-      reportStreamer = reportFormat.newReportStreamer();
+
+      if(reportInput.getOverrideExportStreamerSupplier() != null)
+      {
+         reportStreamer = reportInput.getOverrideExportStreamerSupplier().get();
+      }
+      else
+      {
+         reportStreamer = reportFormat.newReportStreamer();
+      }
+
+      reportStreamer.preRun(reportInput.getReportDestination(), views);
 
       ////////////////////////////////////////////////////////////////////////////////////////////////
       // foreach data source, do a query (possibly more than 1, if it goes to multiple table views) //
       ////////////////////////////////////////////////////////////////////////////////////////////////
-      for(QReportDataSource dataSource : report.getDataSources())
+      for(QReportDataSource dataSource : dataSources)
       {
          //////////////////////////////////////////////////////////////////////////////
          // make a list of the views that use this data source for various purposes. //
          //////////////////////////////////////////////////////////////////////////////
-         List<QReportView> dataSourceTableViews = report.getViews().stream()
+         List<QReportView> dataSourceTableViews = views.stream()
             .filter(v -> v.getType().equals(ReportType.TABLE))
             .filter(v -> v.getDataSourceName().equals(dataSource.getName()))
             .toList();
 
-         List<QReportView> dataSourceSummaryViews = report.getViews().stream()
+         List<QReportView> dataSourceSummaryViews = views.stream()
             .filter(v -> v.getType().equals(ReportType.SUMMARY))
             .filter(v -> v.getDataSourceName().equals(dataSource.getName()))
             .toList();
 
-         List<QReportView> dataSourceVariantViews = report.getViews().stream()
+         List<QReportView> dataSourceVariantViews = views.stream()
             .filter(v -> v.getType().equals(ReportType.SUMMARY))
             .filter(v -> v.getVarianceDataSourceName() != null && v.getVarianceDataSourceName().equals(dataSource.getName()))
             .toList();
@@ -183,24 +216,50 @@ public class GenerateReportAction
                ////////////////////////////////////////////////////////////////////////////////////
                // start the table-view (e.g., open this tab in xlsx) and then run the query-loop //
                ////////////////////////////////////////////////////////////////////////////////////
-               startTableView(reportInput, dataSource, dataSourceTableView);
+               startTableView(reportInput, dataSource, dataSourceTableView, reportFormat);
                gatherData(reportInput, dataSource, dataSourceTableView, dataSourceSummaryViews, dataSourceVariantViews);
             }
          }
       }
 
+      //////////////////////
+      // add pivot sheets //
+      //////////////////////
+      for(QReportView view : views)
+      {
+         if(view.getType().equals(ReportType.PIVOT))
+         {
+            if(reportFormat.getSupportsNativePivotTables())
+            {
+               startTableView(reportInput, null, view, reportFormat);
+            }
+            else
+            {
+               LOG.warn("Request to render a report with a PIVOT type view, for a format that does not support native pivot tables", logPair("reportFormat", reportFormat));
+            }
+
+            //////////////////////////////////////////////////////////////////////////
+            // there's no data to add to a pivot table, so nothing else to do here. //
+            //////////////////////////////////////////////////////////////////////////
+         }
+      }
+
       outputSummaries(reportInput);
 
+      reportOutput.setTotalRecordCount(countByDataSource.values().stream().mapToInt(Integer::intValue).sum());
+
       reportStreamer.finish();
 
       try
       {
-         reportInput.getReportOutputStream().close();
+         reportInput.getReportDestination().getReportOutputStream().close();
       }
       catch(Exception e)
       {
          throw (new QReportingException("Error completing report", e));
       }
+
+      return (reportOutput);
    }
 
 
@@ -208,28 +267,48 @@ public class GenerateReportAction
    /*******************************************************************************
     **
     *******************************************************************************/
-   private void startTableView(ReportInput reportInput, QReportDataSource dataSource, QReportView reportView) throws QException
+   private QReportMetaData getReportMetaData(ReportInput reportInput) throws QException
    {
-      QTableMetaData table = reportInput.getInstance().getTable(dataSource.getSourceTable());
+      if(reportInput.getReportMetaData() != null)
+      {
+         return reportInput.getReportMetaData();
+      }
 
+      if(StringUtils.hasContent(reportInput.getReportName()))
+      {
+         return QContext.getQInstance().getReport(reportInput.getReportName());
+      }
+
+      throw (new QReportingException("ReportInput did not contain required parameters to identify the report being generated"));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private void startTableView(ReportInput reportInput, QReportDataSource dataSource, QReportView reportView, ReportFormat reportFormat) throws QException
+   {
       QMetaDataVariableInterpreter variableInterpreter = new QMetaDataVariableInterpreter();
       variableInterpreter.addValueMap("input", reportInput.getInputValues());
 
       ExportInput exportInput = new ExportInput();
-      exportInput.setReportFormat(reportFormat);
-      exportInput.setFilename(reportInput.getFilename());
+      exportInput.setReportDestination(reportInput.getReportDestination());
       exportInput.setTitleRow(getTitle(reportView, variableInterpreter));
       exportInput.setIncludeHeaderRow(reportView.getIncludeHeaderRow());
-      exportInput.setReportOutputStream(reportInput.getReportOutputStream());
 
       JoinsContext joinsContext = null;
-      if(StringUtils.hasContent(dataSource.getSourceTable()))
+      if(dataSource != null)
       {
-         joinsContext = new JoinsContext(exportInput.getInstance(), dataSource.getSourceTable(), dataSource.getQueryJoins(), dataSource.getQueryFilter());
+         if(StringUtils.hasContent(dataSource.getSourceTable()))
+         {
+            joinsContext = new JoinsContext(exportInput.getInstance(), dataSource.getSourceTable(), dataSource.getQueryJoins(), dataSource.getQueryFilter());
+            countDataSourceRecords(reportInput, dataSource, reportFormat);
+         }
       }
 
       List<QFieldMetaData> fields = new ArrayList<>();
-      for(QReportField column : reportView.getColumns())
+      for(QReportField column : CollectionUtils.nonNullList(reportView.getColumns()))
       {
          if(column.getIsVirtual())
          {
@@ -241,7 +320,7 @@ public class GenerateReportAction
             JoinsContext.FieldAndTableNameOrAlias fieldAndTableNameOrAlias = joinsContext == null ? null : joinsContext.getFieldAndTableNameOrAlias(effectiveFieldName);
             if(fieldAndTableNameOrAlias == null || fieldAndTableNameOrAlias.field() == null)
             {
-               throw new QReportingException("Could not find field named [" + effectiveFieldName + "] in dataSource [" + dataSource.getName() + "]");
+               throw new QReportingException("Could not find field named [" + effectiveFieldName + "] in dataSource [" + (dataSource == null ? null : dataSource.getName()) + "]");
             }
 
             QFieldMetaData field = fieldAndTableNameOrAlias.field().clone();
@@ -255,7 +334,7 @@ public class GenerateReportAction
       }
 
       reportStreamer.setDisplayFormats(getDisplayFormatMap(fields));
-      reportStreamer.start(exportInput, fields, reportView.getLabel());
+      reportStreamer.start(exportInput, fields, reportView.getLabel(), reportView);
    }
 
 
@@ -263,7 +342,36 @@ public class GenerateReportAction
    /*******************************************************************************
     **
     *******************************************************************************/
-   private void gatherData(ReportInput reportInput, QReportDataSource dataSource, QReportView tableView, List<QReportView> summaryViews, List<QReportView> variantViews) throws QException
+   private void countDataSourceRecords(ReportInput reportInput, QReportDataSource dataSource, ReportFormat reportFormat) throws QException
+   {
+      QQueryFilter queryFilter = dataSource.getQueryFilter() == null ? new QQueryFilter() : dataSource.getQueryFilter().clone();
+      setInputValuesInQueryFilter(reportInput, queryFilter);
+
+      CountInput countInput = new CountInput();
+      countInput.setTableName(dataSource.getSourceTable());
+      countInput.setFilter(queryFilter);
+      countInput.setQueryJoins(dataSource.getQueryJoins());
+      CountOutput countOutput = new CountAction().execute(countInput);
+
+      if(countOutput.getCount() != null)
+      {
+         countByDataSource.put(dataSource.getName(), countOutput.getCount());
+
+         if(reportFormat.getMaxRows() != null && countOutput.getCount() > reportFormat.getMaxRows())
+         {
+            throw (new QUserFacingException("The requested report would include more rows ("
+               + String.format("%,d", countOutput.getCount()) + ") than the maximum allowed ("
+               + String.format("%,d", reportFormat.getMaxRows()) + ") for the selected file format (" + reportFormat + ")."));
+         }
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private Integer gatherData(ReportInput reportInput, QReportDataSource dataSource, QReportView tableView, List<QReportView> summaryViews, List<QReportView> variantViews) throws QException
    {
       ////////////////////////////////////////////////////////////////////////////////////////
       // check if this view has a transform step - if so, set it up now and run its pre-run //
@@ -290,6 +398,9 @@ public class GenerateReportAction
       RunBackendStepInput   finalTransformStepInput  = transformStepInput;
       RunBackendStepOutput  finalTransformStepOutput = transformStepOutput;
 
+      String        tableLabel    = ObjectUtils.tryElse(() -> QContext.getQInstance().getTable(dataSource.getSourceTable()).getLabel(), Objects.requireNonNullElse(dataSource.getSourceTable(), ""));
+      AtomicInteger consumedCount = new AtomicInteger(0);
+
       /////////////////////////////////////////////////////////////////
       // run a record pipe loop, over the query for this data source //
       /////////////////////////////////////////////////////////////////
@@ -306,6 +417,7 @@ public class GenerateReportAction
             queryInput.setTableName(dataSource.getSourceTable());
             queryInput.setFilter(queryFilter);
             queryInput.setQueryJoins(dataSource.getQueryJoins());
+            queryInput.withQueryHint(QueryInput.QueryHint.POTENTIALLY_LARGE_NUMBER_OF_RESULTS);
 
             queryInput.setShouldTranslatePossibleValues(true);
             queryInput.setFieldsToTranslatePossibleValues(setupFieldsToTranslatePossibleValues(reportInput, dataSource, new JoinsContext(reportInput.getInstance(), dataSource.getSourceTable(), dataSource.getQueryJoins(), queryInput.getFilter())));
@@ -345,10 +457,21 @@ public class GenerateReportAction
          if(finalTransformStep != null)
          {
             finalTransformStepInput.setRecords(records);
-            finalTransformStep.run(finalTransformStepInput, finalTransformStepOutput);
+            finalTransformStep.runOnePage(finalTransformStepInput, finalTransformStepOutput);
             records = finalTransformStepOutput.getRecords();
          }
 
+         Integer total = countByDataSource.get(dataSource.getName());
+         if(total != null)
+         {
+            reportInput.getAsyncJobCallback().updateStatus("Processing " + tableLabel + " records", consumedCount.get() + 1, total);
+         }
+         else
+         {
+            reportInput.getAsyncJobCallback().updateStatus("Processing " + tableLabel + " records (" + String.format("%,d", consumedCount.get() + 1) + ")");
+         }
+         consumedCount.getAndAdd(records.size());
+
          return (consumeRecords(reportInput, dataSource, records, tableView, summaryViews, variantViews));
       });
 
@@ -359,6 +482,8 @@ public class GenerateReportAction
       {
          transformStep.postRun(new BackendStepPostRunInput(transformStepInput), new BackendStepPostRunOutput(transformStepOutput));
       }
+
+      return consumedCount.get();
    }
 
 
@@ -366,11 +491,11 @@ public class GenerateReportAction
    /*******************************************************************************
     **
     *******************************************************************************/
-   private Set<String> setupFieldsToTranslatePossibleValues(ReportInput reportInput, QReportDataSource dataSource, JoinsContext joinsContext)
+   private Set<String> setupFieldsToTranslatePossibleValues(ReportInput reportInput, QReportDataSource dataSource, JoinsContext joinsContext) throws QException
    {
       Set<String> fieldsToTranslatePossibleValues = new HashSet<>();
 
-      for(QReportView view : report.getViews())
+      for(QReportView view : views)
       {
          for(QReportField column : CollectionUtils.nonNullList(view.getColumns()))
          {
@@ -384,15 +509,16 @@ public class GenerateReportAction
             }
          }
 
-         for(String summaryField : CollectionUtils.nonNullList(view.getPivotFields()))
+         for(String summaryFieldName : CollectionUtils.nonNullList(view.getSummaryFields()))
          {
             ///////////////////////////////////////////////////////////////////////////////
             // all pivotFields that are possible value sources are implicitly translated //
             ///////////////////////////////////////////////////////////////////////////////
-            QTableMetaData table = reportInput.getInstance().getTable(dataSource.getSourceTable());
-            if(table.getField(summaryField).getPossibleValueSourceName() != null)
+            QTableMetaData    mainTable         = QContext.getQInstance().getTable(dataSource.getSourceTable());
+            FieldAndJoinTable fieldAndJoinTable = getFieldAndJoinTable(mainTable, summaryFieldName);
+            if(fieldAndJoinTable.field().getPossibleValueSourceName() != null)
             {
-               fieldsToTranslatePossibleValues.add(summaryField);
+               fieldsToTranslatePossibleValues.add(summaryFieldName);
             }
          }
       }
@@ -405,7 +531,33 @@ public class GenerateReportAction
    /*******************************************************************************
     **
     *******************************************************************************/
-   private void setInputValuesInQueryFilter(ReportInput reportInput, QQueryFilter queryFilter)
+   public static FieldAndJoinTable getFieldAndJoinTable(QTableMetaData mainTable, String fieldName) throws QException
+   {
+      if(fieldName.indexOf('.') > -1)
+      {
+         String joinTableName = fieldName.replaceAll("\\..*", "");
+         String joinFieldName = fieldName.replaceAll(".*\\.", "");
+
+         QTableMetaData joinTable = QContext.getQInstance().getTable(joinTableName);
+         if(joinTable == null)
+         {
+            throw (new QException("Unrecognized join table name: " + joinTableName));
+         }
+
+         return new FieldAndJoinTable(joinTable.getField(joinFieldName), joinTable);
+      }
+      else
+      {
+         return new FieldAndJoinTable(mainTable.getField(fieldName), mainTable);
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private void setInputValuesInQueryFilter(ReportInput reportInput, QQueryFilter queryFilter) throws QException
    {
       if(queryFilter == null || queryFilter.getCriteria() == null)
       {
@@ -494,26 +646,27 @@ public class GenerateReportAction
    /*******************************************************************************
     **
     *******************************************************************************/
-   private void addRecordsToSummaryAggregates(QReportView view, QTableMetaData table, List<QRecord> records, Map<String, Map<SummaryKey, Map<String, AggregatesInterface<?>>>> aggregatesMap)
+   private void addRecordsToSummaryAggregates(QReportView view, QTableMetaData table, List<QRecord> records, Map<String, Map<SummaryKey, Map<String, AggregatesInterface<?, ?>>>> aggregatesMap) throws QException
    {
-      Map<SummaryKey, Map<String, AggregatesInterface<?>>> viewAggregates = aggregatesMap.computeIfAbsent(view.getName(), (name) -> new HashMap<>());
+      Map<SummaryKey, Map<String, AggregatesInterface<?, ?>>> viewAggregates = aggregatesMap.computeIfAbsent(view.getName(), (name) -> new HashMap<>());
 
       for(QRecord record : records)
       {
          SummaryKey key = new SummaryKey();
-         for(String summaryField : view.getPivotFields())
+         for(String summaryFieldName : view.getSummaryFields())
          {
-            Serializable summaryValue = record.getValue(summaryField);
-            if(table.getField(summaryField).getPossibleValueSourceName() != null)
+            FieldAndJoinTable fieldAndJoinTable = getFieldAndJoinTable(table, summaryFieldName);
+            Serializable      summaryValue      = record.getValue(summaryFieldName);
+            if(fieldAndJoinTable.field().getPossibleValueSourceName() != null)
             {
                //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
                // so, this is kinda a thing - where we implicitly use possible-value labels (e.g., display values) for pivot fields... //
                //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-               summaryValue = record.getDisplayValue(summaryField);
+               summaryValue = record.getDisplayValue(summaryFieldName);
             }
-            key.add(summaryField, summaryValue);
+            key.add(summaryFieldName, summaryValue);
 
-            if(view.getIncludePivotSubTotals() && key.getKeys().size() < view.getPivotFields().size())
+            if(view.getIncludeSummarySubTotals() && key.getKeys().size() < view.getSummaryFields().size())
             {
                /////////////////////////////////////////////////////////////////////////////////////////
                // be careful here, with these key objects, and their identity, being used as map keys //
@@ -532,9 +685,9 @@ public class GenerateReportAction
    /*******************************************************************************
     **
     *******************************************************************************/
-   private void addRecordToSummaryKeyAggregates(QTableMetaData table, QRecord record, Map<SummaryKey, Map<String, AggregatesInterface<?>>> viewAggregates, SummaryKey key)
+   private void addRecordToSummaryKeyAggregates(QTableMetaData table, QRecord record, Map<SummaryKey, Map<String, AggregatesInterface<?, ?>>> viewAggregates, SummaryKey key) throws QException
    {
-      Map<String, AggregatesInterface<?>> keyAggregates = viewAggregates.computeIfAbsent(key, (name) -> new HashMap<>());
+      Map<String, AggregatesInterface<?, ?>> keyAggregates = viewAggregates.computeIfAbsent(key, (name) -> new HashMap<>());
       addRecordToAggregatesMap(table, record, keyAggregates);
    }
 
@@ -543,23 +696,74 @@ public class GenerateReportAction
    /*******************************************************************************
     **
     *******************************************************************************/
-   private void addRecordToAggregatesMap(QTableMetaData table, QRecord record, Map<String, AggregatesInterface<?>> aggregatesMap)
+   private void addRecordToAggregatesMap(QTableMetaData table, QRecord record, Map<String, AggregatesInterface<?, ?>> aggregatesMap) throws QException
    {
-      for(QFieldMetaData field : table.getFields().values())
+      //////////////////////////////////////////////////////////////////////////////////////
+      // todo - an optimization could be, to only compute aggregates that we'll need...   //
+      // Only if we measure and see this to be slow - it may be, lots of BigDecimal math? //
+      //////////////////////////////////////////////////////////////////////////////////////
+      for(String fieldName : record.getValues().keySet())
       {
-         if(field.getType().equals(QFieldType.INTEGER))
+         QFieldMetaData field = null;
+         try
+         {
+            //////////////////////////////////////////////////////
+            // todo - memoize this, if we ever need to optimize //
+            //////////////////////////////////////////////////////
+            FieldAndJoinTable fieldAndJoinTable = getFieldAndJoinTable(table, fieldName);
+            field = fieldAndJoinTable.field();
+         }
+         catch(Exception e)
+         {
+            //////////////////////////////////////////////////////////////////////////////////////
+            // for non-real-fields... let's skip for now - but maybe treat as string in future? //
+            //////////////////////////////////////////////////////////////////////////////////////
+            LOG.debug("Couldn't find field in table qInstance - won't compute aggregates for it", logPair("fieldName", fieldName));
+            continue;
+         }
+
+         if(StringUtils.hasContent(field.getPossibleValueSourceName()))
          {
             @SuppressWarnings("unchecked")
-            AggregatesInterface<Integer> fieldAggregates = (AggregatesInterface<Integer>) aggregatesMap.computeIfAbsent(field.getName(), (name) -> new IntegerAggregates());
-            fieldAggregates.add(record.getValueInteger(field.getName()));
+            AggregatesInterface<String, ?> fieldAggregates = (AggregatesInterface<String, ?>) aggregatesMap.computeIfAbsent(fieldName, (name) -> new StringAggregates());
+            fieldAggregates.add(record.getDisplayValue(fieldName));
+         }
+         else if(field.getType().equals(QFieldType.INTEGER))
+         {
+            @SuppressWarnings("unchecked")
+            AggregatesInterface<Integer, ?> fieldAggregates = (AggregatesInterface<Integer, ?>) aggregatesMap.computeIfAbsent(fieldName, (name) -> new IntegerAggregates());
+            fieldAggregates.add(record.getValueInteger(fieldName));
+         }
+         else if(field.getType().equals(QFieldType.LONG))
+         {
+            @SuppressWarnings("unchecked")
+            AggregatesInterface<Long, ?> fieldAggregates = (AggregatesInterface<Long, ?>) aggregatesMap.computeIfAbsent(fieldName, (name) -> new LongAggregates());
+            fieldAggregates.add(record.getValueLong(fieldName));
          }
          else if(field.getType().equals(QFieldType.DECIMAL))
          {
             @SuppressWarnings("unchecked")
-            AggregatesInterface<BigDecimal> fieldAggregates = (AggregatesInterface<BigDecimal>) aggregatesMap.computeIfAbsent(field.getName(), (name) -> new BigDecimalAggregates());
-            fieldAggregates.add(record.getValueBigDecimal(field.getName()));
+            AggregatesInterface<BigDecimal, ?> fieldAggregates = (AggregatesInterface<BigDecimal, ?>) aggregatesMap.computeIfAbsent(fieldName, (name) -> new BigDecimalAggregates());
+            fieldAggregates.add(record.getValueBigDecimal(fieldName));
+         }
+         else if(field.getType().equals(QFieldType.DATE_TIME))
+         {
+            @SuppressWarnings("unchecked")
+            AggregatesInterface<Instant, ?> fieldAggregates = (AggregatesInterface<Instant, ?>) aggregatesMap.computeIfAbsent(fieldName, (name) -> new InstantAggregates());
+            fieldAggregates.add(record.getValueInstant(fieldName));
+         }
+         else if(field.getType().equals(QFieldType.DATE))
+         {
+            @SuppressWarnings("unchecked")
+            AggregatesInterface<LocalDate, ?> fieldAggregates = (AggregatesInterface<LocalDate, ?>) aggregatesMap.computeIfAbsent(fieldName, (name) -> new LocalDateAggregates());
+            fieldAggregates.add(record.getValueLocalDate(fieldName));
+         }
+         if(field.getType().isStringLike())
+         {
+            @SuppressWarnings("unchecked")
+            AggregatesInterface<String, ?> fieldAggregates = (AggregatesInterface<String, ?>) aggregatesMap.computeIfAbsent(fieldName, (name) -> new StringAggregates());
+            fieldAggregates.add(record.getValueString(fieldName));
          }
-         // todo - more types (dates, at least?)
       }
    }
 
@@ -568,24 +772,22 @@ public class GenerateReportAction
    /*******************************************************************************
     **
     *******************************************************************************/
-   private void outputSummaries(ReportInput reportInput) throws QReportingException, QFormulaException
+   private void outputSummaries(ReportInput reportInput) throws QException
    {
-      List<QReportView> reportViews = report.getViews().stream().filter(v -> v.getType().equals(ReportType.SUMMARY)).toList();
+      List<QReportView> reportViews = views.stream().filter(v -> v.getType().equals(ReportType.SUMMARY)).toList();
       for(QReportView view : reportViews)
       {
-         QReportDataSource dataSource    = report.getDataSource(view.getDataSourceName());
+         QReportDataSource dataSource    = getDataSource(view.getDataSourceName());
          QTableMetaData    table         = reportInput.getInstance().getTable(dataSource.getSourceTable());
          SummaryOutput     summaryOutput = computeSummaryRowsForView(reportInput, view, table);
 
          ExportInput exportInput = new ExportInput();
-         exportInput.setReportFormat(reportFormat);
-         exportInput.setFilename(reportInput.getFilename());
+         exportInput.setReportDestination(reportInput.getReportDestination());
          exportInput.setTitleRow(summaryOutput.titleRow);
          exportInput.setIncludeHeaderRow(view.getIncludeHeaderRow());
-         exportInput.setReportOutputStream(reportInput.getReportOutputStream());
 
          reportStreamer.setDisplayFormats(getDisplayFormatMap(view));
-         reportStreamer.start(exportInput, getFields(table, view), view.getLabel());
+         reportStreamer.start(exportInput, getFields(table, view), view.getLabel(), view);
 
          reportStreamer.addRecords(summaryOutput.summaryRows); // todo - what if this set is huge?
 
@@ -598,6 +800,24 @@ public class GenerateReportAction
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private QReportDataSource getDataSource(String dataSourceName)
+   {
+      for(QReportDataSource dataSource : CollectionUtils.nonNullList(dataSources))
+      {
+         if(dataSource.getName().equals(dataSourceName))
+         {
+            return (dataSource);
+         }
+      }
+
+      return (null);
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/
@@ -625,13 +845,13 @@ public class GenerateReportAction
    /*******************************************************************************
     **
     *******************************************************************************/
-   private List<QFieldMetaData> getFields(QTableMetaData table, QReportView view)
+   private List<QFieldMetaData> getFields(QTableMetaData table, QReportView view) throws QException
    {
       List<QFieldMetaData> fields = new ArrayList<>();
-      for(String pivotField : view.getPivotFields())
+      for(String summaryFieldName : view.getSummaryFields())
       {
-         QFieldMetaData field = table.getField(pivotField);
-         fields.add(new QFieldMetaData(pivotField, field.getType()).withLabel(field.getLabel())); // todo do we need the type?  if so need table as input here
+         FieldAndJoinTable fieldAndJoinTable = getFieldAndJoinTable(table, summaryFieldName);
+         fields.add(new QFieldMetaData(summaryFieldName, fieldAndJoinTable.field().getType()).withLabel(fieldAndJoinTable.field().getLabel())); // todo do we need the type?  if so need table as input here
       }
       for(QReportField column : view.getColumns())
       {
@@ -661,11 +881,11 @@ public class GenerateReportAction
       // create summary rows //
       /////////////////////////
       List<QRecord> summaryRows = new ArrayList<>();
-      for(Map.Entry<SummaryKey, Map<String, AggregatesInterface<?>>> entry : summaryAggregates.getOrDefault(view.getName(), Collections.emptyMap()).entrySet())
+      for(Map.Entry<SummaryKey, Map<String, AggregatesInterface<?, ?>>> entry : summaryAggregates.getOrDefault(view.getName(), Collections.emptyMap()).entrySet())
       {
-         SummaryKey                          summaryKey      = entry.getKey();
-         Map<String, AggregatesInterface<?>> fieldAggregates = entry.getValue();
-         Map<String, Serializable>           summaryValues   = getSummaryValuesForInterpreter(fieldAggregates);
+         SummaryKey                             summaryKey      = entry.getKey();
+         Map<String, AggregatesInterface<?, ?>> fieldAggregates = entry.getValue();
+         Map<String, Serializable>              summaryValues   = getSummaryValuesForInterpreter(fieldAggregates);
          variableInterpreter.addValueMap("pivot", summaryValues);
          variableInterpreter.addValueMap("summary", summaryValues);
 
@@ -674,9 +894,9 @@ public class GenerateReportAction
 
          if(!varianceAggregates.isEmpty())
          {
-            Map<SummaryKey, Map<String, AggregatesInterface<?>>> varianceMap    = varianceAggregates.getOrDefault(view.getName(), Collections.emptyMap());
-            Map<String, AggregatesInterface<?>>                  varianceSubMap = varianceMap.getOrDefault(summaryKey, Collections.emptyMap());
-            Map<String, Serializable>                            varianceValues = getSummaryValuesForInterpreter(varianceSubMap);
+            Map<SummaryKey, Map<String, AggregatesInterface<?, ?>>> varianceMap    = varianceAggregates.getOrDefault(view.getName(), Collections.emptyMap());
+            Map<String, AggregatesInterface<?, ?>>                  varianceSubMap = varianceMap.getOrDefault(summaryKey, Collections.emptyMap());
+            Map<String, Serializable>                               varianceValues = getSummaryValuesForInterpreter(varianceSubMap);
             variableInterpreter.addValueMap("variancePivot", varianceValues);
             variableInterpreter.addValueMap("variance", varianceValues);
          }
@@ -695,7 +915,7 @@ public class GenerateReportAction
          ///////////////////////////////////////////////////////////////////////////////
          // for summary subtotals, add the text "Total" to the last field in this key //
          ///////////////////////////////////////////////////////////////////////////////
-         if(summaryKey.getKeys().size() < view.getPivotFields().size())
+         if(summaryKey.getKeys().size() < view.getSummaryFields().size())
          {
             String fieldName = summaryKey.getKeys().get(summaryKey.getKeys().size() - 1).getA();
             summaryRow.setValue(fieldName, summaryRow.getValueString(fieldName) + " Total");
@@ -733,11 +953,11 @@ public class GenerateReportAction
       {
          totalRow = new QRecord();
 
-         for(String pivotField : view.getPivotFields())
+         for(String summaryField : view.getSummaryFields())
          {
             if(totalRow.getValues().isEmpty())
             {
-               totalRow.setValue(pivotField, "Totals");
+               totalRow.setValue(summaryField, "Totals");
             }
          }
 
@@ -857,18 +1077,24 @@ public class GenerateReportAction
    /*******************************************************************************
     **
     *******************************************************************************/
-   private Map<String, Serializable> getSummaryValuesForInterpreter(Map<String, AggregatesInterface<?>> fieldAggregates)
+   private Map<String, Serializable> getSummaryValuesForInterpreter(Map<String, AggregatesInterface<?, ?>> fieldAggregates)
    {
       Map<String, Serializable> summaryValuesForInterpreter = new HashMap<>();
-      for(Map.Entry<String, AggregatesInterface<?>> subEntry : fieldAggregates.entrySet())
+      for(Map.Entry<String, AggregatesInterface<?, ?>> subEntry : fieldAggregates.entrySet())
       {
-         String                 fieldName  = subEntry.getKey();
-         AggregatesInterface<?> aggregates = subEntry.getValue();
+         String                    fieldName  = subEntry.getKey();
+         AggregatesInterface<?, ?> aggregates = subEntry.getValue();
          summaryValuesForInterpreter.put("sum." + fieldName, aggregates.getSum());
          summaryValuesForInterpreter.put("count." + fieldName, aggregates.getCount());
+         summaryValuesForInterpreter.put("count_nums." + fieldName, aggregates.getCount());
          summaryValuesForInterpreter.put("min." + fieldName, aggregates.getMin());
          summaryValuesForInterpreter.put("max." + fieldName, aggregates.getMax());
          summaryValuesForInterpreter.put("average." + fieldName, aggregates.getAverage());
+         summaryValuesForInterpreter.put("product." + fieldName, aggregates.getProduct());
+         summaryValuesForInterpreter.put("var." + fieldName, aggregates.getVariance());
+         summaryValuesForInterpreter.put("varp." + fieldName, aggregates.getVarP());
+         summaryValuesForInterpreter.put("std_dev." + fieldName, aggregates.getStandardDeviation());
+         summaryValuesForInterpreter.put("std_devp." + fieldName, aggregates.getStdDevP());
       }
       return summaryValuesForInterpreter;
    }
@@ -882,4 +1108,27 @@ public class GenerateReportAction
    {
    }
 
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public record FieldAndJoinTable(QFieldMetaData field, QTableMetaData joinTable)
+   {
+
+      /*******************************************************************************
+       **
+       *******************************************************************************/
+      public String getLabel(QTableMetaData mainTable)
+      {
+         if(mainTable.getName().equals(joinTable.getName()))
+         {
+            return (field.getLabel());
+         }
+         else
+         {
+            return (joinTable.getLabel() + ": " + field.getLabel());
+         }
+      }
+   }
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/JsonExportStreamer.java

@@ -26,17 +26,23 @@ import java.io.IOException;
 import java.io.OutputStream;
 import java.io.Serializable;
 import java.nio.charset.StandardCharsets;
+import java.util.Arrays;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Optional;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
 import com.kingsrook.qqq.backend.core.exceptions.QReportingException;
 import com.kingsrook.qqq.backend.core.logging.QLogger;
 import com.kingsrook.qqq.backend.core.model.actions.reporting.ExportInput;
+import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportDestination;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportView;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 import com.kingsrook.qqq.backend.core.utils.JsonUtils;
-import com.kingsrook.qqq.backend.core.utils.StringUtils;
+import com.kingsrook.qqq.backend.core.utils.memoization.Memoization;
 
 
 /*******************************************************************************
@@ -46,13 +52,23 @@ public class JsonExportStreamer implements ExportStreamerInterface
 {
    private static final QLogger LOG = QLogger.getLogger(JsonExportStreamer.class);
 
+   private boolean prettyPrint = true;
+
    private ExportInput          exportInput;
    private QTableMetaData       table;
    private List<QFieldMetaData> fields;
    private OutputStream         outputStream;
 
-   private boolean needComma   = false;
-   private boolean prettyPrint = true;
+   private boolean multipleViews       = false;
+   private boolean haveStartedAnyViews = false;
+
+   private boolean needCommaBeforeRecord = false;
+
+   private byte[] indent       = new byte[0];
+   private String indentString = "";
+
+   private Pattern                     colonLetterPattern    = Pattern.compile(":([A-Z]+)($|[A-Z][a-z])");
+   private Memoization<String, String> fieldLabelMemoization = new Memoization<>();
 
 
 
@@ -69,21 +85,124 @@ public class JsonExportStreamer implements ExportStreamerInterface
     **
     *******************************************************************************/
    @Override
-   public void start(ExportInput exportInput, List<QFieldMetaData> fields, String label) throws QReportingException
+   public void preRun(ReportDestination reportDestination, List<QReportView> views) throws QReportingException
+   {
+      outputStream = reportDestination.getReportOutputStream();
+
+      if(views.size() > 1)
+      {
+         multipleViews = true;
+      }
+
+      if(multipleViews)
+      {
+         try
+         {
+            indentIfPretty(outputStream);
+            outputStream.write('[');
+            newlineIfPretty(outputStream);
+            increaseIndent();
+         }
+         catch(IOException e)
+         {
+            throw (new QReportingException("Error starting report output", e));
+         }
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public void start(ExportInput exportInput, List<QFieldMetaData> fields, String label, QReportView view) throws QReportingException
    {
       this.exportInput = exportInput;
       this.fields = fields;
       table = exportInput.getTable();
-      outputStream = this.exportInput.getReportOutputStream();
+
+      needCommaBeforeRecord = false;
 
       try
       {
+         if(multipleViews)
+         {
+            if(haveStartedAnyViews)
+            {
+               /////////////////////////
+               // close the last view //
+               /////////////////////////
+               newlineIfPretty(outputStream);
+
+               decreaseIndent();
+               indentIfPretty(outputStream);
+               outputStream.write(']');
+               newlineIfPretty(outputStream);
+
+               decreaseIndent();
+               indentIfPretty(outputStream);
+               outputStream.write('}');
+               outputStream.write(',');
+               newlineIfPretty(outputStream);
+            }
+
+            /////////////////////////////////////////////////////////////
+            // open a new view, as an object, with a name & data entry //
+            /////////////////////////////////////////////////////////////
+            indentIfPretty(outputStream);
+            outputStream.write('{');
+            newlineIfPretty(outputStream);
+            increaseIndent();
+
+            indentIfPretty(outputStream);
+            outputStream.write(String.format("""
+               "name":"%s",""", label).getBytes(StandardCharsets.UTF_8));
+            newlineIfPretty(outputStream);
+
+            indentIfPretty(outputStream);
+            outputStream.write("""
+               "data":""".getBytes(StandardCharsets.UTF_8));
+            newlineIfPretty(outputStream);
+         }
+
+         //////////////////////////////////////////////
+         // start the array of entries for this view //
+         //////////////////////////////////////////////
+         indentIfPretty(outputStream);
          outputStream.write('[');
+         increaseIndent();
       }
       catch(IOException e)
       {
          throw (new QReportingException("Error starting report output", e));
       }
+
+      haveStartedAnyViews = true;
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private void increaseIndent()
+   {
+      indent = new byte[indent.length + 3];
+      Arrays.fill(indent, (byte) ' ');
+      indentString = new String(indent);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private void decreaseIndent()
+   {
+      indent = new byte[Math.max(0, indent.length - 3)];
+      Arrays.fill(indent, (byte) ' ');
+      indentString = new String(indent);
    }
 
 
@@ -111,7 +230,7 @@ public class JsonExportStreamer implements ExportStreamerInterface
    {
       try
       {
-         if(needComma)
+         if(needCommaBeforeRecord)
          {
             outputStream.write(',');
          }
@@ -119,21 +238,25 @@ public class JsonExportStreamer implements ExportStreamerInterface
          Map<String, Serializable> mapForJson = new LinkedHashMap<>();
          for(QFieldMetaData field : fields)
          {
-            String labelForJson = StringUtils.lcFirst(field.getLabel().replace(" ", ""));
-            mapForJson.put(labelForJson, qRecord.getValue(field.getName()));
+            mapForJson.put(getLabelForJson(field), qRecord.getValue(field.getName()));
          }
 
          String json = prettyPrint ? JsonUtils.toPrettyJson(mapForJson) : JsonUtils.toJson(mapForJson);
+         if(prettyPrint)
+         {
+            json = json.replaceAll("(?s)\n", "\n" + indentString);
+         }
 
          if(prettyPrint)
          {
             outputStream.write('\n');
          }
 
+         indentIfPretty(outputStream);
          outputStream.write(json.getBytes(StandardCharsets.UTF_8));
 
          outputStream.flush(); // todo - less often?
-         needComma = true;
+         needCommaBeforeRecord = true;
       }
       catch(Exception e)
       {
@@ -143,6 +266,73 @@ public class JsonExportStreamer implements ExportStreamerInterface
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   String getLabelForJson(QFieldMetaData field)
+   {
+      //////////////////////////////////////////////////////////////////////////
+      // memoize, to avoid running these regex/replacements millions of times //
+      //////////////////////////////////////////////////////////////////////////
+      Optional<String> result = fieldLabelMemoization.getResult(field.getName(), fieldName ->
+      {
+         String labelForJson = field.getLabel().replace(" ", "");
+
+         /////////////////////////////////////////////////////////////////////////////
+         // now fix up any field-name-parts after the table: portion of a join name //
+         // lineItem:SKU to become lineItem:sku                                     //
+         // parcel:SLAStatus to become parcel:slaStatus                             //
+         // order:Client to become order:client                                     //
+         /////////////////////////////////////////////////////////////////////////////
+         Matcher allCaps                          = Pattern.compile("^[A-Z]+$").matcher(labelForJson);
+         Matcher startsAllCapsThenNextWordMatcher = Pattern.compile("([A-Z]+)([A-Z][a-z].*)").matcher(labelForJson);
+         Matcher startsOneCapMatcher              = Pattern.compile("([A-Z])(.*)").matcher(labelForJson);
+
+         if(allCaps.matches())
+         {
+            labelForJson = allCaps.replaceAll(m -> m.group().toLowerCase());
+         }
+         else if(startsAllCapsThenNextWordMatcher.matches())
+         {
+            labelForJson = startsAllCapsThenNextWordMatcher.replaceAll(m -> m.group(1).toLowerCase() + m.group(2));
+         }
+         else if(startsOneCapMatcher.matches())
+         {
+            labelForJson = startsOneCapMatcher.replaceAll(m -> m.group(1).toLowerCase() + m.group(2));
+         }
+
+         /////////////////////////////////////////////////////////////////////////////
+         // now fix up any field-name-parts after the table: portion of a join name //
+         // lineItem:SKU to become lineItem:sku                                     //
+         // parcel:SLAStatus to become parcel:slaStatus                             //
+         // order:Client to become order:client                                     //
+         /////////////////////////////////////////////////////////////////////////////
+         Matcher colonThenAllCapsThenEndMatcher      = Pattern.compile("(.*:)([A-Z]+)$").matcher(labelForJson);
+         Matcher colonThenAllCapsThenNextWordMatcher = Pattern.compile("(.*:)([A-Z]+)([A-Z][a-z].*)").matcher(labelForJson);
+         Matcher colonThenOneCapMatcher              = Pattern.compile("(.*:)([A-Z])(.*)").matcher(labelForJson);
+
+         if(colonThenAllCapsThenEndMatcher.matches())
+         {
+            labelForJson = colonThenAllCapsThenEndMatcher.replaceAll(m -> m.group(1) + m.group(2).toLowerCase());
+         }
+         else if(colonThenAllCapsThenNextWordMatcher.matches())
+         {
+            labelForJson = colonThenAllCapsThenNextWordMatcher.replaceAll(m -> m.group(1) + m.group(2).toLowerCase() + m.group(3));
+         }
+         else if(colonThenOneCapMatcher.matches())
+         {
+            labelForJson = colonThenOneCapMatcher.replaceAll(m -> m.group(1) + m.group(2).toLowerCase() + m.group(3));
+         }
+
+         System.out.println("Label: " + labelForJson);
+         return (labelForJson);
+      });
+
+      return result.orElse(field.getName());
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/
@@ -162,11 +352,34 @@ public class JsonExportStreamer implements ExportStreamerInterface
    {
       try
       {
-         if(prettyPrint)
-         {
-            outputStream.write('\n');
-         }
+         //////////////////////////////////////////////
+         // close the array of entries for this view //
+         //////////////////////////////////////////////
+         newlineIfPretty(outputStream);
+
+         decreaseIndent();
+         indentIfPretty(outputStream);
          outputStream.write(']');
+         newlineIfPretty(outputStream);
+
+         if(multipleViews)
+         {
+            ////////////////////////////////////////////
+            // close this view, if there are multiple //
+            ////////////////////////////////////////////
+            decreaseIndent();
+            indentIfPretty(outputStream);
+            outputStream.write('}');
+            newlineIfPretty(outputStream);
+
+            /////////////////////////////
+            // close the list of views //
+            /////////////////////////////
+            decreaseIndent();
+            indentIfPretty(outputStream);
+            outputStream.write(']');
+            newlineIfPretty(outputStream);
+         }
       }
       catch(IOException e)
       {
@@ -174,4 +387,30 @@ public class JsonExportStreamer implements ExportStreamerInterface
       }
    }
 
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private void newlineIfPretty(OutputStream outputStream) throws IOException
+   {
+      if(prettyPrint)
+      {
+         outputStream.write('\n');
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private void indentIfPretty(OutputStream outputStream) throws IOException
+   {
+      if(prettyPrint)
+      {
+         outputStream.write(indent);
+      }
+   }
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/ListOfMapsExportStreamer.java

@@ -31,6 +31,7 @@ import com.kingsrook.qqq.backend.core.logging.QLogger;
 import com.kingsrook.qqq.backend.core.model.actions.reporting.ExportInput;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportView;
 
 
 /*******************************************************************************
@@ -87,7 +88,7 @@ public class ListOfMapsExportStreamer implements ExportStreamerInterface
     **
     *******************************************************************************/
    @Override
-   public void start(ExportInput exportInput, List<QFieldMetaData> fields, String label) throws QReportingException
+   public void start(ExportInput exportInput, List<QFieldMetaData> fields, String label, QReportView view) throws QReportingException
    {
       this.exportInput = exportInput;
       this.fields = fields;

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/RecordPipe.java

@@ -24,6 +24,7 @@ package com.kingsrook.qqq.backend.core.actions.reporting;
 
 import java.util.ArrayList;
 import java.util.List;
+import java.util.Objects;
 import java.util.concurrent.ArrayBlockingQueue;
 import java.util.concurrent.TimeUnit;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
@@ -43,8 +44,9 @@ public class RecordPipe
 
    private static final long BLOCKING_SLEEP_MILLIS = 100;
    private static final long MAX_SLEEP_LOOP_MILLIS = 300_000; // 5 minutes
+   private static final int  DEFAULT_CAPACITY      = 1_000;
 
-   private int capacity = 1_000;
+   private int capacity = DEFAULT_CAPACITY;
    private ArrayBlockingQueue<QRecord> queue = new ArrayBlockingQueue<>(capacity);
 
    private boolean isTerminated = false;
@@ -70,11 +72,13 @@ public class RecordPipe
 
    /*******************************************************************************
     ** Construct a record pipe, with an alternative capacity for the internal queue.
+    **
+    ** overrideCapacity is allowed to be null - in which case, DEFAULT_CAPACITY is used.
     *******************************************************************************/
    public RecordPipe(Integer overrideCapacity)
    {
-      this.capacity = overrideCapacity;
-      queue = new ArrayBlockingQueue<>(overrideCapacity);
+      this.capacity = Objects.requireNonNullElse(overrideCapacity, DEFAULT_CAPACITY);
+      queue = new ArrayBlockingQueue<>(this.capacity);
    }
 
 
@@ -138,7 +142,7 @@ public class RecordPipe
          {
             if(now - sleepLoopStartTime > MAX_SLEEP_LOOP_MILLIS)
             {
-               LOG.warn("Giving up adding record to pipe, due to pipe being full for more than {} millis", MAX_SLEEP_LOOP_MILLIS);
+               LOG.warn("Giving up adding record to pipe, due to pipe being full for more than " + MAX_SLEEP_LOOP_MILLIS + " millis");
                throw (new IllegalStateException("Giving up adding record to pipe, due to pipe staying full too long."));
             }
             LOG.trace("Record pipe.add failed (due to full pipe).  Blocking.");

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/ReportUtils.java

@@ -0,0 +1,51 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.actions.reporting;
+
+
+import java.util.List;
+import java.util.Optional;
+import com.kingsrook.qqq.backend.core.exceptions.QReportingException;
+import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportView;
+
+
+/*******************************************************************************
+ **
+ *******************************************************************************/
+public class ReportUtils
+{
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public static QReportView getSourceViewForPivotTableView(List<QReportView> views, QReportView pivotTableView) throws QReportingException
+   {
+      Optional<QReportView> sourceView = views.stream().filter(v -> v.getName().equals(pivotTableView.getPivotTableSourceViewName())).findFirst();
+      if(sourceView.isEmpty())
+      {
+         throw (new QReportingException("Could not find data view [" + pivotTableView.getPivotTableSourceViewName() + "] for pivot table view [" + pivotTableView.getName() + "]"));
+      }
+
+      return sourceView.get();
+   }
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/excel/fastexcel/BoldHeaderAndFooterFastExcelStyler.java

@@ -1,6 +1,6 @@
 /*
  * QQQ - Low-code Application Framework for Engineers.
- * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
  * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
  * contact@kingsrook.com
  * https://github.com/Kingsrook/
@@ -19,7 +19,7 @@
  * along with this program.  If not, see <https://www.gnu.org/licenses/>.
  */
 
-package com.kingsrook.qqq.backend.core.actions.reporting.excelformatting;
+package com.kingsrook.qqq.backend.core.actions.reporting.excel.fastexcel;
 
 
 import org.dhatim.fastexcel.BorderSide;
@@ -30,7 +30,7 @@ import org.dhatim.fastexcel.StyleSetter;
 /*******************************************************************************
  ** Version of excel styler that does bold headers and footers, with basic borders.
  *******************************************************************************/
-public class BoldHeaderAndFooterExcelStyler implements ExcelStylerInterface
+public class BoldHeaderAndFooterFastExcelStyler implements FastExcelStylerInterface
 {
 
    /*******************************************************************************
@@ -60,6 +60,9 @@ public class BoldHeaderAndFooterExcelStyler implements ExcelStylerInterface
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
    @Override
    public void styleTotalsRow(StyleSetter totalsRowStyle)
    {

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/excel/fastexcel/ExcelFastexcelExportStreamer.java

@@ -1,6 +1,6 @@
 /*
  * QQQ - Low-code Application Framework for Engineers.
- * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
  * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
  * contact@kingsrook.com
  * https://github.com/Kingsrook/
@@ -19,7 +19,7 @@
  * along with this program.  If not, see <https://www.gnu.org/licenses/>.
  */
 
-package com.kingsrook.qqq.backend.core.actions.reporting;
+package com.kingsrook.qqq.backend.core.actions.reporting.excel.fastexcel;
 
 
 import java.io.OutputStream;
@@ -34,8 +34,8 @@ import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
-import com.kingsrook.qqq.backend.core.actions.reporting.excelformatting.ExcelStylerInterface;
-import com.kingsrook.qqq.backend.core.actions.reporting.excelformatting.PlainExcelStyler;
+import com.kingsrook.qqq.backend.core.actions.reporting.ExportStreamerInterface;
+import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.exceptions.QReportingException;
 import com.kingsrook.qqq.backend.core.logging.QLogger;
 import com.kingsrook.qqq.backend.core.model.actions.reporting.ExportInput;
@@ -43,7 +43,9 @@ import com.kingsrook.qqq.backend.core.model.data.QRecord;
 import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.DisplayFormat;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportView;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+import com.kingsrook.qqq.backend.core.utils.ObjectUtils;
 import com.kingsrook.qqq.backend.core.utils.StringUtils;
 import com.kingsrook.qqq.backend.core.utils.ValueUtils;
 import org.dhatim.fastexcel.StyleSetter;
@@ -52,19 +54,19 @@ import org.dhatim.fastexcel.Worksheet;
 
 
 /*******************************************************************************
- ** Excel export format implementation
+ ** Excel export format implementation - built using fastexcel library
  *******************************************************************************/
-public class ExcelExportStreamer implements ExportStreamerInterface
+public class ExcelFastexcelExportStreamer implements ExportStreamerInterface
 {
-   private static final QLogger LOG = QLogger.getLogger(ExcelExportStreamer.class);
+   private static final QLogger LOG = QLogger.getLogger(ExcelFastexcelExportStreamer.class);
 
    private ExportInput          exportInput;
    private QTableMetaData       table;
    private List<QFieldMetaData> fields;
    private OutputStream         outputStream;
 
-   private ExcelStylerInterface excelStylerInterface = new PlainExcelStyler();
-   private Map<String, String>  excelCellFormats;
+   private FastExcelStylerInterface fastExcelStylerInterface = new PlainFastExcelStyler();
+   private Map<String, String>      excelCellFormats;
 
    private Workbook  workbook;
    private Worksheet worksheet;
@@ -76,7 +78,7 @@ public class ExcelExportStreamer implements ExportStreamerInterface
    /*******************************************************************************
     **
     *******************************************************************************/
-   public ExcelExportStreamer()
+   public ExcelFastexcelExportStreamer()
    {
    }
 
@@ -105,14 +107,14 @@ public class ExcelExportStreamer implements ExportStreamerInterface
     ** Starts a new worksheet in the current workbook.  Can be called multiple times.
     *******************************************************************************/
    @Override
-   public void start(ExportInput exportInput, List<QFieldMetaData> fields, String label) throws QReportingException
+   public void start(ExportInput exportInput, List<QFieldMetaData> fields, String label, QReportView view) throws QReportingException
    {
       try
       {
          this.exportInput = exportInput;
          this.fields = fields;
          table = exportInput.getTable();
-         outputStream = this.exportInput.getReportOutputStream();
+         outputStream = this.exportInput.getReportDestination().getReportOutputStream();
          this.row = 0;
          this.sheetCount++;
 
@@ -121,7 +123,7 @@ public class ExcelExportStreamer implements ExportStreamerInterface
          /////////////////////////////////////////////////////////////////////////////////////////////////////
          if(workbook == null)
          {
-            String    appName  = "QQQ";
+            String    appName = ObjectUtils.tryAndRequireNonNullElse(() -> QContext.getQInstance().getBranding().getAppName(), "QQQ");
             QInstance instance = exportInput.getInstance();
             if(instance != null && instance.getBranding() != null && instance.getBranding().getCompanyName() != null)
             {
@@ -167,7 +169,7 @@ public class ExcelExportStreamer implements ExportStreamerInterface
             worksheet.range(row, 0, row, fields.size() - 1).merge();
 
             StyleSetter titleStyle = worksheet.range(row, 0, row, fields.size() - 1).style();
-            excelStylerInterface.styleTitleRow(titleStyle);
+            fastExcelStylerInterface.styleTitleRow(titleStyle);
             titleStyle.set();
 
             row++;
@@ -187,7 +189,7 @@ public class ExcelExportStreamer implements ExportStreamerInterface
             }
 
             StyleSetter headerStyle = worksheet.range(row, 0, row, fields.size() - 1).style();
-            excelStylerInterface.styleHeaderRow(headerStyle);
+            fastExcelStylerInterface.styleHeaderRow(headerStyle);
             headerStyle.set();
 
             row++;
@@ -315,7 +317,7 @@ public class ExcelExportStreamer implements ExportStreamerInterface
       writeRecord(record);
 
       StyleSetter totalsRowStyle = worksheet.range(row, 0, row, fields.size() - 1).style();
-      excelStylerInterface.styleTotalsRow(totalsRowStyle);
+      fastExcelStylerInterface.styleTotalsRow(totalsRowStyle);
       totalsRowStyle.set();
    }
 

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/excel/fastexcel/FastExcelStylerInterface.java

@@ -1,6 +1,6 @@
 /*
  * QQQ - Low-code Application Framework for Engineers.
- * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
  * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
  * contact@kingsrook.com
  * https://github.com/Kingsrook/
@@ -19,7 +19,7 @@
  * along with this program.  If not, see <https://www.gnu.org/licenses/>.
  */
 
-package com.kingsrook.qqq.backend.core.actions.reporting.excelformatting;
+package com.kingsrook.qqq.backend.core.actions.reporting.excel.fastexcel;
 
 
 import org.dhatim.fastexcel.StyleSetter;
@@ -29,7 +29,7 @@ import org.dhatim.fastexcel.StyleSetter;
  ** Interface for classes that know how to apply styles to an Excel stream being
  ** built by fastexcel.
  *******************************************************************************/
-public interface ExcelStylerInterface
+public interface FastExcelStylerInterface
 {
 
    /*******************************************************************************

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/excel/fastexcel/PlainFastExcelStyler.java

@@ -0,0 +1,43 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.actions.reporting.excel.fastexcel;
+
+
+import org.dhatim.fastexcel.StyleSetter;
+
+
+/*******************************************************************************
+ ** Excel styler that does nothing - just takes defaults (which are all no-op) from the interface.
+ *******************************************************************************/
+public class PlainFastExcelStyler implements FastExcelStylerInterface
+{
+
+   /*******************************************************************************
+    ** ... sorry, but adding this gives us test coverage on this class, even though
+    ** we're just deferring to super...
+    *******************************************************************************/
+   @Override
+   public void styleHeaderRow(StyleSetter headerRowStyle)
+   {
+      FastExcelStylerInterface.super.styleHeaderRow(headerRowStyle);
+   }
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/excel/poi/BoldHeaderAndFooterPoiExcelStyler.java

@@ -0,0 +1,93 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.actions.reporting.excel.poi;
+
+
+import org.apache.poi.ss.usermodel.BorderStyle;
+import org.apache.poi.ss.usermodel.CreationHelper;
+import org.apache.poi.ss.usermodel.Font;
+import org.apache.poi.ss.usermodel.HorizontalAlignment;
+import org.apache.poi.xssf.usermodel.XSSFCellStyle;
+import org.apache.poi.xssf.usermodel.XSSFWorkbook;
+
+
+/*******************************************************************************
+ ** Version of POI excel styler that does bold headers and footers, with basic borders.
+ *******************************************************************************/
+public class BoldHeaderAndFooterPoiExcelStyler implements PoiExcelStylerInterface
+{
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public XSSFCellStyle createStyleForTitle(XSSFWorkbook workbook, CreationHelper createHelper)
+   {
+      Font font = workbook.createFont();
+      font.setFontHeightInPoints((short) 14);
+      font.setBold(true);
+
+      XSSFCellStyle cellStyle = workbook.createCellStyle();
+      cellStyle.setFont(font);
+      cellStyle.setAlignment(HorizontalAlignment.CENTER);
+
+      return (cellStyle);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public XSSFCellStyle createStyleForHeader(XSSFWorkbook workbook, CreationHelper createHelper)
+   {
+      Font font = workbook.createFont();
+      font.setBold(true);
+
+      XSSFCellStyle cellStyle = workbook.createCellStyle();
+      cellStyle.setFont(font);
+      cellStyle.setBorderBottom(BorderStyle.THIN);
+
+      return (cellStyle);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public XSSFCellStyle createStyleForFooter(XSSFWorkbook workbook, CreationHelper createHelper)
+   {
+      Font font = workbook.createFont();
+      font.setBold(true);
+
+      XSSFCellStyle cellStyle = workbook.createCellStyle();
+      cellStyle.setFont(font);
+      cellStyle.setBorderTop(BorderStyle.THIN);
+      cellStyle.setBorderBottom(BorderStyle.DOUBLE);
+
+      return (cellStyle);
+   }
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/excel/poi/ExcelPoiBasedStreamingExportStreamer.java

@@ -0,0 +1,819 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.actions.reporting.excel.poi;
+
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.io.OutputStreamWriter;
+import java.io.Serializable;
+import java.io.Writer;
+import java.time.Instant;
+import java.time.LocalDate;
+import java.time.LocalDateTime;
+import java.time.ZoneId;
+import java.time.ZonedDateTime;
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipInputStream;
+import java.util.zip.ZipOutputStream;
+import com.kingsrook.qqq.backend.core.actions.reporting.ExportStreamerInterface;
+import com.kingsrook.qqq.backend.core.actions.reporting.ReportUtils;
+import com.kingsrook.qqq.backend.core.exceptions.QReportingException;
+import com.kingsrook.qqq.backend.core.instances.QInstanceEnricher;
+import com.kingsrook.qqq.backend.core.logging.QLogger;
+import com.kingsrook.qqq.backend.core.model.actions.reporting.ExportInput;
+import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportDestination;
+import com.kingsrook.qqq.backend.core.model.actions.reporting.pivottable.PivotTableGroupBy;
+import com.kingsrook.qqq.backend.core.model.actions.reporting.pivottable.PivotTableValue;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.DisplayFormat;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
+import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportField;
+import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportView;
+import com.kingsrook.qqq.backend.core.model.metadata.reporting.ReportType;
+import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
+import com.kingsrook.qqq.backend.core.utils.StringUtils;
+import com.kingsrook.qqq.backend.core.utils.ValueUtils;
+import org.apache.poi.ss.SpreadsheetVersion;
+import org.apache.poi.ss.usermodel.CreationHelper;
+import org.apache.poi.ss.usermodel.DataConsolidateFunction;
+import org.apache.poi.ss.usermodel.DateUtil;
+import org.apache.poi.ss.util.AreaReference;
+import org.apache.poi.ss.util.CellReference;
+import org.apache.poi.ss.util.WorkbookUtil;
+import org.apache.poi.xssf.usermodel.XSSFCell;
+import org.apache.poi.xssf.usermodel.XSSFCellStyle;
+import org.apache.poi.xssf.usermodel.XSSFPivotTable;
+import org.apache.poi.xssf.usermodel.XSSFRow;
+import org.apache.poi.xssf.usermodel.XSSFSheet;
+import org.apache.poi.xssf.usermodel.XSSFWorkbook;
+
+
+/*******************************************************************************
+ ** Excel export format implementation using POI library, but with modifications
+ ** to actually stream output rather than use any temp files.
+ **
+ ** For a rough outline:
+ ** - create a basically empty Excel workbook using POI - empty meaning, without
+ ** data rows.
+ ** - have POI write that workbook out into a byte[] - which will be a zip full
+ ** of xml (e.g., xlsx).
+ ** - then open a new ZipOutputStream wrapper around the OutputStream we took in
+ ** as the report destination (e.g., streamed into storage or http output)
+ ** - Copy over all entries from the xlsx into our new zip-output-stream, other than
+ ** ones that are the actual sheets that we want to put data into.
+ ** - For the sheet entries, use the StreamedPoiSheetWriter class to write the
+ ** report's data directly out as Excel XML (not using POI).
+ ** - Pivot tables require a bit of an additional hack - to write a "pivot cache
+ ** definition", which, while we won't put all the data in it (because we'll tell
+ ** it refreshOnLoad="true"), we will at least need to know how many cols & rows
+ ** are in the data-sheet (which we wouldn't know until we streamed that sheet!)
+ *******************************************************************************/
+public class ExcelPoiBasedStreamingExportStreamer implements ExportStreamerInterface
+{
+   private static final QLogger LOG = QLogger.getLogger(ExcelPoiBasedStreamingExportStreamer.class);
+
+   private List<QReportView>    views;
+   private ExportInput          exportInput;
+   private List<QFieldMetaData> fields;
+   private OutputStream         outputStream;
+   private ZipOutputStream      zipOutputStream;
+
+   public static final String EXCEL_DATE_FORMAT      = "yyyy-MM-dd";
+   public static final String EXCEL_DATE_TIME_FORMAT = "yyyy-MM-dd H:mm:ss";
+
+   private PoiExcelStylerInterface    poiExcelStylerInterface = getStylerInterface();
+   private Map<String, String>        excelCellFormats;
+   private Map<String, XSSFCellStyle> styles                  = new HashMap<>();
+
+   private int rowNo      = 0;
+   private int sheetIndex = 1;
+
+   private Map<String, String> pivotViewToCacheDefinitionReferenceMap = new HashMap<>();
+
+   private Writer              activeSheetWriter = null;
+   private StreamedSheetWriter sheetWriter       = null;
+
+   private QReportView                       currentView      = null;
+   private Map<String, List<QFieldMetaData>> fieldsPerView    = new HashMap<>();
+   private Map<String, Integer>              rowsPerView      = new HashMap<>();
+   private Map<String, String>               labelViewsByName = new HashMap<>();
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public ExcelPoiBasedStreamingExportStreamer()
+   {
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public void preRun(ReportDestination reportDestination, List<QReportView> views) throws QReportingException
+   {
+      try
+      {
+         this.outputStream = reportDestination.getReportOutputStream();
+         this.views = views;
+
+         ///////////////////////////////////////////////////////////////////////////////
+         // create 'template' workbook through poi - with sheets corresponding to our //
+         // actual file this will be a zip file (stream), with entries for all of the //
+         // files in the final xlsx but without any data, so it'll be small           //
+         ///////////////////////////////////////////////////////////////////////////////
+         XSSFWorkbook workbook = new XSSFWorkbook();
+         createStyles(workbook);
+
+         //////////////////////////////////////////////////////////////////////////////////////////////////
+         // for each of the sheets, create it in the workbook, and put a reference to it in the sheetMap //
+         //////////////////////////////////////////////////////////////////////////////////////////////////
+         Map<String, XSSFSheet> sheetMapByExcelReference = new HashMap<>();
+         Map<String, XSSFSheet> sheetMapByViewName       = new HashMap<>();
+
+         int sheetCounter = 1;
+         for(QReportView view : views)
+         {
+            String label = Objects.requireNonNullElse(view.getLabel(), "Sheet " + sheetCounter);
+            label = WorkbookUtil.createSafeSheetName(label);
+
+            /////////////////////////////////////////////////////////////////////////////////////////////
+            // track the actually-used sheet labels (needed for referencing in pivot table generation) //
+            /////////////////////////////////////////////////////////////////////////////////////////////
+            labelViewsByName.put(view.getName(), label);
+
+            XSSFSheet sheet          = workbook.createSheet(label);
+            String    sheetReference = sheet.getPackagePart().getPartName().getName().substring(1);
+            sheetMapByExcelReference.put(sheetReference, sheet);
+            sheetMapByViewName.put(view.getName(), sheet);
+            sheetCounter++;
+         }
+
+         ////////////////////////////////////////////////////
+         // if any views are pivot tables, create them now //
+         ////////////////////////////////////////////////////
+         List<String> pivotViewNames = new ArrayList<>();
+         for(QReportView view : views)
+         {
+            if(ReportType.PIVOT.equals(view.getType()))
+            {
+               pivotViewNames.add(view.getName());
+
+               XSSFSheet   pivotTableSheet = Objects.requireNonNull(sheetMapByViewName.get(view.getName()), "Could not get pivot table sheet view by name: " + view.getName());
+               XSSFSheet   dataSheet       = Objects.requireNonNull(sheetMapByViewName.get(view.getPivotTableSourceViewName()), "Could not get pivot table source sheet by view name: " + view.getPivotTableSourceViewName());
+               QReportView dataView        = ReportUtils.getSourceViewForPivotTableView(views, view);
+               createPivotTableTemplate(pivotTableSheet, view, dataSheet, dataView);
+            }
+         }
+         Iterator<String> pivotViewNameIterator = pivotViewNames.iterator();
+
+         /////////////////////////////////////////////////////////
+         // write that template worksheet zip out to byte array //
+         /////////////////////////////////////////////////////////
+         ByteArrayOutputStream templateBAOS = new ByteArrayOutputStream();
+         workbook.write(templateBAOS);
+         templateBAOS.close();
+         byte[] templateBytes = templateBAOS.toByteArray();
+
+         /////////////////////////////////////////////////////////////////////////////////////////////
+         // open up a zipOutputStream around the output stream that the report is to be written to. //
+         // note:  this stream is closed in the finish method - see more comments there.            //
+         /////////////////////////////////////////////////////////////////////////////////////////////
+         this.zipOutputStream = new ZipOutputStream(this.outputStream);
+
+         /////////////////////////////////////////////////////////////////////////////////////////////////
+         // copy over all the entries in the template zip that aren't the sheets into the output stream //
+         /////////////////////////////////////////////////////////////////////////////////////////////////
+         ZipInputStream zipInputStream   = new ZipInputStream(new ByteArrayInputStream(templateBytes));
+         ZipEntry       zipTemplateEntry = null;
+         byte[]         buffer           = new byte[2048];
+         while((zipTemplateEntry = zipInputStream.getNextEntry()) != null)
+         {
+            if(zipTemplateEntry.getName().matches(".*/pivotCacheDefinition.*.xml"))
+            {
+               //////////////////////////////////////////////////////////////////////////////////////////////////////
+               // if this zip entry is a pivotCacheDefinition, then don't write it to the output stream right now. //
+               // instead, just map the pivot view's name to the zipTemplateEntry name                             //
+               //////////////////////////////////////////////////////////////////////////////////////////////////////
+               if(!pivotViewNameIterator.hasNext())
+               {
+                  throw new QReportingException("Found a pivot cache definition [" + zipTemplateEntry.getName() + "] in the template ZIP, but no (more) corresponding pivot view names");
+               }
+
+               String pivotViewName = pivotViewNameIterator.next();
+               LOG.info("Holding on a pivot cache definition zip template entry [" + pivotViewName + "] [" + zipTemplateEntry.getName() + "]...");
+               pivotViewToCacheDefinitionReferenceMap.put(pivotViewName, zipTemplateEntry.getName());
+            }
+            else if(!sheetMapByExcelReference.containsKey(zipTemplateEntry.getName()))
+            {
+               ////////////////////////////////////////////////////////////////////////////////////////////////////////////
+               // else, if we don't have this zipTemplateEntry name in our map of sheets, then this is a kinda "meta"    //
+               // file that we don't really care about (e.g., not our sheet data), so just copy it to the output stream. //
+               ////////////////////////////////////////////////////////////////////////////////////////////////////////////
+               LOG.info("Copying zip template entry [" + zipTemplateEntry.getName() + "] to output stream");
+               zipOutputStream.putNextEntry(new ZipEntry(zipTemplateEntry.getName()));
+
+               int length;
+               while((length = zipInputStream.read(buffer)) > 0)
+               {
+                  zipOutputStream.write(buffer, 0, length);
+               }
+
+               zipInputStream.closeEntry();
+            }
+            else
+            {
+               ////////////////////////////////////////////////////////////////////////////////////
+               // else - this is a sheet - so again, don't write it yet - stream its data below. //
+               ////////////////////////////////////////////////////////////////////////////////////
+               LOG.info("Skipping presumed sheet zip template entry [" + zipTemplateEntry.getName() + "] to output stream");
+            }
+         }
+
+         zipInputStream.close();
+      }
+      catch(Exception e)
+      {
+         throw (new QReportingException("Error preparing to generate spreadsheet", e));
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private void createPivotTableTemplate(XSSFSheet pivotTableSheet, QReportView pivotView, XSSFSheet dataSheet, QReportView dataView) throws QReportingException
+   {
+      /////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      // write just enough data to the dataView's sheet so that we can refer to it for creating the pivot table. //
+      // we need to do this, because POI will try to create the pivotCache referring to the data sheet, and if   //
+      // there isn't any data there, it'll crash.                                                                //
+      /////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      XSSFRow headerRow = dataSheet.createRow(0);
+      int     columnNo  = 0;
+      for(QReportField column : dataView.getColumns())
+      {
+         XSSFCell cell = headerRow.createCell(columnNo++);
+         cell.setCellValue(QInstanceEnricher.nameToLabel(column.getName()));
+      }
+
+      XSSFRow valuesRow = dataSheet.createRow(1);
+      columnNo = 0;
+      for(QReportField column : dataView.getColumns())
+      {
+         XSSFCell cell = valuesRow.createCell(columnNo++);
+         cell.setCellValue("Value " + columnNo);
+      }
+
+      /////////////////////////////////////////////////////////////////////////////////////////////////////
+      // for this template version of the pivot table, tell it there are only 2 rows in the source sheet //
+      // as that's all that we wrote above (a header and 1 fake value row)                               //
+      /////////////////////////////////////////////////////////////////////////////////////////////////////
+      int           rows       = 2;
+      String        colsLetter = CellReference.convertNumToColString(dataView.getColumns().size() - 1);
+      AreaReference source     = new AreaReference("A1:" + colsLetter + rows, SpreadsheetVersion.EXCEL2007);
+      CellReference position   = new CellReference("A1");
+
+      //////////////////////////////////////////////////////////////////
+      // tell poi all about our pivot table - rows, cols, and columns //
+      //////////////////////////////////////////////////////////////////
+      XSSFPivotTable pivotTable = pivotTableSheet.createPivotTable(source, position, dataSheet);
+
+      for(PivotTableGroupBy row : CollectionUtils.nonNullList(pivotView.getPivotTableDefinition().getRows()))
+      {
+         int rowLabelColumnIndex = getColumnIndex(dataView.getColumns(), row.getFieldName());
+         pivotTable.addRowLabel(rowLabelColumnIndex);
+      }
+
+      for(PivotTableGroupBy column : CollectionUtils.nonNullList(pivotView.getPivotTableDefinition().getColumns()))
+      {
+         int colLabelColumnIndex = getColumnIndex(dataView.getColumns(), column.getFieldName());
+         pivotTable.addColLabel(colLabelColumnIndex);
+      }
+
+      for(PivotTableValue value : CollectionUtils.nonNullList(pivotView.getPivotTableDefinition().getValues()))
+      {
+         int columnLabelColumnIndex = getColumnIndex(dataView.getColumns(), value.getFieldName());
+
+         String labelPrefix = value.getFunction().name() + " of ";
+         String label       = labelPrefix + QInstanceEnricher.nameToLabel(value.getFieldName());
+         String valueFormat = null;
+
+         Optional<QReportField> optSourceField = dataView.getColumns().stream().filter(c -> c.getName().equals(value.getFieldName())).findFirst();
+         if(optSourceField.isPresent())
+         {
+            QReportField sourceField = optSourceField.get();
+
+            if(StringUtils.hasContent(sourceField.getLabel()))
+            {
+               label = labelPrefix + sourceField.getLabel();
+            }
+
+            if(StringUtils.hasContent(sourceField.getDisplayFormat()))
+            {
+               valueFormat = DisplayFormat.getExcelFormat(sourceField.getDisplayFormat());
+            }
+            else
+            {
+               if(QFieldType.DATE.equals(sourceField.getType()))
+               {
+                  valueFormat = EXCEL_DATE_FORMAT;
+               }
+               else if(QFieldType.DATE_TIME.equals(sourceField.getType()))
+               {
+                  valueFormat = EXCEL_DATE_TIME_FORMAT;
+               }
+            }
+         }
+
+         pivotTable.addColumnLabel(DataConsolidateFunction.valueOf(value.getFunction().name()), columnLabelColumnIndex, label, valueFormat);
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private int getColumnIndex(List<QReportField> columns, String fieldName) throws QReportingException
+   {
+      for(int i = 0; i < columns.size(); i++)
+      {
+         if(columns.get(i).getName().equals(fieldName))
+         {
+            return (i);
+         }
+      }
+
+      throw (new QReportingException("Could not find column by name [" + fieldName + "]"));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private void createStyles(XSSFWorkbook workbook)
+   {
+      CreationHelper createHelper = workbook.getCreationHelper();
+
+      XSSFCellStyle dateStyle = workbook.createCellStyle();
+      dateStyle.setDataFormat(createHelper.createDataFormat().getFormat(EXCEL_DATE_FORMAT));
+      styles.put("date", dateStyle);
+
+      XSSFCellStyle dateTimeStyle = workbook.createCellStyle();
+      dateTimeStyle.setDataFormat(createHelper.createDataFormat().getFormat(EXCEL_DATE_TIME_FORMAT));
+      styles.put("datetime", dateTimeStyle);
+
+      styles.put("title", poiExcelStylerInterface.createStyleForTitle(workbook, createHelper));
+      styles.put("header", poiExcelStylerInterface.createStyleForHeader(workbook, createHelper));
+      styles.put("footer", poiExcelStylerInterface.createStyleForFooter(workbook, createHelper));
+
+      XSSFCellStyle footerDateStyle = poiExcelStylerInterface.createStyleForFooter(workbook, createHelper);
+      footerDateStyle.setDataFormat(createHelper.createDataFormat().getFormat(EXCEL_DATE_FORMAT));
+      styles.put("footer-date", footerDateStyle);
+
+      XSSFCellStyle footerDateTimeStyle = poiExcelStylerInterface.createStyleForFooter(workbook, createHelper);
+      footerDateTimeStyle.setDataFormat(createHelper.createDataFormat().getFormat(EXCEL_DATE_TIME_FORMAT));
+      styles.put("footer-datetime", footerDateTimeStyle);
+   }
+
+
+
+   /*******************************************************************************
+    ** Starts a new worksheet in the current workbook.  Can be called multiple times.
+    *******************************************************************************/
+   @Override
+   public void start(ExportInput exportInput, List<QFieldMetaData> fields, String label, QReportView view) throws QReportingException
+   {
+      try
+      {
+         /////////////////////////////////////////
+         // close previous sheet if one is open //
+         /////////////////////////////////////////
+         closeLastSheetIfOpen();
+
+         if(currentView != null)
+         {
+            this.rowsPerView.put(currentView.getName(), rowNo);
+         }
+
+         this.currentView = view;
+         this.exportInput = exportInput;
+         this.fields = fields;
+         this.rowNo = 0;
+
+         this.fieldsPerView.put(view.getName(), fields);
+
+         //////////////////////////////////////////
+         // start the new sheet as:              //
+         // - a new entry in the zipOutputStream //
+         // - with a new output stream writer    //
+         // - and with a SpreadsheetWriter       //
+         //////////////////////////////////////////
+         zipOutputStream.putNextEntry(new ZipEntry("xl/worksheets/sheet" + this.sheetIndex++ + ".xml"));
+         activeSheetWriter = new OutputStreamWriter(zipOutputStream);
+         sheetWriter = new StreamedSheetWriter(activeSheetWriter);
+
+         if(ReportType.PIVOT.equals(view.getType()))
+         {
+            writePivotTable(view, ReportUtils.getSourceViewForPivotTableView(views, view));
+         }
+         else
+         {
+            sheetWriter.beginSheet();
+
+            ////////////////////////////////////////////////
+            // put the title and header rows in the sheet //
+            ////////////////////////////////////////////////
+            writeTitleAndHeader();
+         }
+      }
+      catch(Exception e)
+      {
+         throw (new QReportingException("Error starting worksheet", e));
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private void writeTitleAndHeader() throws QReportingException
+   {
+      try
+      {
+         ///////////////
+         // title row //
+         ///////////////
+         if(StringUtils.hasContent(exportInput.getTitleRow()))
+         {
+            sheetWriter.insertRow(rowNo++);
+            sheetWriter.createCell(0, exportInput.getTitleRow(), styles.get("title").getIndex());
+            sheetWriter.endRow();
+         }
+
+         ////////////////
+         // header row //
+         ////////////////
+         if(exportInput.getIncludeHeaderRow())
+         {
+            sheetWriter.insertRow(rowNo++);
+            XSSFCellStyle headerStyle = styles.get("header");
+
+            int col = 0;
+            for(QFieldMetaData column : fields)
+            {
+               sheetWriter.createCell(col, column.getLabel(), headerStyle.getIndex());
+               col++;
+            }
+
+            sheetWriter.endRow();
+         }
+      }
+      catch(Exception e)
+      {
+         throw (new QReportingException("Error starting Excel report"));
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public void addRecords(List<QRecord> qRecords) throws QReportingException
+   {
+      LOG.info("Consuming [" + qRecords.size() + "] records from the pipe");
+
+      try
+      {
+         for(QRecord qRecord : qRecords)
+         {
+            writeRecord(qRecord);
+         }
+      }
+      catch(Exception e)
+      {
+         LOG.error("Exception generating excel file", e);
+         try
+         {
+            outputStream.close();
+         }
+         catch(IOException ex)
+         {
+            LOG.warn("Secondary error closing excel output stream", e);
+         }
+
+         throw (new QReportingException("Error generating Excel report", e));
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private void writeRecord(QRecord qRecord) throws IOException
+   {
+      writeRecord(qRecord, false);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private void writeRecord(QRecord qRecord, boolean isFooter) throws IOException
+   {
+      sheetWriter.insertRow(rowNo++);
+
+      int styleIndex         = -1;
+      int dateStyleIndex     = styles.get("date").getIndex();
+      int dateTimeStyleIndex = styles.get("datetime").getIndex();
+      if(isFooter)
+      {
+         styleIndex = styles.get("footer").getIndex();
+         dateStyleIndex = styles.get("footer-date").getIndex();
+         dateTimeStyleIndex = styles.get("footer-datetime").getIndex();
+      }
+
+      int col = 0;
+      for(QFieldMetaData field : fields)
+      {
+         Serializable value = qRecord.getValue(field.getName());
+
+         if(value != null)
+         {
+            if(value instanceof String s)
+            {
+               sheetWriter.createCell(col, s, styleIndex);
+            }
+            else if(value instanceof Number n)
+            {
+               sheetWriter.createCell(col, n.doubleValue(), styleIndex);
+
+               if(excelCellFormats != null)
+               {
+                  String format = excelCellFormats.get(field.getName());
+                  if(format != null)
+                  {
+                     ////////////////////////////////////////////////////////////////////////////////////////////
+                     // todo - so - for this streamed/zip approach, we need to know all styles before we start //
+                     // any sheets.  but, right now Report action only calls us with per-sheet styles when     //
+                     // it's starting individual sheets.  so, we can't quite support this at this time.        //
+                     // "just" need to change GenerateReportAction to look up all cell formats for all sheets  //
+                     // before preRun is called... and change all existing streamer classes to handle that too //
+                     ////////////////////////////////////////////////////////////////////////////////////////////
+                     // worksheet.style(rowNo, col).format(format).set();
+                  }
+               }
+            }
+            else if(value instanceof Boolean b)
+            {
+               sheetWriter.createCell(col, b, styleIndex);
+            }
+            else if(value instanceof Date d)
+            {
+               sheetWriter.createCell(col, DateUtil.getExcelDate(d), dateStyleIndex);
+            }
+            else if(value instanceof LocalDate d)
+            {
+               sheetWriter.createCell(col, DateUtil.getExcelDate(d), dateStyleIndex);
+            }
+            else if(value instanceof LocalDateTime d)
+            {
+               sheetWriter.createCell(col, DateUtil.getExcelDate(d), dateStyleIndex);
+            }
+            else if(value instanceof ZonedDateTime d)
+            {
+               sheetWriter.createCell(col, DateUtil.getExcelDate(d.toLocalDateTime()), dateTimeStyleIndex);
+            }
+            else if(value instanceof Instant i)
+            {
+               sheetWriter.createCell(col, DateUtil.getExcelDate(i.atZone(ZoneId.systemDefault()).toLocalDateTime()), dateTimeStyleIndex);
+            }
+            else
+            {
+               sheetWriter.createCell(col, ValueUtils.getValueAsString(value), styleIndex);
+            }
+         }
+
+         col++;
+      }
+
+      sheetWriter.endRow();
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public void addTotalsRow(QRecord record) throws QReportingException
+   {
+      try
+      {
+         writeRecord(record, true);
+      }
+      catch(Exception e)
+      {
+         throw (new QReportingException("Error adding totals row", e));
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public void finish() throws QReportingException
+   {
+      try
+      {
+         //////////////////////////////////////////////
+         // close the last open sheet if one is open //
+         //////////////////////////////////////////////
+         closeLastSheetIfOpen();
+
+         /////////////////////////////////////////////////////////////////////////////////////
+         // so, we DO need a close here, on the zipOutputStream, to finish its "zippiness". //
+         // even though, doing so also closes the outputStream from the caller that this    //
+         // zipOutputStream is wrapping (and the caller will likely call close on too)...   //
+         /////////////////////////////////////////////////////////////////////////////////////
+         zipOutputStream.close();
+      }
+      catch(Exception e)
+      {
+         throw (new QReportingException("Error finishing Excel report", e));
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private void closeLastSheetIfOpen() throws IOException
+   {
+      //////////////////////////////////////////////////////////////////////////////////////////////////////
+      // if we have an active sheet writer:                                                               //
+      // - end the current sheet in the spreadsheet writer (write some closing xml, unless it's a pivot!) //
+      // - flush the contents through the activeSheetWriter                                               //
+      // - close the zip entry in the output stream.                                                      //
+      //////////////////////////////////////////////////////////////////////////////////////////////////////
+      if(activeSheetWriter != null)
+      {
+         if(!ReportType.PIVOT.equals(currentView.getType()))
+         {
+            sheetWriter.endSheet();
+         }
+
+         activeSheetWriter.flush();
+         zipOutputStream.closeEntry();
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** display formats is a map of field name to Excel format strings (e.g., $#,##0.00)
+    *******************************************************************************/
+   @Override
+   public void setDisplayFormats(Map<String, String> displayFormats)
+   {
+      this.excelCellFormats = new HashMap<>();
+      for(Map.Entry<String, String> entry : displayFormats.entrySet())
+      {
+         String excelFormat = DisplayFormat.getExcelFormat(entry.getValue());
+         if(excelFormat != null)
+         {
+            excelCellFormats.put(entry.getKey(), excelFormat);
+         }
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private void writePivotTable(QReportView pivotTableView, QReportView dataView) throws QReportingException
+   {
+      try
+      {
+         //////////////////////////////////////////////////////////////////////////////////
+         // write the xml file that is the pivot table sheet.                            //
+         // note that the ZipEntry here will have been started above in the start method //
+         //////////////////////////////////////////////////////////////////////////////////
+         activeSheetWriter.write("""
+            <?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+              <worksheet xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main" xmlns:r="http://schemas.openxmlformats.org/officeDocument/2006/relationships" xmlns:mx="http://schemas.microsoft.com/office/mac/excel/2008/main" xmlns:mc="http://schemas.openxmlformats.org/markup-compatibility/2006" xmlns:mv="urn:schemas-microsoft-com:mac:vml" xmlns:x14="http://schemas.microsoft.com/office/spreadsheetml/2009/9/main" xmlns:x15="http://schemas.microsoft.com/office/spreadsheetml/2010/11/main" xmlns:x14ac="http://schemas.microsoft.com/office/spreadsheetml/2009/9/ac" xmlns:xm="http://schemas.microsoft.com/office/excel/2006/main">
+                 <sheetPr>
+                   <outlinePr summaryBelow="0" summaryRight="0"/>
+                 </sheetPr>
+                 <sheetViews>
+                   <sheetView workbookViewId="0"/>
+                 </sheetViews>
+                 <sheetFormatPr customHeight="1" defaultColWidth="14.43" defaultRowHeight="15.0"/>
+                 <sheetData>
+                   <row r="1"/>
+                 </sheetData>
+              </worksheet>
+            """);
+         activeSheetWriter.flush();
+
+         ////////////////////////////////////////////////////////////////////////////
+         // start a new zip entry, for this pivot view's cacheDefinition reference //
+         ////////////////////////////////////////////////////////////////////////////
+         zipOutputStream.putNextEntry(new ZipEntry(pivotViewToCacheDefinitionReferenceMap.get(pivotTableView.getName())));
+
+         /////////////////////////////////////////////////////////
+         // prepare the xml for each field (e.g., w/ its label) //
+         /////////////////////////////////////////////////////////
+         List<String> cachedFieldElements = new ArrayList<>();
+         for(QFieldMetaData column : this.fieldsPerView.get(dataView.getName()))
+         {
+            cachedFieldElements.add(String.format("""
+               <cacheField numFmtId="0" name="%s">
+                  <sharedItems/>
+               </cacheField>
+               """, column.getLabel()));
+         }
+
+         /////////////////////////////////////////////////////////////////////////////////////
+         // write the xml file that is the pivot cache definition (structure only, no data) //
+         /////////////////////////////////////////////////////////////////////////////////////
+         activeSheetWriter = new OutputStreamWriter(zipOutputStream);
+         activeSheetWriter.write(String.format("""
+               <?xml version="1.0" encoding="UTF-8"?>
+               <pivotCacheDefinition xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main" xmlns:r="http://schemas.openxmlformats.org/officeDocument/2006/relationships" createdVersion="3" minRefreshableVersion="3" refreshedVersion="3" refreshedBy="Apache POI" refreshedDate="1.711395767702E12" refreshOnLoad="true" r:id="rId1">
+                 <cacheSource type="worksheet">
+                   <worksheetSource sheet="%s" ref="A1:%s%d"/>
+                 </cacheSource>
+                 <cacheFields count="%d">
+                   %s
+                 </cacheFields>
+               </pivotCacheDefinition>
+               """,
+            StreamedSheetWriter.cleanseValue(labelViewsByName.get(dataView.getName())),
+            CellReference.convertNumToColString(dataView.getColumns().size() - 1),
+            rowsPerView.get(dataView.getName()),
+            dataView.getColumns().size(),
+            StringUtils.join("\n", cachedFieldElements)));
+      }
+      catch(Exception e)
+      {
+         throw (new QReportingException("Error writing pivot table", e));
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   protected PoiExcelStylerInterface getStylerInterface()
+   {
+      return (new PlainPoiExcelStyler());
+   }
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/excel/poi/PlainPoiExcelStyler.java

@@ -0,0 +1,46 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.actions.reporting.excel.poi;
+
+
+import org.apache.poi.ss.usermodel.CreationHelper;
+import org.apache.poi.xssf.usermodel.XSSFCellStyle;
+import org.apache.poi.xssf.usermodel.XSSFWorkbook;
+
+
+/*******************************************************************************
+ ** Excel styler that does nothing - just takes defaults (which are all no-op) from the interface.
+ *******************************************************************************/
+public class PlainPoiExcelStyler implements PoiExcelStylerInterface
+{
+
+   /*******************************************************************************
+    ** ... sorry, but adding this gives us test coverage on this class, even though
+    ** we're just deferring to super...
+    *******************************************************************************/
+   @Override
+   public XSSFCellStyle createStyleForHeader(XSSFWorkbook workbook, CreationHelper createHelper)
+   {
+      return PoiExcelStylerInterface.super.createStyleForHeader(workbook, createHelper);
+   }
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/excel/poi/PoiExcelStylerInterface.java

@@ -0,0 +1,59 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.actions.reporting.excel.poi;
+
+
+import org.apache.poi.ss.usermodel.CreationHelper;
+import org.apache.poi.xssf.usermodel.XSSFCellStyle;
+import org.apache.poi.xssf.usermodel.XSSFWorkbook;
+
+
+/*******************************************************************************
+ ** Interface for classes that know how to apply styles to an Excel stream being
+ ** built by POI.
+ *******************************************************************************/
+public interface PoiExcelStylerInterface
+{
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   default XSSFCellStyle createStyleForTitle(XSSFWorkbook workbook, CreationHelper createHelper)
+   {
+      return (workbook.createCellStyle());
+   }
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   default XSSFCellStyle createStyleForHeader(XSSFWorkbook workbook, CreationHelper createHelper)
+   {
+      return (workbook.createCellStyle());
+   }
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   default XSSFCellStyle createStyleForFooter(XSSFWorkbook workbook, CreationHelper createHelper)
+   {
+      return (workbook.createCellStyle());
+   }
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/excel/poi/StreamedSheetWriter.java

@@ -0,0 +1,242 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.actions.reporting.excel.poi;
+
+
+import java.io.IOException;
+import java.io.Writer;
+import java.util.HashMap;
+import java.util.Map;
+import org.apache.poi.ss.util.CellReference;
+
+
+/*******************************************************************************
+ ** Write excel formatted XML to a Writer.
+ ** Originally from https://coderanch.com/t/548897/java/Generate-large-excel-POI
+ *******************************************************************************/
+public class StreamedSheetWriter
+{
+   private final Writer writer;
+   private       int    rowNo;
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public StreamedSheetWriter(Writer writer)
+   {
+      this.writer = writer;
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public void beginSheet() throws IOException
+   {
+      writer.write("""
+         <?xml version="1.0" encoding="UTF-8"?>
+            <worksheet xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main">
+               <sheetData>""");
+
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public void endSheet() throws IOException
+   {
+      writer.write("""
+            </sheetData>
+         </worksheet>""");
+   }
+
+
+
+   /*******************************************************************************
+    ** Insert a new row
+    **
+    ** @param rowNo 0-based row number
+    *******************************************************************************/
+   public void insertRow(int rowNo) throws IOException
+   {
+      writer.write("<row r=\"" + (rowNo + 1) + "\">\n");
+      this.rowNo = rowNo;
+   }
+
+
+
+   /*******************************************************************************
+    ** Insert row end marker
+    *******************************************************************************/
+   public void endRow() throws IOException
+   {
+      writer.write("</row>\n");
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public void createCell(int columnIndex, String value, int styleIndex) throws IOException
+   {
+      String ref = new CellReference(rowNo, columnIndex).formatAsString();
+      writer.write("<c r=\"" + ref + "\" t=\"inlineStr\"");
+      if(styleIndex != -1)
+      {
+         writer.write(" s=\"" + styleIndex + "\"");
+      }
+
+      String cleanValue = cleanseValue(value);
+
+      writer.write(">");
+      writer.write("<is><t>" + cleanValue + "</t></is>");
+      writer.write("</c>");
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public static String cleanseValue(String value)
+   {
+      if(value != null)
+      {
+         StringBuilder rs = new StringBuilder();
+         for(int i = 0; i < value.length(); i++)
+         {
+            char c = value.charAt(i);
+            if(c == '&')
+            {
+               rs.append("&amp;");
+            }
+            else if(c == '<')
+            {
+               rs.append("&lt;");
+            }
+            else if(c == '>')
+            {
+               rs.append("&gt;");
+            }
+            else if(c == '\'')
+            {
+               rs.append("&apos;");
+            }
+            else if(c == '"')
+            {
+               rs.append("&quot;");
+            }
+            else if (c < 32 && c != '\t' && c != '\n')
+            {
+               rs.append(' ');
+            }
+            else
+            {
+               rs.append(c);
+            }
+         }
+
+         Map<String, Integer> m = new HashMap();
+         m.computeIfAbsent("s", (s) -> 3);
+
+         value = rs.toString();
+      }
+
+      return (value);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public void createCell(int columnIndex, String value) throws IOException
+   {
+      createCell(columnIndex, value, -1);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public void createCell(int columnIndex, double value, int styleIndex) throws IOException
+   {
+      String ref = new CellReference(rowNo, columnIndex).formatAsString();
+      writer.write("<c r=\"" + ref + "\" t=\"n\"");
+      if(styleIndex != -1)
+      {
+         writer.write(" s=\"" + styleIndex + "\"");
+      }
+      writer.write(">");
+      writer.write("<v>" + value + "</v>");
+      writer.write("</c>");
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public void createCell(int columnIndex, double value) throws IOException
+   {
+      createCell(columnIndex, value, -1);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public void createCell(int columnIndex, Boolean value) throws IOException
+   {
+      createCell(columnIndex, value, -1);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public void createCell(int columnIndex, Boolean value, int styleIndex) throws IOException
+   {
+      String ref = new CellReference(rowNo, columnIndex).formatAsString();
+      writer.write("<c r=\"" + ref + "\" t=\"b\"");
+      if(styleIndex != -1)
+      {
+         writer.write(" s=\"" + styleIndex + "\"");
+      }
+      writer.write(">");
+      if(value != null)
+      {
+         writer.write("<v>" + (value ? 1 : 0) + "</v>");
+      }
+      writer.write("</c>");
+   }
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/DeleteAction.java

@@ -35,9 +35,8 @@ import java.util.Set;
 import java.util.stream.Collectors;
 import com.kingsrook.qqq.backend.core.actions.ActionHelper;
 import com.kingsrook.qqq.backend.core.actions.audits.DMLAuditAction;
-import com.kingsrook.qqq.backend.core.actions.customizers.AbstractPostDeleteCustomizer;
-import com.kingsrook.qqq.backend.core.actions.customizers.AbstractPreDeleteCustomizer;
 import com.kingsrook.qqq.backend.core.actions.customizers.QCodeLoader;
+import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizerInterface;
 import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizers;
 import com.kingsrook.qqq.backend.core.actions.interfaces.DeleteInterface;
 import com.kingsrook.qqq.backend.core.actions.tables.helpers.ValidateRecordSecurityLockHelper;
@@ -250,7 +249,7 @@ public class DeleteAction
       //////////////////////////////////////////////////////////////
       // finally, run the post-delete customizer, if there is one //
       //////////////////////////////////////////////////////////////
-      Optional<AbstractPostDeleteCustomizer> postDeleteCustomizer = QCodeLoader.getTableCustomizer(AbstractPostDeleteCustomizer.class, table, TableCustomizers.POST_DELETE_RECORD.getRole());
+      Optional<TableCustomizerInterface> postDeleteCustomizer = QCodeLoader.getTableCustomizer(table, TableCustomizers.POST_DELETE_RECORD.getRole());
       if(postDeleteCustomizer.isPresent() && oldRecordList.isPresent())
       {
          ////////////////////////////////////////////////////////////////////////////
@@ -260,8 +259,7 @@ public class DeleteAction
 
          try
          {
-            postDeleteCustomizer.get().setDeleteInput(deleteInput);
-            List<QRecord> postCustomizerResult = postDeleteCustomizer.get().apply(recordsForCustomizer);
+            List<QRecord> postCustomizerResult = postDeleteCustomizer.get().postDelete(deleteInput, recordsForCustomizer);
 
             ///////////////////////////////////////////////////////
             // check if any records got errors in the customizer //
@@ -327,13 +325,11 @@ public class DeleteAction
       ///////////////////////////////////////////////////////////////////////////
       // after all validations, run the pre-delete customizer, if there is one //
       ///////////////////////////////////////////////////////////////////////////
-      Optional<AbstractPreDeleteCustomizer> preDeleteCustomizer = QCodeLoader.getTableCustomizer(AbstractPreDeleteCustomizer.class, table, TableCustomizers.PRE_DELETE_RECORD.getRole());
+      Optional<TableCustomizerInterface> preDeleteCustomizer = QCodeLoader.getTableCustomizer(table, TableCustomizers.PRE_DELETE_RECORD.getRole());
       List<QRecord>                         customizerResult    = oldRecordList.get();
       if(preDeleteCustomizer.isPresent())
       {
-         preDeleteCustomizer.get().setDeleteInput(deleteInput);
-         preDeleteCustomizer.get().setIsPreview(isPreview);
-         customizerResult = preDeleteCustomizer.get().apply(oldRecordList.get());
+         customizerResult = preDeleteCustomizer.get().preDelete(deleteInput, oldRecordList.get(), isPreview);
       }
 
       /////////////////////////////////////////////////////////////////////////
@@ -474,7 +470,7 @@ public class DeleteAction
                QRecord recordWithError = new QRecord();
                recordsWithErrors.add(recordWithError);
                recordWithError.setValue(primaryKeyField.getName(), primaryKeyValue);
-               recordWithError.addError(new NotFoundStatusMessage("No record was found to delete for " + primaryKeyField.getLabel() + " = " + primaryKeyValue));
+               recordWithError.addError(new NotFoundStatusMessage("No record was found to delete for " + Objects.requireNonNullElse(primaryKeyField.getLabel(), primaryKeyField.getName()) + " = " + primaryKeyValue));
             }
          }
       }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/GetAction.java

@@ -27,8 +27,8 @@ import java.util.List;
 import java.util.Map;
 import java.util.Optional;
 import com.kingsrook.qqq.backend.core.actions.ActionHelper;
-import com.kingsrook.qqq.backend.core.actions.customizers.AbstractPostQueryCustomizer;
 import com.kingsrook.qqq.backend.core.actions.customizers.QCodeLoader;
+import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizerInterface;
 import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizers;
 import com.kingsrook.qqq.backend.core.actions.interfaces.GetInterface;
 import com.kingsrook.qqq.backend.core.actions.tables.helpers.GetActionCacheHelper;
@@ -49,6 +49,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleDispatcher;
 import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleInterface;
+import com.kingsrook.qqq.backend.core.utils.ObjectUtils;
 
 
 /*******************************************************************************
@@ -57,7 +58,7 @@ import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleInterface;
  *******************************************************************************/
 public class GetAction
 {
-   private Optional<AbstractPostQueryCustomizer> postGetRecordCustomizer;
+   private Optional<TableCustomizerInterface> postGetRecordCustomizer;
 
    private GetInput                 getInput;
    private QPossibleValueTranslator qPossibleValueTranslator;
@@ -87,7 +88,7 @@ public class GetAction
          throw (new QException("Requested to Get a record from an unrecognized table: " + getInput.getTableName()));
       }
 
-      postGetRecordCustomizer = QCodeLoader.getTableCustomizer(AbstractPostQueryCustomizer.class, table, TableCustomizers.POST_QUERY_RECORD.getRole());
+      postGetRecordCustomizer = QCodeLoader.getTableCustomizer(table, TableCustomizers.POST_QUERY_RECORD.getRole());
       this.getInput = getInput;
 
       QBackendModuleDispatcher qBackendModuleDispatcher = new QBackendModuleDispatcher();
@@ -107,9 +108,11 @@ public class GetAction
       }
 
       GetOutput getOutput;
+      boolean usingDefaultGetInterface = false;
       if(getInterface == null)
       {
          getInterface = new DefaultGetInterface();
+         usingDefaultGetInterface = true;
       }
 
       getInterface.validateInput(getInput);
@@ -123,10 +126,11 @@ public class GetAction
          new GetActionCacheHelper().handleCaching(getInput, getOutput);
       }
 
-      ////////////////////////////////////////////////////////
-      // if the record is found, perform post-actions on it //
-      ////////////////////////////////////////////////////////
-      if(getOutput.getRecord() != null)
+      ////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      // if the record is found, perform post-actions on it                                                         //
+      // unless the defaultGetInterface was used - as it just does a query, and the query will do the post-actions. //
+      ////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      if(getOutput.getRecord() != null && !usingDefaultGetInterface)
       {
          getOutput.setRecord(postRecordActions(getOutput.getRecord()));
       }
@@ -202,7 +206,7 @@ public class GetAction
       }
       else
       {
-         throw (new QException("No primaryKey or uniqueKey was passed to Get"));
+         throw (new QException("Unable to get " + ObjectUtils.tryElse(() -> queryInput.getTable().getLabel(), queryInput.getTableName()) + ".  Missing required input."));
       }
 
       queryInput.setFilter(filter);
@@ -216,12 +220,12 @@ public class GetAction
     ** Run the necessary actions on a record.  This may include setting display values,
     ** translating possible values, and running post-record customizations.
     *******************************************************************************/
-   public QRecord postRecordActions(QRecord record)
+   public QRecord postRecordActions(QRecord record) throws QException
    {
       QRecord returnRecord = record;
       if(this.postGetRecordCustomizer.isPresent())
       {
-         returnRecord = postGetRecordCustomizer.get().apply(List.of(record)).get(0);
+         returnRecord = postGetRecordCustomizer.get().postQuery(getInput, List.of(record)).get(0);
       }
 
       if(getInput.getShouldTranslatePossibleValues())

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/InsertAction.java

@@ -28,6 +28,7 @@ import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
+import java.util.Objects;
 import java.util.Optional;
 import java.util.Set;
 import java.util.stream.Collectors;
@@ -37,9 +38,9 @@ import com.kingsrook.qqq.backend.core.actions.QBackendTransaction;
 import com.kingsrook.qqq.backend.core.actions.audits.DMLAuditAction;
 import com.kingsrook.qqq.backend.core.actions.automation.AutomationStatus;
 import com.kingsrook.qqq.backend.core.actions.automation.RecordAutomationStatusUpdater;
-import com.kingsrook.qqq.backend.core.actions.customizers.AbstractPostInsertCustomizer;
 import com.kingsrook.qqq.backend.core.actions.customizers.AbstractPreInsertCustomizer;
 import com.kingsrook.qqq.backend.core.actions.customizers.QCodeLoader;
+import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizerInterface;
 import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizers;
 import com.kingsrook.qqq.backend.core.actions.interfaces.InsertInterface;
 import com.kingsrook.qqq.backend.core.actions.tables.helpers.UniqueKeyHelper;
@@ -61,6 +62,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.tables.Association;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.UniqueKey;
 import com.kingsrook.qqq.backend.core.model.statusmessages.BadInputStatusMessage;
+import com.kingsrook.qqq.backend.core.model.statusmessages.DuplicateKeyBadInputStatusMessage;
 import com.kingsrook.qqq.backend.core.model.statusmessages.QWarningMessage;
 import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleDispatcher;
 import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleInterface;
@@ -168,13 +170,12 @@ public class InsertAction extends AbstractQActionFunction<InsertInput, InsertOut
       //////////////////////////////////////////////////////////////
       // finally, run the post-insert customizer, if there is one //
       //////////////////////////////////////////////////////////////
-      Optional<AbstractPostInsertCustomizer> postInsertCustomizer = QCodeLoader.getTableCustomizer(AbstractPostInsertCustomizer.class, table, TableCustomizers.POST_INSERT_RECORD.getRole());
+      Optional<TableCustomizerInterface> postInsertCustomizer = QCodeLoader.getTableCustomizer(table, TableCustomizers.POST_INSERT_RECORD.getRole());
       if(postInsertCustomizer.isPresent())
       {
          try
          {
-            postInsertCustomizer.get().setInsertInput(insertInput);
-            insertOutput.setRecords(postInsertCustomizer.get().apply(insertOutput.getRecords()));
+            insertOutput.setRecords(postInsertCustomizer.get().postInsert(insertInput, insertOutput.getRecords()));
          }
          catch(Exception e)
          {
@@ -232,31 +233,29 @@ public class InsertAction extends AbstractQActionFunction<InsertInput, InsertOut
       // load the pre-insert customizer and set it up, if there is one //
       // then we'll run it based on its WhenToRun value                //
       ///////////////////////////////////////////////////////////////////
-      Optional<AbstractPreInsertCustomizer> preInsertCustomizer = QCodeLoader.getTableCustomizer(AbstractPreInsertCustomizer.class, table, TableCustomizers.PRE_INSERT_RECORD.getRole());
+      Optional<TableCustomizerInterface> preInsertCustomizer = QCodeLoader.getTableCustomizer(table, TableCustomizers.PRE_INSERT_RECORD.getRole());
       if(preInsertCustomizer.isPresent())
       {
-         preInsertCustomizer.get().setInsertInput(insertInput);
-         preInsertCustomizer.get().setIsPreview(isPreview);
-         runPreInsertCustomizerIfItIsTime(insertInput, preInsertCustomizer, AbstractPreInsertCustomizer.WhenToRun.BEFORE_ALL_VALIDATIONS);
+         runPreInsertCustomizerIfItIsTime(insertInput, isPreview, preInsertCustomizer, AbstractPreInsertCustomizer.WhenToRun.BEFORE_ALL_VALIDATIONS);
       }
 
       setDefaultValuesInRecords(table, insertInput.getRecords());
 
-      ValueBehaviorApplier.applyFieldBehaviors(ValueBehaviorApplier.Action.INSERT, insertInput.getInstance(), table, insertInput.getRecords());
+      ValueBehaviorApplier.applyFieldBehaviors(ValueBehaviorApplier.Action.INSERT, insertInput.getInstance(), table, insertInput.getRecords(), null);
 
-      runPreInsertCustomizerIfItIsTime(insertInput, preInsertCustomizer, AbstractPreInsertCustomizer.WhenToRun.BEFORE_UNIQUE_KEY_CHECKS);
+      runPreInsertCustomizerIfItIsTime(insertInput, isPreview, preInsertCustomizer, AbstractPreInsertCustomizer.WhenToRun.BEFORE_UNIQUE_KEY_CHECKS);
       setErrorsIfUniqueKeyErrors(insertInput, table);
 
-      runPreInsertCustomizerIfItIsTime(insertInput, preInsertCustomizer, AbstractPreInsertCustomizer.WhenToRun.BEFORE_REQUIRED_FIELD_CHECKS);
+      runPreInsertCustomizerIfItIsTime(insertInput, isPreview, preInsertCustomizer, AbstractPreInsertCustomizer.WhenToRun.BEFORE_REQUIRED_FIELD_CHECKS);
       if(insertInput.getInputSource().shouldValidateRequiredFields())
       {
          validateRequiredFields(insertInput);
       }
 
-      runPreInsertCustomizerIfItIsTime(insertInput, preInsertCustomizer, AbstractPreInsertCustomizer.WhenToRun.BEFORE_SECURITY_CHECKS);
+      runPreInsertCustomizerIfItIsTime(insertInput, isPreview, preInsertCustomizer, AbstractPreInsertCustomizer.WhenToRun.BEFORE_SECURITY_CHECKS);
       ValidateRecordSecurityLockHelper.validateSecurityFields(insertInput.getTable(), insertInput.getRecords(), ValidateRecordSecurityLockHelper.Action.INSERT);
 
-      runPreInsertCustomizerIfItIsTime(insertInput, preInsertCustomizer, AbstractPreInsertCustomizer.WhenToRun.AFTER_ALL_VALIDATIONS);
+      runPreInsertCustomizerIfItIsTime(insertInput, isPreview, preInsertCustomizer, AbstractPreInsertCustomizer.WhenToRun.AFTER_ALL_VALIDATIONS);
    }
 
 
@@ -290,13 +289,13 @@ public class InsertAction extends AbstractQActionFunction<InsertInput, InsertOut
    /*******************************************************************************
     **
     *******************************************************************************/
-   private void runPreInsertCustomizerIfItIsTime(InsertInput insertInput, Optional<AbstractPreInsertCustomizer> preInsertCustomizer, AbstractPreInsertCustomizer.WhenToRun whenToRun) throws QException
+   private void runPreInsertCustomizerIfItIsTime(InsertInput insertInput, boolean isPreview, Optional<TableCustomizerInterface> preInsertCustomizer, AbstractPreInsertCustomizer.WhenToRun whenToRun) throws QException
    {
       if(preInsertCustomizer.isPresent())
       {
-         if(whenToRun.equals(preInsertCustomizer.get().getWhenToRun()))
+         if(whenToRun.equals(preInsertCustomizer.get().whenToRunPreInsert(insertInput, isPreview)))
          {
-            insertInput.setRecords(preInsertCustomizer.get().apply(insertInput.getRecords()));
+            insertInput.setRecords(preInsertCustomizer.get().preInsert(insertInput, insertInput.getRecords(), isPreview));
          }
       }
    }
@@ -321,7 +320,7 @@ public class InsertAction extends AbstractQActionFunction<InsertInput, InsertOut
             {
                if(record.getValue(requiredField.getName()) == null || (requiredField.getType().isStringLike() && record.getValueString(requiredField.getName()).trim().equals("")))
                {
-                  record.addError(new BadInputStatusMessage("Missing value in required field: " + requiredField.getLabel()));
+                  record.addError(new BadInputStatusMessage("Missing value in required field: " + Objects.requireNonNullElse(requiredField.getLabel(), requiredField.getName())));
                }
             }
          }
@@ -416,7 +415,7 @@ public class InsertAction extends AbstractQActionFunction<InsertInput, InsertOut
                Optional<List<Serializable>> keyValues = UniqueKeyHelper.getKeyValues(table, uniqueKey, record);
                if(keyValues.isPresent() && (existingKeys.get(uniqueKey).contains(keyValues.get()) || keysInThisList.get(uniqueKey).contains(keyValues.get())))
                {
-                  record.addError(new BadInputStatusMessage("Another record already exists with this " + uniqueKey.getDescription(table)));
+                  record.addError(new DuplicateKeyBadInputStatusMessage("Another record already exists with this " + uniqueKey.getDescription(table)));
                   foundDupe = true;
                   break;
                }
@@ -455,7 +454,7 @@ public class InsertAction extends AbstractQActionFunction<InsertInput, InsertOut
    private QBackendModuleInterface getBackendModuleInterface(QBackendMetaData backend) throws QException
    {
       QBackendModuleDispatcher qBackendModuleDispatcher = new QBackendModuleDispatcher();
-      QBackendModuleInterface qModule = qBackendModuleDispatcher.getQBackendModule(backend);
+      QBackendModuleInterface  qModule                  = qBackendModuleDispatcher.getQBackendModule(backend);
       return (qModule);
    }
 

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/QueryAction.java

@@ -31,8 +31,8 @@ import java.util.Map;
 import java.util.Optional;
 import java.util.Set;
 import com.kingsrook.qqq.backend.core.actions.ActionHelper;
-import com.kingsrook.qqq.backend.core.actions.customizers.AbstractPostQueryCustomizer;
 import com.kingsrook.qqq.backend.core.actions.customizers.QCodeLoader;
+import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizerInterface;
 import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizers;
 import com.kingsrook.qqq.backend.core.actions.interfaces.QueryInterface;
 import com.kingsrook.qqq.backend.core.actions.reporting.BufferedRecordPipe;
@@ -73,7 +73,7 @@ public class QueryAction
 {
    private static final QLogger LOG = QLogger.getLogger(QueryAction.class);
 
-   private Optional<AbstractPostQueryCustomizer> postQueryRecordCustomizer;
+   private Optional<TableCustomizerInterface> postQueryRecordCustomizer;
 
    private QueryInput               queryInput;
    private QueryInterface           queryInterface;
@@ -100,7 +100,7 @@ public class QueryAction
       }
 
       QBackendMetaData backend = queryInput.getBackend();
-      postQueryRecordCustomizer = QCodeLoader.getTableCustomizer(AbstractPostQueryCustomizer.class, table, TableCustomizers.POST_QUERY_RECORD.getRole());
+      postQueryRecordCustomizer = QCodeLoader.getTableCustomizer(table, TableCustomizers.POST_QUERY_RECORD.getRole());
       this.queryInput = queryInput;
 
       if(queryInput.getRecordPipe() != null)
@@ -169,6 +169,7 @@ public class QueryAction
             nextLevelQueryInput.setTableName(association.getAssociatedTableName());
             nextLevelQueryInput.setIncludeAssociations(true);
             nextLevelQueryInput.setAssociationNamesToInclude(buildNextLevelAssociationNamesToInclude(association.getName(), queryInput.getAssociationNamesToInclude()));
+            nextLevelQueryInput.setTransaction(queryInput.getTransaction());
 
             QQueryFilter filter = new QQueryFilter();
             nextLevelQueryInput.setFilter(filter);
@@ -264,7 +265,7 @@ public class QueryAction
    {
       if(this.postQueryRecordCustomizer.isPresent())
       {
-         records = postQueryRecordCustomizer.get().apply(records);
+         records = postQueryRecordCustomizer.get().postQuery(queryInput, records);
       }
 
       if(queryInput.getShouldTranslatePossibleValues())

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/ReplaceAction.java

@@ -47,6 +47,7 @@ import com.kingsrook.qqq.backend.core.model.data.QRecord;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.UniqueKey;
 import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
+import org.apache.commons.lang.BooleanUtils;
 
 
 /*******************************************************************************
@@ -79,9 +80,11 @@ public class ReplaceAction extends AbstractQActionFunction<ReplaceInput, Replace
 
       try
       {
-         QTableMetaData table           = input.getTable();
-         UniqueKey      uniqueKey       = input.getKey();
-         String         primaryKeyField = table.getPrimaryKeyField();
+         QTableMetaData table                     = input.getTable();
+         UniqueKey      uniqueKey                 = input.getKey();
+         String         primaryKeyField           = table.getPrimaryKeyField();
+         boolean        allowNullKeyValuesToEqual = BooleanUtils.isTrue(input.getAllowNullKeyValuesToEqual());
+
          if(transaction == null)
          {
             transaction = QBackendTransaction.openFor(new InsertInput(input.getTableName()));
@@ -98,10 +101,11 @@ public class ReplaceAction extends AbstractQActionFunction<ReplaceInput, Replace
             // originally it was thought that we'd need to pass the filter in here           //
             // but, it's been decided not to.  the filter only applies to what we can delete //
             ///////////////////////////////////////////////////////////////////////////////////
-            Map<List<Serializable>, Serializable> existingKeys = UniqueKeyHelper.getExistingKeys(transaction, table, page, uniqueKey);
+            Map<List<Serializable>, Serializable> existingKeys = UniqueKeyHelper.getExistingKeys(transaction, table, page, uniqueKey, allowNullKeyValuesToEqual);
+
             for(QRecord record : page)
             {
-               Optional<List<Serializable>> keyValues = UniqueKeyHelper.getKeyValues(table, uniqueKey, record);
+               Optional<List<Serializable>> keyValues = UniqueKeyHelper.getKeyValues(table, uniqueKey, record, allowNullKeyValuesToEqual);
                if(keyValues.isPresent())
                {
                   if(existingKeys.containsKey(keyValues.get()))
@@ -136,19 +140,33 @@ public class ReplaceAction extends AbstractQActionFunction<ReplaceInput, Replace
          UpdateOutput updateOutput = new UpdateAction().execute(updateInput);
          output.setUpdateOutput(updateOutput);
 
-         QQueryFilter deleteFilter = new QQueryFilter(new QFilterCriteria(primaryKeyField, QCriteriaOperator.NOT_IN, primaryKeysToKeep));
-         if(input.getFilter() != null)
+         if(input.getPerformDeletes())
          {
-            deleteFilter.addSubFilter(input.getFilter());
+            QQueryFilter deleteFilter = new QQueryFilter(new QFilterCriteria(primaryKeyField, QCriteriaOperator.NOT_IN, primaryKeysToKeep));
+            if(input.getFilter() != null)
+            {
+               deleteFilter.addSubFilter(input.getFilter());
+            }
+
+            DeleteInput deleteInput = new DeleteInput();
+            deleteInput.setTableName(table.getName());
+            deleteInput.setQueryFilter(deleteFilter);
+            deleteInput.setTransaction(transaction);
+            deleteInput.setOmitDmlAudit(input.getOmitDmlAudit());
+            DeleteOutput deleteOutput = new DeleteAction().execute(deleteInput);
+            output.setDeleteOutput(deleteOutput);
          }
 
-         DeleteInput deleteInput = new DeleteInput();
-         deleteInput.setTableName(table.getName());
-         deleteInput.setQueryFilter(deleteFilter);
-         deleteInput.setTransaction(transaction);
-         deleteInput.setOmitDmlAudit(input.getOmitDmlAudit());
-         DeleteOutput deleteOutput = new DeleteAction().execute(deleteInput);
-         output.setDeleteOutput(deleteOutput);
+         if(input.getSetPrimaryKeyInInsertedRecords())
+         {
+            for(int i = 0; i < insertList.size(); i++)
+            {
+               if(i < insertOutput.getRecords().size())
+               {
+                  insertList.get(i).setValue(primaryKeyField, insertOutput.getRecords().get(i).getValue(primaryKeyField));
+               }
+            }
+         }
 
          if(weOwnTheTransaction)
          {

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/StorageAction.java

@@ -0,0 +1,120 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.actions.tables;
+
+
+import java.io.InputStream;
+import java.io.OutputStream;
+import com.kingsrook.qqq.backend.core.actions.ActionHelper;
+import com.kingsrook.qqq.backend.core.actions.interfaces.QStorageInterface;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.actions.tables.storage.StorageInput;
+import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleDispatcher;
+import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleInterface;
+
+
+/*******************************************************************************
+ ** Action to do (generally, "mass") storage operations in a backend.
+ **
+ ** e.g., store a (potentially large) file - specifically - by working with it
+ ** as either an InputStream or OutputStream.
+ **
+ ** May not be implemented in all backends.
+ **
+ *******************************************************************************/
+public class StorageAction
+{
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public OutputStream createOutputStream(StorageInput storageInput) throws QException
+   {
+      QBackendModuleInterface qBackendModuleInterface = preAction(storageInput);
+      QStorageInterface       storageInterface        = qBackendModuleInterface.getStorageInterface();
+      return (storageInterface.createOutputStream(storageInput));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public InputStream getInputStream(StorageInput storageInput) throws QException
+   {
+      QBackendModuleInterface qBackendModuleInterface = preAction(storageInput);
+      QStorageInterface       storageInterface        = qBackendModuleInterface.getStorageInterface();
+      return (storageInterface.getInputStream(storageInput));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private QBackendModuleInterface preAction(StorageInput storageInput) throws QException
+   {
+      ActionHelper.validateSession(storageInput);
+
+      if(storageInput.getTableName() == null)
+      {
+         throw (new QException("Table name was not specified in storage input"));
+      }
+
+      QTableMetaData table = storageInput.getTable();
+      if(table == null)
+      {
+         throw (new QException("A table named [" + storageInput.getTableName() + "] was not found in the active QInstance"));
+      }
+
+      QBackendMetaData         backend                  = storageInput.getBackend();
+      QBackendModuleDispatcher qBackendModuleDispatcher = new QBackendModuleDispatcher();
+      QBackendModuleInterface  qModule                  = qBackendModuleDispatcher.getQBackendModule(backend);
+      return (qModule);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public void makePublic(StorageInput storageInput) throws QException
+   {
+      QBackendModuleInterface qBackendModuleInterface = preAction(storageInput);
+      QStorageInterface       storageInterface        = qBackendModuleInterface.getStorageInterface();
+      storageInterface.makePublic(storageInput);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public String getDownloadURL(StorageInput storageInput) throws QException
+   {
+      QBackendModuleInterface qBackendModuleInterface = preAction(storageInput);
+      QStorageInterface       storageInterface        = qBackendModuleInterface.getStorageInterface();
+      return (storageInterface.getDownloadURL(storageInput));
+   }
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/UpdateAction.java

@@ -34,9 +34,8 @@ import com.kingsrook.qqq.backend.core.actions.ActionHelper;
 import com.kingsrook.qqq.backend.core.actions.audits.DMLAuditAction;
 import com.kingsrook.qqq.backend.core.actions.automation.AutomationStatus;
 import com.kingsrook.qqq.backend.core.actions.automation.RecordAutomationStatusUpdater;
-import com.kingsrook.qqq.backend.core.actions.customizers.AbstractPostUpdateCustomizer;
-import com.kingsrook.qqq.backend.core.actions.customizers.AbstractPreUpdateCustomizer;
 import com.kingsrook.qqq.backend.core.actions.customizers.QCodeLoader;
+import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizerInterface;
 import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizers;
 import com.kingsrook.qqq.backend.core.actions.interfaces.UpdateInterface;
 import com.kingsrook.qqq.backend.core.actions.tables.helpers.ValidateRecordSecurityLockHelper;
@@ -57,6 +56,8 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateOutput;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.DynamicDefaultValueBehavior;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.FieldBehavior;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
 import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinOn;
@@ -67,11 +68,13 @@ import com.kingsrook.qqq.backend.core.model.metadata.tables.Association;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 import com.kingsrook.qqq.backend.core.model.statusmessages.BadInputStatusMessage;
 import com.kingsrook.qqq.backend.core.model.statusmessages.NotFoundStatusMessage;
+import com.kingsrook.qqq.backend.core.model.statusmessages.QErrorMessage;
 import com.kingsrook.qqq.backend.core.model.statusmessages.QWarningMessage;
 import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleDispatcher;
 import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleInterface;
 import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
 import com.kingsrook.qqq.backend.core.utils.ValueUtils;
+import org.apache.commons.lang.BooleanUtils;
 import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
 
 
@@ -189,14 +192,12 @@ public class UpdateAction
       //////////////////////////////////////////////////////////////
       // finally, run the post-update customizer, if there is one //
       //////////////////////////////////////////////////////////////
-      Optional<AbstractPostUpdateCustomizer> postUpdateCustomizer = QCodeLoader.getTableCustomizer(AbstractPostUpdateCustomizer.class, table, TableCustomizers.POST_UPDATE_RECORD.getRole());
+      Optional<TableCustomizerInterface> postUpdateCustomizer = QCodeLoader.getTableCustomizer(table, TableCustomizers.POST_UPDATE_RECORD.getRole());
       if(postUpdateCustomizer.isPresent())
       {
          try
          {
-            postUpdateCustomizer.get().setUpdateInput(updateInput);
-            oldRecordList.ifPresent(l -> postUpdateCustomizer.get().setOldRecordList(l));
-            updateOutput.setRecords(postUpdateCustomizer.get().apply(updateOutput.getRecords()));
+            updateOutput.setRecords(postUpdateCustomizer.get().postUpdate(updateInput, updateOutput.getRecords(), oldRecordList));
          }
          catch(Exception e)
          {
@@ -244,7 +245,13 @@ public class UpdateAction
       /////////////////////////////
       // run standard validators //
       /////////////////////////////
-      ValueBehaviorApplier.applyFieldBehaviors(ValueBehaviorApplier.Action.UPDATE, updateInput.getInstance(), table, updateInput.getRecords());
+      Set<FieldBehavior<?>> behaviorsToOmit = null;
+      if(BooleanUtils.isTrue(updateInput.getOmitModifyDateUpdate()))
+      {
+         behaviorsToOmit = Set.of(DynamicDefaultValueBehavior.MODIFY_DATE);
+      }
+
+      ValueBehaviorApplier.applyFieldBehaviors(ValueBehaviorApplier.Action.UPDATE, updateInput.getInstance(), table, updateInput.getRecords(), behaviorsToOmit);
       validatePrimaryKeysAreGiven(updateInput);
 
       if(oldRecordList.isPresent())
@@ -264,13 +271,10 @@ public class UpdateAction
       ///////////////////////////////////////////////////////////////////////////
       // after all validations, run the pre-update customizer, if there is one //
       ///////////////////////////////////////////////////////////////////////////
-      Optional<AbstractPreUpdateCustomizer> preUpdateCustomizer = QCodeLoader.getTableCustomizer(AbstractPreUpdateCustomizer.class, table, TableCustomizers.PRE_UPDATE_RECORD.getRole());
+      Optional<TableCustomizerInterface> preUpdateCustomizer = QCodeLoader.getTableCustomizer(table, TableCustomizers.PRE_UPDATE_RECORD.getRole());
       if(preUpdateCustomizer.isPresent())
       {
-         preUpdateCustomizer.get().setUpdateInput(updateInput);
-         preUpdateCustomizer.get().setIsPreview(isPreview);
-         oldRecordList.ifPresent(l -> preUpdateCustomizer.get().setOldRecordList(l));
-         updateInput.setRecords(preUpdateCustomizer.get().apply(updateInput.getRecords()));
+         updateInput.setRecords(preUpdateCustomizer.get().preUpdate(updateInput, updateInput.getRecords(), isPreview, oldRecordList));
       }
    }
 
@@ -390,7 +394,12 @@ public class UpdateAction
                   QRecord      oldRecord = lookedUpRecords.get(value);
                   QFieldType   fieldType = table.getField(lock.getFieldName()).getType();
                   Serializable lockValue = ValueUtils.getValueAsFieldType(fieldType, oldRecord.getValue(lock.getFieldName()));
-                  ValidateRecordSecurityLockHelper.validateRecordSecurityValue(table, record, lock, lockValue, fieldType, ValidateRecordSecurityLockHelper.Action.UPDATE);
+
+                  List<QErrorMessage> errors = ValidateRecordSecurityLockHelper.validateRecordSecurityValue(table, lock, lockValue, fieldType, ValidateRecordSecurityLockHelper.Action.UPDATE);
+                  if(CollectionUtils.nullSafeHasContents(errors))
+                  {
+                     errors.forEach(e -> record.addError(e));
+                  }
                }
             }
          }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/helpers/QueryStatManager.java

@@ -214,71 +214,78 @@ public class QueryStatManager
     *******************************************************************************/
    public void add(QueryStat queryStat)
    {
-      if(queryStat == null)
+      try
       {
-         return;
-      }
-
-      if(active)
-      {
-         ////////////////////////////////////////////////////////////////////////////////////////
-         // set fields that we need to capture now (rather than when the thread to store runs) //
-         ////////////////////////////////////////////////////////////////////////////////////////
-         if(queryStat.getFirstResultTimestamp() == null)
+         if(queryStat == null)
          {
-            queryStat.setFirstResultTimestamp(Instant.now());
-         }
-
-         if(queryStat.getStartTimestamp() != null && queryStat.getFirstResultTimestamp() != null && queryStat.getFirstResultMillis() == null)
-         {
-            long millis = queryStat.getFirstResultTimestamp().toEpochMilli() - queryStat.getStartTimestamp().toEpochMilli();
-            queryStat.setFirstResultMillis((int) millis);
-         }
-
-         if(queryStat.getFirstResultMillis() != null && queryStat.getFirstResultMillis() < minMillisToStore)
-         {
-            //////////////////////////////////////////////////////////////
-            // discard this record if it's under the min millis setting //
-            //////////////////////////////////////////////////////////////
             return;
          }
 
-         if(queryStat.getSessionId() == null && QContext.getQSession() != null)
+         if(active)
          {
-            queryStat.setSessionId(QContext.getQSession().getUuid());
-         }
-
-         if(queryStat.getAction() == null)
-         {
-            if(!QContext.getActionStack().isEmpty())
+            ////////////////////////////////////////////////////////////////////////////////////////
+            // set fields that we need to capture now (rather than when the thread to store runs) //
+            ////////////////////////////////////////////////////////////////////////////////////////
+            if(queryStat.getFirstResultTimestamp() == null)
             {
-               queryStat.setAction(QContext.getActionStack().peek().getActionIdentity());
+               queryStat.setFirstResultTimestamp(Instant.now());
             }
-            else
+
+            if(queryStat.getStartTimestamp() != null && queryStat.getFirstResultTimestamp() != null && queryStat.getFirstResultMillis() == null)
             {
-               boolean   expected = false;
-               Exception e        = new Exception("Unexpected empty action stack");
-               for(StackTraceElement stackTraceElement : e.getStackTrace())
+               long millis = queryStat.getFirstResultTimestamp().toEpochMilli() - queryStat.getStartTimestamp().toEpochMilli();
+               queryStat.setFirstResultMillis((int) millis);
+            }
+
+            if(queryStat.getFirstResultMillis() != null && queryStat.getFirstResultMillis() < minMillisToStore)
+            {
+               //////////////////////////////////////////////////////////////
+               // discard this record if it's under the min millis setting //
+               //////////////////////////////////////////////////////////////
+               return;
+            }
+
+            if(queryStat.getSessionId() == null && QContext.getQSession() != null)
+            {
+               queryStat.setSessionId(QContext.getQSession().getUuid());
+            }
+
+            if(queryStat.getAction() == null)
+            {
+               if(QContext.getActionStack() != null && !QContext.getActionStack().isEmpty())
                {
-                  String className = stackTraceElement.getClassName();
-                  if(className.contains(QueryStatManagerInsertJob.class.getName()))
+                  queryStat.setAction(QContext.getActionStack().peek().getActionIdentity());
+               }
+               else
+               {
+                  boolean   expected = false;
+                  Exception e        = new Exception("Unexpected empty action stack");
+                  for(StackTraceElement stackTraceElement : e.getStackTrace())
                   {
-                     expected = true;
-                     break;
+                     String className = stackTraceElement.getClassName();
+                     if(className.contains(QueryStatManagerInsertJob.class.getName()))
+                     {
+                        expected = true;
+                        break;
+                     }
+                  }
+
+                  if(!expected)
+                  {
+                     LOG.debug(e);
                   }
                }
+            }
 
-               if(!expected)
-               {
-                  LOG.debug(e);
-               }
+            synchronized(this)
+            {
+               queryStats.add(queryStat);
             }
          }
-
-         synchronized(this)
-         {
-            queryStats.add(queryStat);
-         }
+      }
+      catch(Exception e)
+      {
+         LOG.debug("Error adding query stat", e);
       }
    }
 

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/helpers/UniqueKeyHelper.java

@@ -54,7 +54,7 @@ public class UniqueKeyHelper
    /*******************************************************************************
     **
     *******************************************************************************/
-   public static Map<List<Serializable>, Serializable> getExistingKeys(QBackendTransaction transaction, QTableMetaData table, List<QRecord> recordList, UniqueKey uniqueKey) throws QException
+   public static Map<List<Serializable>, Serializable> getExistingKeys(QBackendTransaction transaction, QTableMetaData table, List<QRecord> recordList, UniqueKey uniqueKey, boolean allowNullKeyValuesToEqual) throws QException
    {
       List<String>                          ukFieldNames    = uniqueKey.getFieldNames();
       Map<List<Serializable>, Serializable> existingRecords = new HashMap<>();
@@ -112,7 +112,7 @@ public class UniqueKeyHelper
          QueryOutput queryOutput = new QueryAction().execute(queryInput);
          for(QRecord record : queryOutput.getRecords())
          {
-            Optional<List<Serializable>> keyValues = getKeyValues(table, uniqueKey, record);
+            Optional<List<Serializable>> keyValues = getKeyValues(table, uniqueKey, record, allowNullKeyValuesToEqual);
             if(keyValues.isPresent())
             {
                existingRecords.put(keyValues.get(), record.getValue(table.getPrimaryKeyField()));
@@ -128,7 +128,17 @@ public class UniqueKeyHelper
    /*******************************************************************************
     **
     *******************************************************************************/
-   public static Optional<List<Serializable>> getKeyValues(QTableMetaData table, UniqueKey uniqueKey, QRecord record)
+   public static Map<List<Serializable>, Serializable> getExistingKeys(QBackendTransaction transaction, QTableMetaData table, List<QRecord> recordList, UniqueKey uniqueKey) throws QException
+   {
+      return (getExistingKeys(transaction, table, recordList, uniqueKey, false));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public static Optional<List<Serializable>> getKeyValues(QTableMetaData table, UniqueKey uniqueKey, QRecord record, boolean allowNullKeyValuesToEqual)
    {
       try
       {
@@ -138,7 +148,19 @@ public class UniqueKeyHelper
             QFieldMetaData field      = table.getField(fieldName);
             Serializable   value      = record.getValue(fieldName);
             Serializable   typedValue = ValueUtils.getValueAsFieldType(field.getType(), value);
-            keyValues.add(typedValue == null ? new NullUniqueKeyValue() : typedValue);
+
+            ///////////////////////////////////////////////////////////////////////////////////
+            // if null value, look at flag to determine if a null should be used (which will //
+            // allow keys to match), or a NullUniqueKeyValue, (which will never match)       //
+            ///////////////////////////////////////////////////////////////////////////////////
+            if(typedValue == null)
+            {
+               keyValues.add(allowNullKeyValuesToEqual ? null : new NullUniqueKeyValue());
+            }
+            else
+            {
+               keyValues.add(typedValue);
+            }
          }
          return (Optional.of(keyValues));
       }
@@ -150,6 +172,16 @@ public class UniqueKeyHelper
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public static Optional<List<Serializable>> getKeyValues(QTableMetaData table, UniqueKey uniqueKey, QRecord record)
+   {
+      return (getKeyValues(table, uniqueKey, record, false));
+   }
+
+
+
    /*******************************************************************************
     ** To make a list of unique key values here behave like they do in an RDBMS
     ** (which is what we're trying to mimic - which is - 2 null values in a field

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/helpers/ValidateRecordSecurityLockHelper.java

@@ -24,6 +24,7 @@ package com.kingsrook.qqq.backend.core.actions.tables.helpers;
 
 import java.io.Serializable;
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -42,13 +43,16 @@ import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
 import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinOn;
 import com.kingsrook.qqq.backend.core.model.metadata.joins.QJoinMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.security.MultiRecordSecurityLock;
 import com.kingsrook.qqq.backend.core.model.metadata.security.NullValueBehaviorUtil;
 import com.kingsrook.qqq.backend.core.model.metadata.security.QSecurityKeyType;
 import com.kingsrook.qqq.backend.core.model.metadata.security.RecordSecurityLock;
 import com.kingsrook.qqq.backend.core.model.metadata.security.RecordSecurityLockFilters;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 import com.kingsrook.qqq.backend.core.model.statusmessages.PermissionDeniedMessage;
+import com.kingsrook.qqq.backend.core.model.statusmessages.QErrorMessage;
 import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
+import com.kingsrook.qqq.backend.core.utils.JsonUtils;
 import com.kingsrook.qqq.backend.core.utils.ListingHash;
 import com.kingsrook.qqq.backend.core.utils.StringUtils;
 import com.kingsrook.qqq.backend.core.utils.ValueUtils;
@@ -81,160 +85,273 @@ public class ValidateRecordSecurityLockHelper
     *******************************************************************************/
    public static void validateSecurityFields(QTableMetaData table, List<QRecord> records, Action action) throws QException
    {
-      List<RecordSecurityLock> locksToCheck = getRecordSecurityLocks(table, action);
-      if(CollectionUtils.nullSafeIsEmpty(locksToCheck))
+      MultiRecordSecurityLock locksToCheck = getRecordSecurityLocks(table, action);
+      if(locksToCheck == null || CollectionUtils.nullSafeIsEmpty(locksToCheck.getLocks()))
+      {
+         return;
+      }
+
+      //////////////////////////////////////////////////////////////////////////////////////////
+      // we will be relying on primary keys being set in records - but (at least for inserts) //
+      // we might not have pkeys - so make them up (and clear them out at the end)            //
+      //////////////////////////////////////////////////////////////////////////////////////////
+      Map<Serializable, QRecord> madeUpPrimaryKeys = makeUpPrimaryKeysIfNeeded(records, table);
+
+      ////////////////////////////////
+      // actually check lock values //
+      ////////////////////////////////
+      Map<Serializable, RecordWithErrors> errorRecords = new HashMap<>();
+      evaluateRecordLocks(table, records, action, locksToCheck, errorRecords, new ArrayList<>());
+
+      /////////////////////////////////
+      // propagate errors to records //
+      /////////////////////////////////
+      for(RecordWithErrors recordWithErrors : errorRecords.values())
+      {
+         recordWithErrors.propagateErrorsToRecord(locksToCheck);
+      }
+
+      /////////////////////////////////
+      // remove made-up primary keys //
+      /////////////////////////////////
+      String primaryKeyField = table.getPrimaryKeyField();
+      for(QRecord record : madeUpPrimaryKeys.values())
+      {
+         record.setValue(primaryKeyField, null);
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** For a list of `records` from a `table`, and a given `action`, evaluate a
+    ** `recordSecurityLock` (which may be a multi-lock) - populating the input map
+    ** of `errorRecords` - key'ed by primary key value (real or made up), with
+    ** error messages existing in a tree, with positions matching the multi-lock
+    ** tree that we're navigating, as tracked by `treePosition`.
+    **
+    ** Recursively processes multi-locks (and the top-level call is always with a
+    ** multi-lock - as the table's recordLocks list converted to an AND-multi-lock).
+    **
+    ** Of note - for the case of READ_WRITE locks, we're only evaluating the values
+    ** on the record, to see if they're allowed for us to store (because if we didn't
+    ** have the key, we wouldn't have been able to read the value (which is verified
+    ** outside of here, in UpdateAction/DeleteAction).
+    **
+    ** BUT - WRITE locks - in their case, we read the record no matter what, and in
+    ** here we need to verify we have a key that allows us to WRITE the record.
+    *******************************************************************************/
+   private static void evaluateRecordLocks(QTableMetaData table, List<QRecord> records, Action action, RecordSecurityLock recordSecurityLock, Map<Serializable, RecordWithErrors> errorRecords, List<Integer> treePosition) throws QException
+   {
+      if(recordSecurityLock instanceof MultiRecordSecurityLock multiRecordSecurityLock)
+      {
+         /////////////////////////////////////////////
+         // for multi-locks, make recursive descent //
+         /////////////////////////////////////////////
+         int i = 0;
+         for(RecordSecurityLock childLock : CollectionUtils.nonNullList(multiRecordSecurityLock.getLocks()))
+         {
+            treePosition.add(i);
+            evaluateRecordLocks(table, records, action, childLock, errorRecords, treePosition);
+            treePosition.remove(treePosition.size() - 1);
+            i++;
+         }
+
+         return;
+      }
+
+      /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      // if this lock has an all-access key, and the user has that key, then there can't be any errors here, so return early //
+      /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      QSecurityKeyType securityKeyType = QContext.getQInstance().getSecurityKeyType(recordSecurityLock.getSecurityKeyType());
+      if(StringUtils.hasContent(securityKeyType.getAllAccessKeyName()) && QContext.getQSession().hasSecurityKeyValue(securityKeyType.getAllAccessKeyName(), true, QFieldType.BOOLEAN))
       {
          return;
       }
 
       ////////////////////////////////
-      // actually check lock values //
+      // proceed w/ non-multi locks //
       ////////////////////////////////
-      for(RecordSecurityLock recordSecurityLock : locksToCheck)
+      String primaryKeyField = table.getPrimaryKeyField();
+      if(CollectionUtils.nullSafeIsEmpty(recordSecurityLock.getJoinNameChain()))
       {
-         if(CollectionUtils.nullSafeIsEmpty(recordSecurityLock.getJoinNameChain()))
+         //////////////////////////////////////////////////////////////////////////////////
+         // handle the value being in the table we're inserting/updating (e.g., no join) //
+         //////////////////////////////////////////////////////////////////////////////////
+         QFieldMetaData field = table.getField(recordSecurityLock.getFieldName());
+
+         for(QRecord record : records)
          {
-            //////////////////////////////////////////////////////////////////////////////////
-            // handle the value being in the table we're inserting/updating (e.g., no join) //
-            //////////////////////////////////////////////////////////////////////////////////
-            QFieldMetaData field = table.getField(recordSecurityLock.getFieldName());
-
-            for(QRecord record : records)
+            if(action.equals(Action.UPDATE) && !record.getValues().containsKey(field.getName()) && RecordSecurityLock.LockScope.READ_AND_WRITE.equals(recordSecurityLock.getLockScope()))
             {
-               if(action.equals(Action.UPDATE) && !record.getValues().containsKey(field.getName()) && RecordSecurityLock.LockScope.READ_AND_WRITE.equals(recordSecurityLock.getLockScope()))
-               {
-                  /////////////////////////////////////////////////////////////////////////////////////////////////////////
-                  // if this is a read-write lock, then if we have the record, it means we were able to read the record. //
-                  // So if we're not updating the security field, then no error can come from it!                        //
-                  /////////////////////////////////////////////////////////////////////////////////////////////////////////
-                  continue;
-               }
+               /////////////////////////////////////////////////////////////////////////////////////////////////////////
+               // if this is a read-write lock, then if we have the record, it means we were able to read the record. //
+               // So if we're not updating the security field, then no error can come from it!                        //
+               /////////////////////////////////////////////////////////////////////////////////////////////////////////
+               continue;
+            }
 
-               Serializable recordSecurityValue = record.getValue(field.getName());
-               validateRecordSecurityValue(table, record, recordSecurityLock, recordSecurityValue, field.getType(), action);
+            Serializable        recordSecurityValue = record.getValue(field.getName());
+            List<QErrorMessage> recordErrors        = validateRecordSecurityValue(table, recordSecurityLock, recordSecurityValue, field.getType(), action);
+            if(CollectionUtils.nullSafeHasContents(recordErrors))
+            {
+               errorRecords.computeIfAbsent(record.getValue(primaryKeyField), (k) -> new RecordWithErrors(record)).addAll(recordErrors, treePosition);
             }
          }
-         else
+      }
+      else
+      {
+         ////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+         // else look for the joined record - if it isn't found, assume a fail - else validate security value if found //
+         ////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+         QJoinMetaData leftMostJoin  = QContext.getQInstance().getJoin(recordSecurityLock.getJoinNameChain().get(0));
+         QJoinMetaData rightMostJoin = QContext.getQInstance().getJoin(recordSecurityLock.getJoinNameChain().get(recordSecurityLock.getJoinNameChain().size() - 1));
+
+         ////////////////////////////////
+         // todo probably, but more... //
+         ////////////////////////////////
+         // if(leftMostJoin.getLeftTable().equals(table.getName()))
+         // {
+         //    leftMostJoin = leftMostJoin.flip();
+         // }
+
+         QTableMetaData rightMostJoinTable = QContext.getQInstance().getTable(rightMostJoin.getRightTable());
+         QTableMetaData leftMostJoinTable  = QContext.getQInstance().getTable(leftMostJoin.getLeftTable());
+
+         for(List<QRecord> inputRecordPage : CollectionUtils.getPages(records, 500))
          {
-            ////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-            // else look for the joined record - if it isn't found, assume a fail - else validate security value if found //
-            ////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-            QJoinMetaData  leftMostJoin       = QContext.getQInstance().getJoin(recordSecurityLock.getJoinNameChain().get(0));
-            QJoinMetaData  rightMostJoin      = QContext.getQInstance().getJoin(recordSecurityLock.getJoinNameChain().get(recordSecurityLock.getJoinNameChain().size() - 1));
-            QTableMetaData rightMostJoinTable = QContext.getQInstance().getTable(rightMostJoin.getRightTable());
-            QTableMetaData leftMostJoinTable  = QContext.getQInstance().getTable(leftMostJoin.getLeftTable());
+            ////////////////////////////////////////////////////////////////////////////////////////////////
+            // set up a query for joined records                                                          //
+            // query will be like (fkey1=? and fkey2=?) OR (fkey1=? and fkey2=?) OR (fkey1=? and fkey2=?) //
+            ////////////////////////////////////////////////////////////////////////////////////////////////
+            QueryInput queryInput = new QueryInput();
+            queryInput.setTableName(leftMostJoin.getLeftTable());
+            QQueryFilter filter = new QQueryFilter().withBooleanOperator(QQueryFilter.BooleanOperator.OR);
+            queryInput.setFilter(filter);
 
-            for(List<QRecord> inputRecordPage : CollectionUtils.getPages(records, 500))
+            for(String joinName : recordSecurityLock.getJoinNameChain())
             {
-               ////////////////////////////////////////////////////////////////////////////////////////////////
-               // set up a query for joined records                                                          //
-               // query will be like (fkey1=? and fkey2=?) OR (fkey1=? and fkey2=?) OR (fkey1=? and fkey2=?) //
-               ////////////////////////////////////////////////////////////////////////////////////////////////
-               QueryInput queryInput = new QueryInput();
-               queryInput.setTableName(leftMostJoin.getLeftTable());
-               QQueryFilter filter = new QQueryFilter().withBooleanOperator(QQueryFilter.BooleanOperator.OR);
-               queryInput.setFilter(filter);
-
-               for(String joinName : recordSecurityLock.getJoinNameChain())
+               ///////////////////////////////////////
+               // we don't need the right-most join //
+               ///////////////////////////////////////
+               if(!joinName.equals(rightMostJoin.getName()))
                {
-                  ///////////////////////////////////////
-                  // we don't need the right-most join //
-                  ///////////////////////////////////////
-                  if(!joinName.equals(rightMostJoin.getName()))
+                  queryInput.withQueryJoin(new QueryJoin().withJoinMetaData(QContext.getQInstance().getJoin(joinName)).withSelect(true));
+               }
+            }
+
+            ///////////////////////////////////////////////////////////////////////////////////////////////////
+            // foreach input record (in this page), put it in a listing hash, with key = list of join-values //
+            // e.g., (17,47)=(QRecord1), (18,48)=(QRecord2,QRecord3)                                         //
+            // also build up the query's sub-filters here (only adding them if they're unique).              //
+            // e.g., 2 order-lines referencing the same orderId don't need to be added to the query twice    //
+            ///////////////////////////////////////////////////////////////////////////////////////////////////
+            ListingHash<List<Serializable>, QRecord> inputRecordMapByJoinFields = new ListingHash<>();
+            for(QRecord inputRecord : inputRecordPage)
+            {
+               List<Serializable> inputRecordJoinValues = new ArrayList<>();
+               QQueryFilter       subFilter             = new QQueryFilter();
+
+               boolean updatingAnyLockJoinFields = false;
+               for(JoinOn joinOn : rightMostJoin.getJoinOns())
+               {
+                  QFieldType   type             = rightMostJoinTable.getField(joinOn.getRightField()).getType();
+                  Serializable inputRecordValue = ValueUtils.getValueAsFieldType(type, inputRecord.getValue(joinOn.getRightField()));
+                  inputRecordJoinValues.add(inputRecordValue);
+
+                  // if we have a value in this field (and it's not the primary key), then it means we're updating part of the lock
+                  if(inputRecordValue != null && !joinOn.getRightField().equals(table.getPrimaryKeyField()))
                   {
-                     queryInput.withQueryJoin(new QueryJoin().withJoinMetaData(QContext.getQInstance().getJoin(joinName)).withSelect(true));
+                     updatingAnyLockJoinFields = true;
                   }
+
+                  subFilter.addCriteria(inputRecordValue == null
+                     ? new QFilterCriteria(rightMostJoin.getLeftTable() + "." + joinOn.getLeftField(), QCriteriaOperator.IS_BLANK)
+                     : new QFilterCriteria(rightMostJoin.getLeftTable() + "." + joinOn.getLeftField(), QCriteriaOperator.EQUALS, inputRecordValue));
                }
 
-               ///////////////////////////////////////////////////////////////////////////////////////////////////
-               // foreach input record (in this page), put it in a listing hash, with key = list of join-values //
-               // e.g., (17,47)=(QRecord1), (18,48)=(QRecord2,QRecord3)                                         //
-               // also build up the query's sub-filters here (only adding them if they're unique).              //
-               // e.g., 2 order-lines referencing the same orderId don't need to be added to the query twice    //
-               ///////////////////////////////////////////////////////////////////////////////////////////////////
-               ListingHash<List<Serializable>, QRecord> inputRecordMapByJoinFields = new ListingHash<>();
-               for(QRecord inputRecord : inputRecordPage)
+               //////////////////////////////////
+               // todo maybe, some version of? //
+               //////////////////////////////////
+               // if(action.equals(Action.UPDATE) && !updatingAnyLockJoinFields && RecordSecurityLock.LockScope.READ_AND_WRITE.equals(recordSecurityLock.getLockScope()))
+               // {
+               //    /////////////////////////////////////////////////////////////////////////////////////////////////////////
+               //    // if this is a read-write lock, then if we have the record, it means we were able to read the record. //
+               //    // So if we're not updating the security field, then no error can come from it!                        //
+               //    /////////////////////////////////////////////////////////////////////////////////////////////////////////
+               //    continue;
+               // }
+
+               if(!inputRecordMapByJoinFields.containsKey(inputRecordJoinValues))
                {
-                  List<Serializable> inputRecordJoinValues = new ArrayList<>();
-                  QQueryFilter       subFilter             = new QQueryFilter();
-
-                  for(JoinOn joinOn : rightMostJoin.getJoinOns())
-                  {
-                     QFieldType   type             = rightMostJoinTable.getField(joinOn.getRightField()).getType();
-                     Serializable inputRecordValue = ValueUtils.getValueAsFieldType(type, inputRecord.getValue(joinOn.getRightField()));
-                     inputRecordJoinValues.add(inputRecordValue);
-
-                     subFilter.addCriteria(inputRecordValue == null
-                        ? new QFilterCriteria(rightMostJoin.getLeftTable() + "." + joinOn.getLeftField(), QCriteriaOperator.IS_BLANK)
-                        : new QFilterCriteria(rightMostJoin.getLeftTable() + "." + joinOn.getLeftField(), QCriteriaOperator.EQUALS, inputRecordValue));
-                  }
-
-                  if(!inputRecordMapByJoinFields.containsKey(inputRecordJoinValues))
-                  {
-                     ////////////////////////////////////////////////////////////////////////////////
-                     // only add this sub-filter if it's for a list of keys we haven't seen before //
-                     ////////////////////////////////////////////////////////////////////////////////
-                     filter.addSubFilter(subFilter);
-                  }
-
-                  inputRecordMapByJoinFields.add(inputRecordJoinValues, inputRecord);
+                  ////////////////////////////////////////////////////////////////////////////////
+                  // only add this sub-filter if it's for a list of keys we haven't seen before //
+                  ////////////////////////////////////////////////////////////////////////////////
+                  filter.addSubFilter(subFilter);
                }
 
-               //////////////////////////////////////////////////////////////////////////////////////////////////////////////
-               // execute the query for joined records - then put them in a map with keys corresponding to the join values //
-               // e.g., (17,47)=(JoinRecord), (18,48)=(JoinRecord)                                                         //
-               //////////////////////////////////////////////////////////////////////////////////////////////////////////////
-               QueryOutput                      queryOutput               = new QueryAction().execute(queryInput);
-               Map<List<Serializable>, QRecord> joinRecordMapByJoinFields = new HashMap<>();
-               for(QRecord joinRecord : queryOutput.getRecords())
-               {
-                  List<Serializable> joinRecordValues = new ArrayList<>();
-                  for(JoinOn joinOn : rightMostJoin.getJoinOns())
-                  {
-                     Serializable joinValue = joinRecord.getValue(rightMostJoin.getLeftTable() + "." + joinOn.getLeftField());
-                     if(joinValue == null && joinRecord.getValues().keySet().stream().anyMatch(n -> !n.contains(".")))
-                     {
-                        joinValue = joinRecord.getValue(joinOn.getLeftField());
-                     }
-                     joinRecordValues.add(joinValue);
-                  }
+               inputRecordMapByJoinFields.add(inputRecordJoinValues, inputRecord);
+            }
 
-                  joinRecordMapByJoinFields.put(joinRecordValues, joinRecord);
+            //////////////////////////////////////////////////////////////////////////////////////////////////////////////
+            // execute the query for joined records - then put them in a map with keys corresponding to the join values //
+            // e.g., (17,47)=(JoinRecord), (18,48)=(JoinRecord)                                                         //
+            //////////////////////////////////////////////////////////////////////////////////////////////////////////////
+            QueryOutput                      queryOutput               = new QueryAction().execute(queryInput);
+            Map<List<Serializable>, QRecord> joinRecordMapByJoinFields = new HashMap<>();
+            for(QRecord joinRecord : queryOutput.getRecords())
+            {
+               List<Serializable> joinRecordValues = new ArrayList<>();
+               for(JoinOn joinOn : rightMostJoin.getJoinOns())
+               {
+                  Serializable joinValue = joinRecord.getValue(rightMostJoin.getLeftTable() + "." + joinOn.getLeftField());
+                  if(joinValue == null && joinRecord.getValues().keySet().stream().anyMatch(n -> !n.contains(".")))
+                  {
+                     joinValue = joinRecord.getValue(joinOn.getLeftField());
+                  }
+                  joinRecordValues.add(joinValue);
                }
 
-               //////////////////////////////////////////////////////////////////////////////////////////////////
-               // now for each input record, look for its joinRecord - if it isn't found, then this insert     //
-               // isn't allowed.  if it is found, then validate its value matches this session's security keys //
-               //////////////////////////////////////////////////////////////////////////////////////////////////
-               for(Map.Entry<List<Serializable>, List<QRecord>> entry : inputRecordMapByJoinFields.entrySet())
+               joinRecordMapByJoinFields.put(joinRecordValues, joinRecord);
+            }
+
+            //////////////////////////////////////////////////////////////////////////////////////////////////
+            // now for each input record, look for its joinRecord - if it isn't found, then this insert     //
+            // isn't allowed.  if it is found, then validate its value matches this session's security keys //
+            //////////////////////////////////////////////////////////////////////////////////////////////////
+            for(Map.Entry<List<Serializable>, List<QRecord>> entry : inputRecordMapByJoinFields.entrySet())
+            {
+               List<Serializable> inputRecordJoinValues = entry.getKey();
+               List<QRecord>      inputRecords          = entry.getValue();
+               if(joinRecordMapByJoinFields.containsKey(inputRecordJoinValues))
                {
-                  List<Serializable> inputRecordJoinValues = entry.getKey();
-                  List<QRecord>      inputRecords          = entry.getValue();
-                  if(joinRecordMapByJoinFields.containsKey(inputRecordJoinValues))
+                  QRecord joinRecord = joinRecordMapByJoinFields.get(inputRecordJoinValues);
+
+                  String         fieldName           = recordSecurityLock.getFieldName().replaceFirst(".*\\.", "");
+                  QFieldMetaData field               = leftMostJoinTable.getField(fieldName);
+                  Serializable   recordSecurityValue = joinRecord.getValue(fieldName);
+                  if(recordSecurityValue == null && joinRecord.getValues().keySet().stream().anyMatch(n -> n.contains(".")))
                   {
-                     QRecord joinRecord = joinRecordMapByJoinFields.get(inputRecordJoinValues);
+                     recordSecurityValue = joinRecord.getValue(recordSecurityLock.getFieldName());
+                  }
 
-                     String         fieldName           = recordSecurityLock.getFieldName().replaceFirst(".*\\.", "");
-                     QFieldMetaData field               = leftMostJoinTable.getField(fieldName);
-                     Serializable   recordSecurityValue = joinRecord.getValue(fieldName);
-                     if(recordSecurityValue == null && joinRecord.getValues().keySet().stream().anyMatch(n -> n.contains(".")))
+                  for(QRecord inputRecord : inputRecords)
+                  {
+                     List<QErrorMessage> recordErrors = validateRecordSecurityValue(table, recordSecurityLock, recordSecurityValue, field.getType(), action);
+                     if(CollectionUtils.nullSafeHasContents(recordErrors))
                      {
-                        recordSecurityValue = joinRecord.getValue(recordSecurityLock.getFieldName());
-                     }
-
-                     for(QRecord inputRecord : inputRecords)
-                     {
-                        validateRecordSecurityValue(table, inputRecord, recordSecurityLock, recordSecurityValue, field.getType(), action);
+                        errorRecords.computeIfAbsent(inputRecord.getValue(primaryKeyField), (k) -> new RecordWithErrors(inputRecord)).addAll(recordErrors, treePosition);
                      }
                   }
-                  else
+               }
+               else
+               {
+                  for(QRecord inputRecord : inputRecords)
                   {
-                     for(QRecord inputRecord : inputRecords)
+                     if(RecordSecurityLock.NullValueBehavior.DENY.equals(NullValueBehaviorUtil.getEffectiveNullValueBehavior(recordSecurityLock)))
                      {
-                        if(RecordSecurityLock.NullValueBehavior.DENY.equals(NullValueBehaviorUtil.getEffectiveNullValueBehavior(recordSecurityLock)))
-                        {
-                           inputRecord.addError(new PermissionDeniedMessage("You do not have permission to " + action.name().toLowerCase() + " this record - the referenced " + leftMostJoinTable.getLabel() + " was not found."));
-                        }
+                        PermissionDeniedMessage error = new PermissionDeniedMessage("You do not have permission to " + action.name().toLowerCase() + " this record - the referenced " + leftMostJoinTable.getLabel() + " was not found.");
+                        errorRecords.computeIfAbsent(inputRecord.getValue(primaryKeyField), (k) -> new RecordWithErrors(inputRecord)).add(error, treePosition);
                      }
                   }
                }
@@ -246,45 +363,81 @@ public class ValidateRecordSecurityLockHelper
 
 
    /*******************************************************************************
-    **
+    ** for tracking errors, we use primary keys.  add "made up" ones to records
+    ** if needed (e.g., insert use-case).
     *******************************************************************************/
-   private static List<RecordSecurityLock> getRecordSecurityLocks(QTableMetaData table, Action action)
+   private static Map<Serializable, QRecord> makeUpPrimaryKeysIfNeeded(List<QRecord> records, QTableMetaData table)
    {
-      List<RecordSecurityLock> recordSecurityLocks = CollectionUtils.nonNullList(table.getRecordSecurityLocks());
-      List<RecordSecurityLock> locksToCheck        = new ArrayList<>();
-
-      recordSecurityLocks = switch(action)
+      String                     primaryKeyField   = table.getPrimaryKeyField();
+      Map<Serializable, QRecord> madeUpPrimaryKeys = new HashMap<>();
+      Integer                    madeUpPrimaryKey  = -1;
+      for(QRecord record : records)
       {
-         case INSERT, UPDATE, DELETE -> RecordSecurityLockFilters.filterForWriteLocks(recordSecurityLocks);
-         case SELECT -> RecordSecurityLockFilters.filterForReadLocks(recordSecurityLocks);
+         if(record.getValue(primaryKeyField) == null)
+         {
+            madeUpPrimaryKeys.put(madeUpPrimaryKey, record);
+            record.setValue(primaryKeyField, madeUpPrimaryKey);
+            madeUpPrimaryKey--;
+         }
+      }
+      return madeUpPrimaryKeys;
+   }
+
+
+
+   /*******************************************************************************
+    ** For a given table & action type, convert the table's record locks to a
+    ** MultiRecordSecurityLock, with only the appropriate lock-scopes being included
+    ** (e.g., read-locks for selects, write-locks for insert/update/delete).
+    *******************************************************************************/
+   @SuppressWarnings("checkstyle:Indentation")
+   static MultiRecordSecurityLock getRecordSecurityLocks(QTableMetaData table, Action action)
+   {
+      List<RecordSecurityLock> allLocksOnTable = CollectionUtils.nonNullList(table.getRecordSecurityLocks());
+      MultiRecordSecurityLock locksOfType = switch(action)
+      {
+         case INSERT, UPDATE, DELETE -> RecordSecurityLockFilters.filterForWriteLockTree(allLocksOnTable);
+         case SELECT -> RecordSecurityLockFilters.filterForReadLockTree(allLocksOnTable);
          default -> throw (new IllegalArgumentException("Unsupported action: " + action));
       };
 
+      if(action.equals(Action.UPDATE))
+      {
+         ////////////////////////////////////////////////////////////////////////////
+         // todo at some point this seemed right, but now it doesn't - needs work. //
+         ////////////////////////////////////////////////////////////////////////////
+         // ////////////////////////////////////////////////////////
+         // // when doing an update, convert all OR's to AND's... //
+         // ////////////////////////////////////////////////////////
+         // updateOperators(locksOfType, MultiRecordSecurityLock.BooleanOperator.AND);
+      }
+
       ////////////////////////////////////////
       // if there are no locks, just return //
       ////////////////////////////////////////
-      if(CollectionUtils.nullSafeIsEmpty(recordSecurityLocks))
+      if(locksOfType == null || CollectionUtils.nullSafeIsEmpty(locksOfType.getLocks()))
       {
          return (null);
       }
 
-      ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-      // decide if any locks need checked - where one may not need checked if it has an all-access key, and the user has all-access //
-      ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-      for(RecordSecurityLock recordSecurityLock : recordSecurityLocks)
+      return (locksOfType);
+   }
+
+
+
+   /*******************************************************************************
+    ** for a full multi-lock tree, set all of the boolean operators to the specified one.
+    *******************************************************************************/
+   private static void updateOperators(MultiRecordSecurityLock multiLock, MultiRecordSecurityLock.BooleanOperator operator)
+   {
+      multiLock.setOperator(operator);
+      for(RecordSecurityLock childLock : multiLock.getLocks())
       {
-         QSecurityKeyType securityKeyType = QContext.getQInstance().getSecurityKeyType(recordSecurityLock.getSecurityKeyType());
-         if(StringUtils.hasContent(securityKeyType.getAllAccessKeyName()) && QContext.getQSession().hasSecurityKeyValue(securityKeyType.getAllAccessKeyName(), true, QFieldType.BOOLEAN))
+         if(childLock instanceof MultiRecordSecurityLock childMultiLock)
          {
-            LOG.trace("Session has " + securityKeyType.getAllAccessKeyName() + " - not checking this lock.");
-         }
-         else
-         {
-            locksToCheck.add(recordSecurityLock);
+            updateOperators(childMultiLock, operator);
          }
       }
-
-      return (locksToCheck);
    }
 
 
@@ -292,7 +445,7 @@ public class ValidateRecordSecurityLockHelper
    /*******************************************************************************
     **
     *******************************************************************************/
-   public static void validateRecordSecurityValue(QTableMetaData table, QRecord record, RecordSecurityLock recordSecurityLock, Serializable recordSecurityValue, QFieldType fieldType, Action action)
+   public static List<QErrorMessage> validateRecordSecurityValue(QTableMetaData table, RecordSecurityLock recordSecurityLock, Serializable recordSecurityValue, QFieldType fieldType, Action action)
    {
       if(recordSecurityValue == null)
       {
@@ -302,7 +455,7 @@ public class ValidateRecordSecurityLockHelper
          if(RecordSecurityLock.NullValueBehavior.DENY.equals(NullValueBehaviorUtil.getEffectiveNullValueBehavior(recordSecurityLock)))
          {
             String lockLabel = CollectionUtils.nullSafeHasContents(recordSecurityLock.getJoinNameChain()) ? recordSecurityLock.getSecurityKeyType() : table.getField(recordSecurityLock.getFieldName()).getLabel();
-            record.addError(new PermissionDeniedMessage("You do not have permission to " + action.name().toLowerCase() + " a record without a value in the field: " + lockLabel));
+            return (List.of(new PermissionDeniedMessage("You do not have permission to " + action.name().toLowerCase() + " a record without a value in the field: " + lockLabel)));
          }
       }
       else
@@ -314,15 +467,305 @@ public class ValidateRecordSecurityLockHelper
                ///////////////////////////////////////////////////////////////////////////////////////////////
                // avoid telling the user a value from a foreign record that they didn't pass in themselves. //
                ///////////////////////////////////////////////////////////////////////////////////////////////
-               record.addError(new PermissionDeniedMessage("You do not have permission to " + action.name().toLowerCase() + " this record."));
+               return (List.of(new PermissionDeniedMessage("You do not have permission to " + action.name().toLowerCase() + " this record.")));
             }
             else
             {
                QFieldMetaData field = table.getField(recordSecurityLock.getFieldName());
-               record.addError(new PermissionDeniedMessage("You do not have permission to " + action.name().toLowerCase() + " a record with a value of " + recordSecurityValue + " in the field: " + field.getLabel()));
+               return (List.of(new PermissionDeniedMessage("You do not have permission to " + action.name().toLowerCase() + " a record with a value of " + recordSecurityValue + " in the field: " + field.getLabel())));
             }
          }
       }
+      return (Collections.emptyList());
+   }
+
+
+
+   /*******************************************************************************
+    ** Class to track errors that we're associating with a record.
+    **
+    ** More complex than it first seems to be needed, because as we're evaluating
+    ** locks, we might find some, but based on the boolean condition associated with
+    ** them, they might not actually be record-level errors.
+    **
+    ** e.g., two locks with an OR relationship - as long as one passes, the record
+    ** should have no errors.  And so-on through the tree of locks/multi-locks.
+    **
+    ** Stores the errors in a tree of ErrorTreeNode objects.
+    **
+    ** References into that tree are achieved via a List of Integer called "tree positions"
+    ** where each entry in the list denotes the index of the tree node at that level.
+    **
+    ** e.g., given this tree:
+    ** <pre>
+    **   A      B
+    **  / \    /|\
+    ** C   D  E F G
+    **     |
+    **     H
+    ** </pre>
+    **
+    ** The positions of each node would be:
+    ** <pre>
+    ** A: [0]
+    ** B: [1]
+    ** C: [0,0]
+    ** D: [0,1]
+    ** E: [1,0]
+    ** F: [1,1]
+    ** G: [1,2]
+    ** H: [0,1,0]
+    ** </pre>
+    *******************************************************************************/
+   static class RecordWithErrors
+   {
+      private QRecord       record;
+      private ErrorTreeNode errorTree;
+
+
+
+      /*******************************************************************************
+       ** Constructor
+       **
+       *******************************************************************************/
+      public RecordWithErrors(QRecord record)
+      {
+         this.record = record;
+      }
+
+
+
+      /*******************************************************************************
+       ** add a list of errors, for a given list of tree positions
+       *******************************************************************************/
+      public void addAll(List<QErrorMessage> recordErrors, List<Integer> treePositions)
+      {
+         if(errorTree == null)
+         {
+            errorTree = new ErrorTreeNode();
+         }
+
+         ErrorTreeNode node = errorTree;
+         for(Integer treePosition : treePositions)
+         {
+            if(node.children == null)
+            {
+               node.children = new ArrayList<>(treePosition);
+            }
+
+            while(treePosition >= node.children.size())
+            {
+               node.children.add(null);
+            }
+
+            if(node.children.get(treePosition) == null)
+            {
+               node.children.set(treePosition, new ErrorTreeNode());
+            }
+
+            node = node.children.get(treePosition);
+         }
+
+         if(node.errors == null)
+         {
+            node.errors = new ArrayList<>();
+         }
+         node.errors.addAll(recordErrors);
+      }
+
+
+
+      /*******************************************************************************
+       ** add a single error to a given tree-position
+       *******************************************************************************/
+      public void add(QErrorMessage error, List<Integer> treePositions)
+      {
+         addAll(List.of(error), treePositions);
+      }
+
+
+
+      /*******************************************************************************
+       ** after the tree of errors has been built - walk a lock-tree (locksToCheck)
+       ** and resolve boolean operations, to get a final list of errors (possibly empty)
+       ** to put on the record.
+       *******************************************************************************/
+      public void propagateErrorsToRecord(MultiRecordSecurityLock locksToCheck)
+      {
+         List<QErrorMessage> errors = recursivePropagation(locksToCheck, new ArrayList<>());
+
+         if(CollectionUtils.nullSafeHasContents(errors))
+         {
+            errors.forEach(e -> record.addError(e));
+         }
+      }
+
+
+
+      /*******************************************************************************
+       ** recursive implementation of the propagation method - e.g., walk tree applying
+       ** boolean logic.
+       *******************************************************************************/
+      private List<QErrorMessage> recursivePropagation(MultiRecordSecurityLock locksToCheck, List<Integer> treePositions)
+      {
+         //////////////////////////////////////////////////////////////////
+         // build a list of errors at this level (and deeper levels too) //
+         //////////////////////////////////////////////////////////////////
+         List<QErrorMessage> errorsFromThisLevel = new ArrayList<>();
+
+         int i = 0;
+         for(RecordSecurityLock lock : locksToCheck.getLocks())
+         {
+            List<QErrorMessage> errorsFromThisLock;
+
+            treePositions.add(i);
+            if(lock instanceof MultiRecordSecurityLock childMultiLock)
+            {
+               errorsFromThisLock = recursivePropagation(childMultiLock, treePositions);
+            }
+            else
+            {
+               errorsFromThisLock = getErrorsFromTree(treePositions);
+            }
+
+            errorsFromThisLevel.addAll(errorsFromThisLock);
+
+            treePositions.remove(treePositions.size() - 1);
+            i++;
+         }
+
+         if(MultiRecordSecurityLock.BooleanOperator.AND.equals(locksToCheck.getOperator()))
+         {
+            //////////////////////////////////////////////////////////////
+            // for an AND - if there were ANY errors, then return them. //
+            //////////////////////////////////////////////////////////////
+            if(!errorsFromThisLevel.isEmpty())
+            {
+               return (errorsFromThisLevel);
+            }
+         }
+         else // OR
+         {
+            //////////////////////////////////////////////////////////
+            // for an OR - only return if ALL conditions had errors //
+            //////////////////////////////////////////////////////////
+            if(errorsFromThisLevel.size() == locksToCheck.getLocks().size())
+            {
+               return (errorsFromThisLevel); // todo something smarter?
+            }
+         }
+
+         ///////////////////////////////////
+         // else - no errors - empty list //
+         ///////////////////////////////////
+         return Collections.emptyList();
+      }
+
+
+
+      /*******************************************************************************
+       **
+       *******************************************************************************/
+      private List<QErrorMessage> getErrorsFromTree(List<Integer> treePositions)
+      {
+         ErrorTreeNode node = errorTree;
+
+         for(Integer treePosition : treePositions)
+         {
+            if(node.children == null)
+            {
+               return Collections.emptyList();
+            }
+
+            if(treePosition >= node.children.size())
+            {
+               return Collections.emptyList();
+            }
+
+            if(node.children.get(treePosition) == null)
+            {
+               return Collections.emptyList();
+            }
+
+            node = node.children.get(treePosition);
+         }
+
+         if(node.errors == null)
+         {
+            return Collections.emptyList();
+         }
+
+         return node.errors;
+      }
+
+
+
+      /*******************************************************************************
+       **
+       *******************************************************************************/
+      @Override
+      public String toString()
+      {
+         try
+         {
+            return JsonUtils.toPrettyJson(this);
+         }
+         catch(Exception e)
+         {
+            return "error in toString";
+         }
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** tree node used by RecordWithErrors
+    *******************************************************************************/
+   static class ErrorTreeNode
+   {
+      private List<QErrorMessage>      errors;
+      private ArrayList<ErrorTreeNode> children;
+
+
+
+      /*******************************************************************************
+       **
+       *******************************************************************************/
+      @Override
+      public String toString()
+      {
+         try
+         {
+            return JsonUtils.toPrettyJson(this);
+         }
+         catch(Exception e)
+         {
+            return "error in toString";
+         }
+      }
+
+
+
+      /*******************************************************************************
+       ** Getter for errors - only here for Jackson/toString
+       **
+       *******************************************************************************/
+      public List<QErrorMessage> getErrors()
+      {
+         return errors;
+      }
+
+
+
+      /*******************************************************************************
+       ** Getter for children - only here for Jackson/toString
+       **
+       *******************************************************************************/
+      public ArrayList<ErrorTreeNode> getChildren()
+      {
+         return children;
+      }
    }
 
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/QPossibleValueTranslator.java

@@ -269,6 +269,10 @@ public class QPossibleValueTranslator
          {
             value = ValueUtils.getValueAsInteger(value);
          }
+         if(field.getType().equals(QFieldType.LONG) && !(value instanceof Long))
+         {
+            value = ValueUtils.getValueAsLong(value);
+         }
       }
       catch(QValueException e)
       {
@@ -366,6 +370,14 @@ public class QPossibleValueTranslator
     *******************************************************************************/
    private String translatePossibleValueCustom(Serializable value, QPossibleValueSource possibleValueSource)
    {
+      /////////////////////////////////
+      // null input gets null output //
+      /////////////////////////////////
+      if(value == null)
+      {
+         return (null);
+      }
+
       try
       {
          QCustomPossibleValueProvider customPossibleValueProvider = QCodeLoader.getCustomPossibleValueProvider(possibleValueSource);
@@ -548,20 +560,47 @@ public class QPossibleValueTranslator
     *******************************************************************************/
    private void primePvsCache(String tableName, List<QPossibleValueSource> possibleValueSources, Collection<Serializable> values)
    {
+      String idField = null;
       for(QPossibleValueSource possibleValueSource : possibleValueSources)
       {
          possibleValueCache.putIfAbsent(possibleValueSource.getName(), new HashMap<>());
+         String thisPvsIdField;
+         if(StringUtils.hasContent(possibleValueSource.getOverrideIdField()))
+         {
+            thisPvsIdField = possibleValueSource.getOverrideIdField();
+         }
+         else
+         {
+            thisPvsIdField = QContext.getQInstance().getTable(tableName).getPrimaryKeyField();
+         }
+
+         if(idField == null)
+         {
+            idField = thisPvsIdField;
+         }
+         else
+         {
+            /////////////////////////////////////////////////////////////////////////////////////////////////////////////
+            // does this ever happen?  maybe not... because, like, the list of values probably wouldn't make sense for //
+            // more than one field in the table...                                                                     //
+            /////////////////////////////////////////////////////////////////////////////////////////////////////////////
+            if(!idField.equals(thisPvsIdField))
+            {
+               for(QPossibleValueSource valueSource : possibleValueSources)
+               {
+                  primePvsCache(tableName, List.of(valueSource), values);
+               }
+            }
+         }
       }
 
       try
       {
-         String primaryKeyField = QContext.getQInstance().getTable(tableName).getPrimaryKeyField();
-
          for(List<Serializable> page : CollectionUtils.getPages(values, 1000))
          {
             QueryInput queryInput = new QueryInput();
             queryInput.setTableName(tableName);
-            queryInput.setFilter(new QQueryFilter().withCriteria(new QFilterCriteria(primaryKeyField, QCriteriaOperator.IN, page)));
+            queryInput.setFilter(new QQueryFilter().withCriteria(new QFilterCriteria(idField, QCriteriaOperator.IN, page)));
             queryInput.setTransaction(getTransaction(tableName));
 
             /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
@@ -606,7 +645,7 @@ public class QPossibleValueTranslator
             ///////////////////////////////////////////////////////////////////////////////////
             for(QRecord record : queryOutput.getRecords())
             {
-               Serializable pkeyValue = record.getValue(primaryKeyField);
+               Serializable pkeyValue = record.getValue(idField);
                for(QPossibleValueSource possibleValueSource : possibleValueSources)
                {
                   QPossibleValue<?> possibleValue = new QPossibleValue<>(pkeyValue, record.getRecordLabel());

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/QValueFormatter.java

@@ -28,7 +28,6 @@ import java.time.LocalDateTime;
 import java.time.LocalTime;
 import java.time.ZonedDateTime;
 import java.time.format.DateTimeFormatter;
-import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
@@ -364,7 +363,9 @@ public class QValueFormatter
             }
          }
 
-         setDisplayValuesInRecord(fieldMap, record);
+         ValueBehaviorApplier.applyFieldBehaviors(ValueBehaviorApplier.Action.FORMATTING, QContext.getQInstance(), table, records, null);
+
+         setDisplayValuesInRecord(table, fieldMap, record, true);
          record.setRecordLabel(formatRecordLabel(table, record));
       }
    }
@@ -374,61 +375,49 @@ public class QValueFormatter
    /*******************************************************************************
     ** For a list of records, set their recordLabels and display values
     *******************************************************************************/
-   public static void setDisplayValuesInRecords(Collection<QFieldMetaData> fields, List<QRecord> records)
+   public static void setDisplayValuesInRecords(QTableMetaData table, Map<String, QFieldMetaData> fields, List<QRecord> records)
    {
       if(records == null)
       {
          return;
       }
 
+      if(table != null)
+      {
+         ValueBehaviorApplier.applyFieldBehaviors(ValueBehaviorApplier.Action.FORMATTING, QContext.getQInstance(), table, records, null);
+      }
+
       for(QRecord record : records)
       {
-         setDisplayValuesInRecord(fields, record);
+         setDisplayValuesInRecord(table, fields, record, true);
       }
    }
 
 
 
    /*******************************************************************************
-    ** For a list of records, set their recordLabels and display values
+    ** For a single record, set its display values - public version of this.
     *******************************************************************************/
-   public static void setDisplayValuesInRecords(Map<String, QFieldMetaData> fields, List<QRecord> records)
+   public static void setDisplayValuesInRecord(QTableMetaData table, Map<String, QFieldMetaData> fields, QRecord record)
    {
-      if(records == null)
-      {
-         return;
-      }
-
-      for(QRecord record : records)
-      {
-         setDisplayValuesInRecord(fields, record);
-      }
+      setDisplayValuesInRecord(table, fields, record, false);
    }
 
 
-
    /*******************************************************************************
-    ** For a list of records, set their display values
+    ** For a single record, set its display values - where caller (meant to stay private)
+    ** can specify if they've already done fieldBehaviors (to avoid re-doing).
     *******************************************************************************/
-   public static void setDisplayValuesInRecord(Collection<QFieldMetaData> fields, QRecord record)
+   private static void setDisplayValuesInRecord(QTableMetaData table, Map<String, QFieldMetaData> fields, QRecord record, boolean alreadyAppliedFieldDisplayBehaviors)
    {
-      for(QFieldMetaData field : fields)
+      if(!alreadyAppliedFieldDisplayBehaviors)
       {
-         if(record.getDisplayValue(field.getName()) == null)
+         if(table != null)
          {
-            String formattedValue = formatValue(field, record.getValue(field.getName()));
-            record.setDisplayValue(field.getName(), formattedValue);
+            ValueBehaviorApplier.applyFieldBehaviors(ValueBehaviorApplier.Action.FORMATTING, QContext.getQInstance(), table, List.of(record), null);
          }
       }
-   }
 
-
-
-   /*******************************************************************************
-    ** For a list of records, set their display values
-    *******************************************************************************/
-   public static void setDisplayValuesInRecord(Map<String, QFieldMetaData> fields, QRecord record)
-   {
       for(Map.Entry<String, QFieldMetaData> entry : fields.entrySet())
       {
          String         fieldName = entry.getKey();
@@ -490,6 +479,13 @@ public class QValueFormatter
             {
                adornmentValues = fileDownloadAdornment.get().getValues();
             }
+            else
+            {
+               ///////////////////////////////////////////////////////
+               // don't change blobs unless they are file-downloads //
+               ///////////////////////////////////////////////////////
+               continue;
+            }
 
             String fileNameField    = ValueUtils.getValueAsString(adornmentValues.get(AdornmentType.FileDownloadValues.FILE_NAME_FIELD));
             String fileNameFormat   = ValueUtils.getValueAsString(adornmentValues.get(AdornmentType.FileDownloadValues.FILE_NAME_FORMAT));

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/SearchPossibleValueSourceAction.java

@@ -275,8 +275,19 @@ public class SearchPossibleValueSourceAction
 
       queryInput.setFilter(queryFilter);
 
-      QueryOutput             queryOutput     = new QueryAction().execute(queryInput);
-      List<Serializable>      ids             = queryOutput.getRecords().stream().map(r -> r.getValue(table.getPrimaryKeyField())).toList();
+      QueryOutput queryOutput = new QueryAction().execute(queryInput);
+
+      String fieldName;
+      if(StringUtils.hasContent(possibleValueSource.getOverrideIdField()))
+      {
+         fieldName = possibleValueSource.getOverrideIdField();
+      }
+      else
+      {
+         fieldName = table.getPrimaryKeyField();
+      }
+
+      List<Serializable> ids = queryOutput.getRecords().stream().map(r -> r.getValue(fieldName)).toList();
       List<QPossibleValue<?>> qPossibleValues = possibleValueTranslator.buildTranslatedPossibleValueList(possibleValueSource, ids);
       output.setResults(qPossibleValues);
 

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/ValueBehaviorApplier.java

@@ -23,16 +23,18 @@ package com.kingsrook.qqq.backend.core.actions.values;
 
 
 import java.util.List;
+import java.util.Set;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
 import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.FieldBehavior;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.FieldDisplayBehavior;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
 
 
 /*******************************************************************************
- ** Utility class to apply value behaviors to records.  
+ ** Utility class to apply value behaviors to records.
  *******************************************************************************/
 public class ValueBehaviorApplier
 {
@@ -43,7 +45,8 @@ public class ValueBehaviorApplier
    public enum Action
    {
       INSERT,
-      UPDATE
+      UPDATE,
+      FORMATTING
    }
 
 
@@ -51,7 +54,7 @@ public class ValueBehaviorApplier
    /*******************************************************************************
     **
     *******************************************************************************/
-   public static void applyFieldBehaviors(Action action, QInstance instance, QTableMetaData table, List<QRecord> recordList)
+   public static void applyFieldBehaviors(Action action, QInstance instance, QTableMetaData table, List<QRecord> recordList, Set<FieldBehavior<?>> behaviorsToOmit)
    {
       if(CollectionUtils.nullSafeIsEmpty(recordList))
       {
@@ -62,7 +65,34 @@ public class ValueBehaviorApplier
       {
          for(FieldBehavior<?> fieldBehavior : CollectionUtils.nonNullCollection(field.getBehaviors()))
          {
-            fieldBehavior.apply(action, recordList, instance, table, field);
+            boolean applyBehavior = true;
+            if(behaviorsToOmit != null && behaviorsToOmit.contains(fieldBehavior))
+            {
+               /////////////////////////////////////////////////////////////////////////////////////////
+               // if we're given a set of behaviors to omit, and this behavior is in there, then skip //
+               /////////////////////////////////////////////////////////////////////////////////////////
+               applyBehavior = false;
+            }
+
+            if(Action.FORMATTING == action && !(fieldBehavior instanceof FieldDisplayBehavior<?>))
+            {
+               ////////////////////////////////////////////////////////////////////////////////////////////////
+               // for the formatting action, do not apply the behavior unless it is a field-display-behavior //
+               ////////////////////////////////////////////////////////////////////////////////////////////////
+               applyBehavior = false;
+            }
+            else if(Action.FORMATTING != action && fieldBehavior instanceof FieldDisplayBehavior<?>)
+            {
+               /////////////////////////////////////////////////////////////////////////////////////////////
+               // for non-formatting actions, do not apply the behavior IF it is a field-display-behavior //
+               /////////////////////////////////////////////////////////////////////////////////////////////
+               applyBehavior = false;
+            }
+
+            if(applyBehavior)
+            {
+               fieldBehavior.apply(action, recordList, instance, table, field);
+            }
          }
       }
    }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/context/CapturedContext.java

@@ -34,5 +34,11 @@ import com.kingsrook.qqq.backend.core.model.session.QSession;
  *******************************************************************************/
 public record CapturedContext(QInstance qInstance, QSession qSession, QBackendTransaction qBackendTransaction, Stack<AbstractActionInput> actionStack)
 {
-
+   /*******************************************************************************
+    ** Simpler constructor
+    *******************************************************************************/
+   public CapturedContext(QInstance qInstance, QSession qSession)
+   {
+      this(qInstance, qSession, null, null);
+   }
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/context/QContext.java

@@ -22,6 +22,9 @@
 package com.kingsrook.qqq.backend.core.context;
 
 
+import java.io.Serializable;
+import java.util.HashMap;
+import java.util.Map;
 import java.util.Optional;
 import java.util.Stack;
 import com.kingsrook.qqq.backend.core.actions.QBackendTransaction;
@@ -31,6 +34,8 @@ import com.kingsrook.qqq.backend.core.logging.QLogger;
 import com.kingsrook.qqq.backend.core.model.actions.AbstractActionInput;
 import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
 import com.kingsrook.qqq.backend.core.model.session.QSession;
+import com.kingsrook.qqq.backend.core.utils.lambdas.UnsafeVoidVoidMethod;
+import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
 
 
 /*******************************************************************************
@@ -47,6 +52,8 @@ public class QContext
    private static ThreadLocal<QBackendTransaction>        qBackendTransactionThreadLocal = new ThreadLocal<>();
    private static ThreadLocal<Stack<AbstractActionInput>> actionStackThreadLocal         = new ThreadLocal<>();
 
+   private static ThreadLocal<Map<String, Serializable>> objectsThreadLocal = new ThreadLocal<>();
+
 
 
    /*******************************************************************************
@@ -100,6 +107,25 @@ public class QContext
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public static <T extends Throwable> void withTemporaryContext(CapturedContext context, UnsafeVoidVoidMethod<T> method) throws T
+   {
+      CapturedContext originalContext = QContext.capture();
+      try
+      {
+         QContext.init(context);
+         method.run();
+      }
+      finally
+      {
+         QContext.init(originalContext);
+      }
+   }
+
+
+
    /*******************************************************************************
     ** Init a new thread with the context captured from a different thread.  e.g.,
     ** when starting some async task.
@@ -132,6 +158,7 @@ public class QContext
       qSessionThreadLocal.remove();
       qBackendTransactionThreadLocal.remove();
       actionStackThreadLocal.remove();
+      objectsThreadLocal.remove();
    }
 
 
@@ -259,4 +286,94 @@ public class QContext
 
       return (Optional.of(actionStackThreadLocal.get().get(0)));
    }
+
+
+
+   /*******************************************************************************
+    ** get one named object from the Context for the current thread.  may return null.
+    *******************************************************************************/
+   public static Serializable getObject(String key)
+   {
+      if(objectsThreadLocal.get() == null)
+      {
+         return null;
+      }
+      return objectsThreadLocal.get().get(key);
+   }
+
+
+
+   /*******************************************************************************
+    ** get one named object from the Context for the current thread, cast to the
+    ** specified type if possible.  if not found, or wrong type, empty is returned.
+    *******************************************************************************/
+   public static <T extends Serializable> Optional<T> getObject(String key, Class<T> type)
+   {
+      Serializable object = getObject(key);
+
+      if(type.isInstance(object))
+      {
+         return Optional.of(type.cast(object));
+      }
+      else if(object == null)
+      {
+         return Optional.empty();
+      }
+      else
+      {
+         LOG.warn("Unexpected type of object found in session under key [" + key + "]",
+            logPair("expectedType", type.getName()),
+            logPair("actualType", object.getClass().getName())
+         );
+         return Optional.empty();
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** put a named object into the Context for the current thread.
+    *******************************************************************************/
+   public static void setObject(String key, Serializable object)
+   {
+      if(objectsThreadLocal.get() == null)
+      {
+         objectsThreadLocal.set(new HashMap<>());
+      }
+      objectsThreadLocal.get().put(key, object);
+   }
+
+
+
+   /*******************************************************************************
+    ** remove a named object from the Context of the current thread.
+    *******************************************************************************/
+   public static void removeObject(String key)
+   {
+      if(objectsThreadLocal.get() != null)
+      {
+         objectsThreadLocal.get().remove(key);
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** get the full map of named objects for the current thread (possibly null).
+    *******************************************************************************/
+   public static Map<String, Serializable> getObjects()
+   {
+      return objectsThreadLocal.get();
+   }
+
+
+
+   /*******************************************************************************
+    ** fully replace the map of named objets for the current thread.
+    *******************************************************************************/
+   public static void setObjects(Map<String, Serializable> objects)
+   {
+      objectsThreadLocal.set(objects);
+   }
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java

@@ -77,6 +77,7 @@ import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.Bulk
 import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkInsertTransformStep;
 import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.ExtractViaQueryStep;
 import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess;
+import com.kingsrook.qqq.backend.core.scheduler.QScheduleManager;
 import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
 import com.kingsrook.qqq.backend.core.utils.ListingHash;
 import com.kingsrook.qqq.backend.core.utils.StringUtils;
@@ -159,6 +160,18 @@ public class QInstanceEnricher
       }
 
       enrichJoins();
+
+      //////////////////////////////////////////////////////////////////////////////
+      // if the instance DOES have 1 or more scheduler, but no schedulable types, //
+      // then go ahead and add the default set that qqq knows about               //
+      //////////////////////////////////////////////////////////////////////////////
+      if(CollectionUtils.nullSafeHasContents(qInstance.getSchedulers()))
+      {
+         if(CollectionUtils.nullSafeIsEmpty(qInstance.getSchedulableTypes()))
+         {
+            QScheduleManager.defineDefaultSchedulableTypesInInstance(qInstance);
+         }
+      }
    }
 
 
@@ -1030,6 +1043,50 @@ public class QInstanceEnricher
 
 
 
+   /*******************************************************************************
+    ** Do a default mapping from an underscore_style field name to a camelCase name.
+    **
+    ** Examples:
+    ** <ul>
+    **   <li>word_another_word_more_words -> wordAnotherWordMoreWords</li>
+    **   <li>l_ul_ul_ul -> lUlUlUl</li>
+    **   <li>tla_first -> tlaFirst</li>
+    **   <li>word_then_tla_in_middle -> wordThenTlaInMiddle</li>
+    **   <li>end_with_tla -> endWithTla</li>
+    **   <li>tla_and_another_tla -> tlaAndAnotherTla</li>
+    **   <li>ALL_CAPS -> allCaps</li>
+    ** </ul>
+    *******************************************************************************/
+   public static String inferNameFromBackendName(String backendName)
+   {
+      StringBuilder rs = new StringBuilder();
+
+      ////////////////////////////////////////////////////////////////////////////////////////
+      // build a list of words in the name, then join them with _ and lower-case the result //
+      ////////////////////////////////////////////////////////////////////////////////////////
+      String[] words = backendName.toLowerCase(Locale.ROOT).split("_");
+      for(int i = 0; i < words.length; i++)
+      {
+         String word = words[i];
+         if(i == 0)
+         {
+            rs.append(word);
+         }
+         else
+         {
+            rs.append(word.substring(0, 1).toUpperCase());
+            if(word.length() > 1)
+            {
+               rs.append(word.substring(1));
+            }
+         }
+      }
+
+      return (rs.toString());
+   }
+
+
+
    /*******************************************************************************
     ** If a app didn't have any sections, generate "sensible defaults"
     *******************************************************************************/

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceHelpContentManager.java

@@ -43,6 +43,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.help.HelpFormat;
 import com.kingsrook.qqq.backend.core.model.metadata.help.HelpRole;
 import com.kingsrook.qqq.backend.core.model.metadata.help.QHelpContent;
 import com.kingsrook.qqq.backend.core.model.metadata.help.QHelpRole;
+import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendStepMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QFieldSection;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
@@ -111,6 +112,7 @@ public class QInstanceHelpContentManager
          String processName = nameValuePairs.get("process");
          String fieldName   = nameValuePairs.get("field");
          String sectionName = nameValuePairs.get("section");
+         String stepName = nameValuePairs.get("step");
          String widgetName  = nameValuePairs.get("widget");
          String slotName    = nameValuePairs.get("slot");
 
@@ -145,12 +147,11 @@ public class QInstanceHelpContentManager
          }
          else if(StringUtils.hasContent(processName))
          {
-            processHelpContentForProcess(key, processName, fieldName, roles, helpContent);
+            processHelpContentForProcess(key, processName, fieldName, stepName, roles, helpContent);
          }
          else if(StringUtils.hasContent(widgetName))
          {
-            processHelpContentForWidget(key, widgetName, slotName, helpContent);
-
+            processHelpContentForWidget(key, widgetName, slotName, roles, helpContent);
          }
       }
       catch(Exception e)
@@ -209,6 +210,10 @@ public class QInstanceHelpContentManager
             optionalSection.get().removeHelpContent(roles);
          }
       }
+      else
+      {
+         LOG.info("Unrecognized key format for table help content", logPair("key", key));
+      }
    }
 
 
@@ -216,7 +221,7 @@ public class QInstanceHelpContentManager
    /*******************************************************************************
     **
     *******************************************************************************/
-   private static void processHelpContentForProcess(String key, String processName, String fieldName, Set<HelpRole> roles, QHelpContent helpContent)
+   private static void processHelpContentForProcess(String key, String processName, String fieldName, String stepName, Set<HelpRole> roles, QHelpContent helpContent)
    {
       QProcessMetaData process = QContext.getQInstance().getProcess(processName);
       if(process == null)
@@ -245,6 +250,30 @@ public class QInstanceHelpContentManager
             optionalField.get().removeHelpContent(roles);
          }
       }
+      else if(StringUtils.hasContent(stepName))
+      {
+         /////////////////////////////
+         // handle a process screen //
+         /////////////////////////////
+         QFrontendStepMetaData frontendStep = process.getFrontendStep(stepName);
+
+         if(frontendStep == null)
+         {
+            LOG.info("Unrecognized process step in help content", logPair("key", key));
+         }
+         else if(helpContent != null)
+         {
+            frontendStep.withHelpContent(helpContent);
+         }
+         else
+         {
+            frontendStep.removeHelpContent(roles);
+         }
+      }
+      else
+      {
+         LOG.info("Unrecognized key format for process help content", logPair("key", key));
+      }
    }
 
 
@@ -252,7 +281,7 @@ public class QInstanceHelpContentManager
    /*******************************************************************************
     **
     *******************************************************************************/
-   private static void processHelpContentForWidget(String key, String widgetName, String slotName, QHelpContent helpContent)
+   private static void processHelpContentForWidget(String key, String widgetName, String slotName, Set<HelpRole> roles, QHelpContent helpContent)
    {
       QWidgetMetaDataInterface widget = QContext.getQInstance().getWidget(widgetName);
       if(!StringUtils.hasContent(slotName))
@@ -265,22 +294,14 @@ public class QInstanceHelpContentManager
       }
       else
       {
-         Map<String, QHelpContent> widgetHelpContent = widget.getHelpContent();
-         if(widgetHelpContent == null)
-         {
-            widgetHelpContent = new HashMap<>();
-         }
-
          if(helpContent != null)
          {
-            widgetHelpContent.put(slotName, helpContent);
+            widget.withHelpContent(slotName, helpContent);
          }
          else
          {
-            widgetHelpContent.remove(slotName);
+            widget.removeHelpContent(slotName, roles);
          }
-
-         widget.setHelpContent(widgetHelpContent);
       }
    }
 

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidator.java

@@ -24,8 +24,11 @@ package com.kingsrook.qqq.backend.core.instances;
 
 import java.io.Serializable;
 import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
 import java.lang.reflect.Modifier;
+import java.text.ParseException;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashSet;
 import java.util.List;
@@ -33,6 +36,7 @@ import java.util.Map;
 import java.util.Objects;
 import java.util.Optional;
 import java.util.Set;
+import java.util.TimeZone;
 import java.util.function.Supplier;
 import java.util.stream.Stream;
 import com.kingsrook.qqq.backend.core.actions.automation.RecordAutomationHandler;
@@ -43,6 +47,7 @@ import com.kingsrook.qqq.backend.core.actions.processes.BackendStep;
 import com.kingsrook.qqq.backend.core.actions.scripts.TestScriptActionInterface;
 import com.kingsrook.qqq.backend.core.actions.values.QCustomPossibleValueProvider;
 import com.kingsrook.qqq.backend.core.exceptions.QInstanceValidationException;
+import com.kingsrook.qqq.backend.core.instances.validation.plugins.QInstanceValidatorPluginInterface;
 import com.kingsrook.qqq.backend.core.logging.QLogger;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy;
@@ -52,11 +57,14 @@ import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
 import com.kingsrook.qqq.backend.core.model.metadata.QSupplementalInstanceMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.authentication.QAuthenticationMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.automation.QAutomationProviderMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
 import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeType;
 import com.kingsrook.qqq.backend.core.model.metadata.dashboard.ParentWidgetMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.dashboard.QWidgetMetaDataInterface;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.AdornmentType;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.FieldAdornment;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.FieldBehavior;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.ValueTooLongBehavior;
 import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinOn;
@@ -64,16 +72,22 @@ import com.kingsrook.qqq.backend.core.model.metadata.joins.QJoinMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppChildMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppSection;
+import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSource;
 import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.processes.QStepMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.processes.QSupplementalProcessMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.queues.QQueueMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.queues.QQueueProviderMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.queues.SQSQueueProviderMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportDataSource;
 import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportField;
+import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportView;
 import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.QScheduleMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.security.FieldSecurityLock;
+import com.kingsrook.qqq.backend.core.model.metadata.security.MultiRecordSecurityLock;
+import com.kingsrook.qqq.backend.core.model.metadata.security.QSecurityKeyType;
 import com.kingsrook.qqq.backend.core.model.metadata.security.RecordSecurityLock;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.AssociatedScript;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.Association;
@@ -90,9 +104,11 @@ import com.kingsrook.qqq.backend.core.model.metadata.tables.cache.CacheOf;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.cache.CacheUseCase;
 import com.kingsrook.qqq.backend.core.modules.authentication.QAuthenticationModuleCustomizerInterface;
 import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
+import com.kingsrook.qqq.backend.core.utils.ListingHash;
 import com.kingsrook.qqq.backend.core.utils.StringUtils;
 import com.kingsrook.qqq.backend.core.utils.ValueUtils;
 import com.kingsrook.qqq.backend.core.utils.lambdas.UnsafeLambda;
+import org.quartz.CronExpression;
 
 
 /*******************************************************************************
@@ -111,6 +127,8 @@ public class QInstanceValidator
 
    private boolean printWarnings = false;
 
+   private static ListingHash<Class<?>, QInstanceValidatorPluginInterface<?>> validatorPlugins = new ListingHash<>();
+
    private List<String> errors = new ArrayList<>();
 
 
@@ -171,6 +189,8 @@ public class QInstanceValidator
          validateSupplementalMetaData(qInstance);
 
          validateUniqueTopLevelNames(qInstance);
+
+         runPlugins(QInstance.class, qInstance, qInstance);
       }
       catch(Exception e)
       {
@@ -189,6 +209,57 @@ public class QInstanceValidator
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public static void addValidatorPlugin(QInstanceValidatorPluginInterface<?> plugin)
+   {
+      Optional<Method> validateMethod = Arrays.stream(plugin.getClass().getDeclaredMethods())
+         .filter(m -> m.getName().equals("validate")
+            && m.getParameterCount() == 3
+            && !m.getParameterTypes()[0].equals(Object.class)
+            && m.getParameterTypes()[1].equals(QInstance.class)
+            && m.getParameterTypes()[2].equals(QInstanceValidator.class)
+         ).findFirst();
+
+      if(validateMethod.isPresent())
+      {
+         Class<?> parameterType = validateMethod.get().getParameterTypes()[0];
+         validatorPlugins.add(parameterType, plugin);
+      }
+      else
+      {
+         LOG.warn("Could not find validate method on validator plugin [" + plugin.getClass().getName() + "] (to infer type being validated) - this plugin will not be used.");
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public static void removeAllValidatorPlugins()
+   {
+      validatorPlugins.clear();
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private <T> void runPlugins(Class<T> c, T t, QInstance qInstance)
+   {
+      for(QInstanceValidatorPluginInterface<?> plugin : CollectionUtils.nonNullList(validatorPlugins.get(c)))
+      {
+         @SuppressWarnings("unchecked")
+         QInstanceValidatorPluginInterface<T> processPlugin = (QInstanceValidatorPluginInterface<T>) plugin;
+         processPlugin.validate(t, qInstance, this);
+      }
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/
@@ -197,6 +268,8 @@ public class QInstanceValidator
       for(QSupplementalInstanceMetaData supplementalInstanceMetaData : CollectionUtils.nonNullMap(qInstance.getSupplementalMetaData()).values())
       {
          supplementalInstanceMetaData.validate(qInstance, this);
+
+         runPlugins(QSupplementalInstanceMetaData.class, supplementalInstanceMetaData, qInstance);
       }
    }
 
@@ -234,6 +307,8 @@ public class QInstanceValidator
             {
                assertCondition(qInstance.getPossibleValueSource(securityKeyType.getPossibleValueSourceName()) != null, "Unrecognized possibleValueSourceName in securityKeyType: " + name);
             }
+
+            runPlugins(QSecurityKeyType.class, securityKeyType, qInstance);
          }
       });
    }
@@ -280,6 +355,8 @@ public class QInstanceValidator
                assertNoException(() -> qInstance.getTable(join.getRightTable()).getField(orderBy.getFieldName()), "Field name " + orderBy.getFieldName() + " in orderBy for join " + joinName + " is not a defined field in the right-table " + join.getRightTable());
             }
          }
+
+         runPlugins(QJoinMetaData.class, join, qInstance);
       });
    }
 
@@ -343,6 +420,8 @@ public class QInstanceValidator
                assertCondition(StringUtils.hasContent(sqsQueueProvider.getBaseURL()), "Missing baseURL for SQSQueueProvider: " + name);
                assertCondition(StringUtils.hasContent(sqsQueueProvider.getRegion()), "Missing region for SQSQueueProvider: " + name);
             }
+
+            runPlugins(QQueueProviderMetaData.class, queueProvider, qInstance);
          });
       }
 
@@ -357,6 +436,13 @@ public class QInstanceValidator
             {
                assertCondition(qInstance.getProcesses() != null && qInstance.getProcess(queue.getProcessName()) != null, "Unrecognized processName for queue: " + name);
             }
+
+            if(queue.getSchedule() != null)
+            {
+               validateScheduleMetaData(queue.getSchedule(), qInstance, "SQSQueueProvider " + name + ", schedule: ");
+            }
+
+            runPlugins(QQueueMetaData.class, queue, qInstance);
          });
       }
    }
@@ -375,6 +461,8 @@ public class QInstanceValidator
             assertCondition(Objects.equals(backendName, backend.getName()), "Inconsistent naming for backend: " + backendName + "/" + backend.getName() + ".");
 
             backend.performValidation(this);
+
+            runPlugins(QBackendMetaData.class, backend, qInstance);
          });
       }
    }
@@ -392,6 +480,8 @@ public class QInstanceValidator
          {
             assertCondition(Objects.equals(name, automationProvider.getName()), "Inconsistent naming for automationProvider: " + name + "/" + automationProvider.getName() + ".");
             assertCondition(automationProvider.getType() != null, "Missing type for automationProvider: " + name);
+
+            runPlugins(QAutomationProviderMetaData.class, automationProvider, qInstance);
          });
       }
    }
@@ -410,6 +500,8 @@ public class QInstanceValidator
          {
             validateSimpleCodeReference("Instance Authentication meta data customizer ", authentication.getCustomizer(), QAuthenticationModuleCustomizerInterface.class);
          }
+
+         runPlugins(QAuthenticationMetaData.class, authentication, qInstance);
       }
    }
 
@@ -532,6 +624,13 @@ public class QInstanceValidator
             {
                supplementalTableMetaData.validate(qInstance, table, this);
             }
+
+            if(table.getShareableTableMetaData() != null)
+            {
+               table.getShareableTableMetaData().validate(qInstance, table, this);
+            }
+
+            runPlugins(QTableMetaData.class, table, qInstance);
          });
       }
    }
@@ -618,7 +717,6 @@ public class QInstanceValidator
    {
       String prefix = "Table " + table.getName() + " ";
 
-      RECORD_SECURITY_LOCKS_LOOP:
       for(RecordSecurityLock recordSecurityLock : CollectionUtils.nonNullList(table.getRecordSecurityLocks()))
       {
          if(!assertCondition(recordSecurityLock != null, prefix + "has a null recordSecurityLock (did you mean to give it a null list of locks?)"))
@@ -626,90 +724,14 @@ public class QInstanceValidator
             continue;
          }
 
-         String securityKeyTypeName = recordSecurityLock.getSecurityKeyType();
-         if(assertCondition(StringUtils.hasContent(securityKeyTypeName), prefix + "has a recordSecurityLock that is missing a securityKeyType"))
+         if(recordSecurityLock instanceof MultiRecordSecurityLock multiRecordSecurityLock)
          {
-            assertCondition(qInstance.getSecurityKeyType(securityKeyTypeName) != null, prefix + "has a recordSecurityLock with an unrecognized securityKeyType: " + securityKeyTypeName);
+            validateMultiRecordSecurityLock(qInstance, table, multiRecordSecurityLock, prefix);
          }
-
-         prefix = "Table " + table.getName() + " recordSecurityLock (of key type " + securityKeyTypeName + ") ";
-
-         assertCondition(recordSecurityLock.getLockScope() != null, prefix + " is missing its lockScope");
-
-         boolean hasAnyBadJoins = false;
-         for(String joinName : CollectionUtils.nonNullList(recordSecurityLock.getJoinNameChain()))
+         else
          {
-            if(!assertCondition(qInstance.getJoin(joinName) != null, prefix + "has an unrecognized joinName: " + joinName))
-            {
-               hasAnyBadJoins = true;
-            }
+            validateRecordSecurityLock(qInstance, table, recordSecurityLock, prefix);
          }
-
-         String fieldName = recordSecurityLock.getFieldName();
-
-         ////////////////////////////////////////////////////////////////////////////////
-         // don't bother trying to validate field names if we know we have a bad join. //
-         ////////////////////////////////////////////////////////////////////////////////
-         if(assertCondition(StringUtils.hasContent(fieldName), prefix + "is missing a fieldName") && !hasAnyBadJoins)
-         {
-            if(fieldName.contains("."))
-            {
-               if(assertCondition(CollectionUtils.nullSafeHasContents(recordSecurityLock.getJoinNameChain()), prefix + "field name " + fieldName + " looks like a join (has a dot), but no joinNameChain was given."))
-               {
-                  List<QueryJoin> joins = new ArrayList<>();
-
-                  ///////////////////////////////////////////////////////////////////////////////////////////////////
-                  // ok - so - the join name chain is going to be like this:                                       //
-                  // for a table:  orderLineItemExtrinsic (that's 2 away from order, where the security field is): //
-                  // - securityFieldName = order.clientId                                                          //
-                  // - joinNameChain = orderJoinOrderLineItem, orderLineItemJoinOrderLineItemExtrinsic             //
-                  // so - to navigate from the table to the security field, we need to reverse the joinNameChain,  //
-                  // and step (via tmpTable variable) back to the securityField                                    //
-                  ///////////////////////////////////////////////////////////////////////////////////////////////////
-                  ArrayList<String> joinNameChain = new ArrayList<>(CollectionUtils.nonNullList(recordSecurityLock.getJoinNameChain()));
-                  Collections.reverse(joinNameChain);
-
-                  QTableMetaData tmpTable = table;
-
-                  for(String joinName : joinNameChain)
-                  {
-                     QJoinMetaData join = qInstance.getJoin(joinName);
-                     if(join == null)
-                     {
-                        errors.add(prefix + "joinNameChain contained an unrecognized join: " + joinName);
-                        continue RECORD_SECURITY_LOCKS_LOOP;
-                     }
-
-                     if(join.getLeftTable().equals(tmpTable.getName()))
-                     {
-                        joins.add(new QueryJoin(join));
-                        tmpTable = qInstance.getTable(join.getRightTable());
-                     }
-                     else if(join.getRightTable().equals(tmpTable.getName()))
-                     {
-                        joins.add(new QueryJoin(join.flip()));
-                        tmpTable = qInstance.getTable(join.getLeftTable());
-                     }
-                     else
-                     {
-                        errors.add(prefix + "joinNameChain could not be followed through join: " + joinName);
-                        continue RECORD_SECURITY_LOCKS_LOOP;
-                     }
-                  }
-
-                  assertCondition(findField(qInstance, table, joins, fieldName), prefix + "has an unrecognized fieldName: " + fieldName);
-               }
-            }
-            else
-            {
-               if(assertCondition(CollectionUtils.nullSafeIsEmpty(recordSecurityLock.getJoinNameChain()), prefix + "field name " + fieldName + " does not look like a join (does not have a dot), but a joinNameChain was given."))
-               {
-                  assertNoException(() -> table.getField(fieldName), prefix + "has an unrecognized fieldName: " + fieldName);
-               }
-            }
-         }
-
-         assertCondition(recordSecurityLock.getNullValueBehavior() != null, prefix + "is missing a nullValueBehavior");
       }
    }
 
@@ -718,7 +740,121 @@ public class QInstanceValidator
    /*******************************************************************************
     **
     *******************************************************************************/
-   private void validateTableField(QInstance qInstance, String tableName, String fieldName, QTableMetaData table, QFieldMetaData field)
+   private void validateMultiRecordSecurityLock(QInstance qInstance, QTableMetaData table, MultiRecordSecurityLock multiRecordSecurityLock, String prefix)
+   {
+      assertCondition(multiRecordSecurityLock.getOperator() != null, prefix + "has a MultiRecordSecurityLock that is missing an operator");
+
+      for(RecordSecurityLock lock : multiRecordSecurityLock.getLocks())
+      {
+         validateRecordSecurityLock(qInstance, table, lock, prefix);
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private void validateRecordSecurityLock(QInstance qInstance, QTableMetaData table, RecordSecurityLock recordSecurityLock, String prefix)
+   {
+      String securityKeyTypeName = recordSecurityLock.getSecurityKeyType();
+      if(assertCondition(StringUtils.hasContent(securityKeyTypeName), prefix + "has a recordSecurityLock that is missing a securityKeyType"))
+      {
+         assertCondition(qInstance.getSecurityKeyType(securityKeyTypeName) != null, prefix + "has a recordSecurityLock with an unrecognized securityKeyType: " + securityKeyTypeName);
+      }
+
+      prefix = "Table " + table.getName() + " recordSecurityLock (of key type " + securityKeyTypeName + ") ";
+
+      assertCondition(recordSecurityLock.getLockScope() != null, prefix + " is missing its lockScope");
+
+      boolean hasAnyBadJoins = false;
+      for(String joinName : CollectionUtils.nonNullList(recordSecurityLock.getJoinNameChain()))
+      {
+         if(!assertCondition(qInstance.getJoin(joinName) != null, prefix + "has an unrecognized joinName: " + joinName))
+         {
+            hasAnyBadJoins = true;
+         }
+      }
+
+      String fieldName = recordSecurityLock.getFieldName();
+
+      ////////////////////////////////////////////////////////////////////////////////
+      // don't bother trying to validate field names if we know we have a bad join. //
+      ////////////////////////////////////////////////////////////////////////////////
+      if(assertCondition(StringUtils.hasContent(fieldName), prefix + "is missing a fieldName") && !hasAnyBadJoins)
+      {
+         if(fieldName.contains("."))
+         {
+            if(assertCondition(CollectionUtils.nullSafeHasContents(recordSecurityLock.getJoinNameChain()), prefix + "field name " + fieldName + " looks like a join (has a dot), but no joinNameChain was given."))
+            {
+               String[] split         = fieldName.split("\\.");
+               String   joinTableName = split[0];
+               String   joinFieldName = split[1];
+
+               List<QueryJoin> joins = new ArrayList<>();
+
+               ///////////////////////////////////////////////////////////////////////////////////////////////////
+               // ok - so - the join name chain is going to be like this:                                       //
+               // for a table:  orderLineItemExtrinsic (that's 2 away from order, where the security field is): //
+               // - securityFieldName = order.clientId                                                          //
+               // - joinNameChain = orderJoinOrderLineItem, orderLineItemJoinOrderLineItemExtrinsic             //
+               // so - to navigate from the table to the security field, we need to reverse the joinNameChain,  //
+               // and step (via tmpTable variable) back to the securityField                                    //
+               ///////////////////////////////////////////////////////////////////////////////////////////////////
+               ArrayList<String> joinNameChain = new ArrayList<>(CollectionUtils.nonNullList(recordSecurityLock.getJoinNameChain()));
+               Collections.reverse(joinNameChain);
+
+               QTableMetaData tmpTable = table;
+
+               for(String joinName : joinNameChain)
+               {
+                  QJoinMetaData join = qInstance.getJoin(joinName);
+                  if(join == null)
+                  {
+                     errors.add(prefix + "joinNameChain contained an unrecognized join: " + joinName);
+                     return;
+                  }
+
+                  if(join.getLeftTable().equals(tmpTable.getName()))
+                  {
+                     joins.add(new QueryJoin(join));
+                     tmpTable = qInstance.getTable(join.getRightTable());
+                  }
+                  else if(join.getRightTable().equals(tmpTable.getName()))
+                  {
+                     joins.add(new QueryJoin(join.flip()));
+                     tmpTable = qInstance.getTable(join.getLeftTable());
+                  }
+                  else
+                  {
+                     errors.add(prefix + "joinNameChain could not be followed through join: " + joinName);
+                     return;
+                  }
+               }
+
+               assertCondition(Objects.equals(tmpTable.getName(), joinTableName), prefix + "has a joinNameChain doesn't end in the expected table [" + joinTableName + "] (was: " + tmpTable.getName() + ")");
+
+               assertCondition(findField(qInstance, table, joins, fieldName), prefix + "has an unrecognized fieldName: " + fieldName);
+            }
+         }
+         else
+         {
+            if(assertCondition(CollectionUtils.nullSafeIsEmpty(recordSecurityLock.getJoinNameChain()), prefix + "field name " + fieldName + " does not look like a join (does not have a dot), but a joinNameChain was given."))
+            {
+               assertNoException(() -> table.getField(fieldName), prefix + "has an unrecognized fieldName: " + fieldName);
+            }
+         }
+      }
+
+      assertCondition(recordSecurityLock.getNullValueBehavior() != null, prefix + "is missing a nullValueBehavior");
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private <T extends FieldBehavior<T>> void validateTableField(QInstance qInstance, String tableName, String fieldName, QTableMetaData table, QFieldMetaData field)
    {
       assertCondition(Objects.equals(fieldName, field.getName()),
          "Inconsistent naming in table " + tableName + " for field " + fieldName + "/" + field.getName() + ".");
@@ -731,12 +867,32 @@ public class QInstanceValidator
 
       String prefix = "Field " + fieldName + " in table " + tableName + " ";
 
+      ///////////////////////////////////////////////////
+      // validate things we know about field behaviors //
+      ///////////////////////////////////////////////////
       ValueTooLongBehavior behavior = field.getBehaviorOrDefault(qInstance, ValueTooLongBehavior.class);
       if(behavior != null && !behavior.equals(ValueTooLongBehavior.PASS_THROUGH))
       {
          assertCondition(field.getMaxLength() != null, prefix + "specifies a ValueTooLongBehavior, but not a maxLength.");
       }
 
+      Set<Class<FieldBehavior<T>>> usedFieldBehaviorTypes = new HashSet<>();
+      if(field.getBehaviors() != null)
+      {
+         for(FieldBehavior<?> fieldBehavior : field.getBehaviors())
+         {
+            Class<FieldBehavior<T>> behaviorClass = (Class<FieldBehavior<T>>) fieldBehavior.getClass();
+
+            errors.addAll(fieldBehavior.validateBehaviorConfiguration(table, field));
+
+            if(!fieldBehavior.allowMultipleBehaviorsOfThisType())
+            {
+               assertCondition(!usedFieldBehaviorTypes.contains(behaviorClass), prefix + "has more than 1 fieldBehavior of type " + behaviorClass.getSimpleName() + ", which is not allowed for this type");
+            }
+            usedFieldBehaviorTypes.add(behaviorClass);
+         }
+      }
+
       if(field.getMaxLength() != null)
       {
          assertCondition(field.getMaxLength() > 0, prefix + "has an invalid maxLength (" + field.getMaxLength() + ") - must be greater than 0.");
@@ -929,6 +1085,11 @@ public class QInstanceValidator
          assertCondition(qInstance.getAutomationProvider(providerName) != null, " has an unrecognized providerName: " + providerName);
       }
 
+      if(automationDetails.getSchedule() != null)
+      {
+         validateScheduleMetaData(automationDetails.getSchedule(), qInstance, prefix + " automationDetails, schedule: ");
+      }
+
       //////////////////////////////////
       // validate the status tracking //
       //////////////////////////////////
@@ -1265,12 +1426,17 @@ public class QInstanceValidator
             ///////////////////////////////////
             // validate steps in the process //
             ///////////////////////////////////
+            Set<String> usedStepNames = new HashSet<>();
             if(assertCondition(CollectionUtils.nullSafeHasContents(process.getStepList()), "At least 1 step must be defined in process " + processName + "."))
             {
                int index = 0;
                for(QStepMetaData step : process.getStepList())
                {
-                  assertCondition(StringUtils.hasContent(step.getName()), "Missing name for a step at index " + index + " in process " + processName);
+                  if(assertCondition(StringUtils.hasContent(step.getName()), "Missing name for a step at index " + index + " in process " + processName))
+                  {
+                     assertCondition(!usedStepNames.contains(step.getName()), "Duplicate step name [" + step.getName() + "] in process " + processName);
+                     usedStepNames.add(step.getName());
+                  }
                   index++;
 
                   ////////////////////////////////////////////
@@ -1316,13 +1482,17 @@ public class QInstanceValidator
             if(process.getSchedule() != null)
             {
                QScheduleMetaData schedule = process.getSchedule();
-               assertCondition(schedule.getRepeatMillis() != null || schedule.getRepeatSeconds() != null, "Either repeat millis or repeat seconds must be set on schedule in process " + processName);
+               validateScheduleMetaData(schedule, qInstance, "Process " + processName + ", schedule: ");
+            }
 
-               if(schedule.getVariantBackend() != null)
-               {
-                  assertCondition(qInstance.getBackend(schedule.getVariantBackend()) != null, "A variant backend was not found for " + schedule.getVariantBackend());
-                  assertCondition(schedule.getVariantRunStrategy() != null, "A variant run strategy was not set for " + schedule.getVariantBackend() + " on schedule in process " + processName);
-               }
+            if(process.getVariantBackend() != null)
+            {
+               assertCondition(qInstance.getBackend(process.getVariantBackend()) != null, "Process " + processName + ", a variant backend was not found named " + process.getVariantBackend());
+               assertCondition(process.getVariantRunStrategy() != null, "A variant run strategy was not set for process " + processName + " (which does specify a variant backend)");
+            }
+            else
+            {
+               assertCondition(process.getVariantRunStrategy() == null, "A variant run strategy was set for process " + processName + " (which isn't allowed, since it does not specify a variant backend)");
             }
 
             for(QSupplementalProcessMetaData supplementalProcessMetaData : CollectionUtils.nonNullMap(process.getSupplementalMetaData()).values())
@@ -1330,12 +1500,57 @@ public class QInstanceValidator
                supplementalProcessMetaData.validate(qInstance, process, this);
             }
 
+            runPlugins(QProcessMetaData.class, process, qInstance);
          });
       }
    }
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private void validateScheduleMetaData(QScheduleMetaData schedule, QInstance qInstance, String prefix)
+   {
+      boolean isRepeat = schedule.getRepeatMillis() != null || schedule.getRepeatSeconds() != null;
+      boolean isCron   = StringUtils.hasContent(schedule.getCronExpression());
+      assertCondition(isRepeat || isCron, prefix + " either repeatMillis or repeatSeconds or cronExpression must be set");
+      assertCondition(!(isRepeat && isCron), prefix + " both a repeat time and cronExpression may not be set");
+
+      if(isCron)
+      {
+         boolean hasDelay = schedule.getInitialDelayMillis() != null || schedule.getInitialDelaySeconds() != null;
+         assertCondition(!hasDelay, prefix + " a cron schedule may not have an initial delay");
+
+         try
+         {
+            CronExpression.validateExpression(schedule.getCronExpression());
+         }
+         catch(ParseException pe)
+         {
+            errors.add(prefix + " invalid cron expression: " + pe.getMessage());
+         }
+
+         if(assertCondition(StringUtils.hasContent(schedule.getCronTimeZoneId()), prefix + " a cron schedule must specify a cronTimeZoneId"))
+         {
+            String[]         availableIDs = TimeZone.getAvailableIDs();
+            Optional<String> first        = Arrays.stream(availableIDs).filter(id -> id.equals(schedule.getCronTimeZoneId())).findFirst();
+            assertCondition(first.isPresent(), prefix + " unrecognized cronTimeZoneId: " + schedule.getCronTimeZoneId());
+         }
+      }
+      else
+      {
+         assertCondition(!StringUtils.hasContent(schedule.getCronTimeZoneId()), prefix + " a non-cron schedule must not specify a cronTimeZoneId");
+      }
+
+      if(assertCondition(StringUtils.hasContent(schedule.getSchedulerName()), prefix + " is missing a scheduler name"))
+      {
+         assertCondition(qInstance.getScheduler(schedule.getSchedulerName()) != null, prefix + " is referencing an unknown scheduler name: " + schedule.getSchedulerName());
+      }
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/
@@ -1436,6 +1651,8 @@ public class QInstanceValidator
                   // view.getTitleFormat(); view.getTitleFields(); // validate these match?
                }
             }
+
+            runPlugins(QReportMetaData.class, report, qInstance);
          });
       }
    }
@@ -1556,9 +1773,9 @@ public class QInstanceValidator
                }
             }
 
-            //////////////////////////////////////////
-            // validate field sections in the table //
-            //////////////////////////////////////////
+            ////////////////////////////////////////
+            // validate field sections in the app //
+            ////////////////////////////////////////
             Set<String> childNamesInSections = new HashSet<>();
             if(app.getSections() != null)
             {
@@ -1586,6 +1803,8 @@ public class QInstanceValidator
             {
                assertCondition(qInstance.getWidget(widgetName) != null, "App " + appName + " widget " + widgetName + " is not a recognized widget.");
             }
+
+            runPlugins(QAppMetaData.class, app, qInstance);
          });
       }
    }
@@ -1618,6 +1837,8 @@ public class QInstanceValidator
                      }
                   }
                }
+
+               runPlugins(QWidgetMetaDataInterface.class, widget, qInstance);
             }
          );
       }
@@ -1703,6 +1924,8 @@ public class QInstanceValidator
                   }
                   default -> errors.add("Unexpected possibleValueSource type: " + possibleValueSource.getType());
                }
+
+               runPlugins(QPossibleValueSource.class, possibleValueSource, qInstance);
             }
          });
       }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/validation/plugins/BasepullExtractStepValidator.java

@@ -0,0 +1,87 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.instances.validation.plugins;
+
+
+import com.kingsrook.qqq.backend.core.actions.customizers.QCodeLoader;
+import com.kingsrook.qqq.backend.core.instances.QInstanceValidator;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
+import com.kingsrook.qqq.backend.core.processes.implementations.basepull.BasepullExtractStepInterface;
+
+
+/*******************************************************************************
+ ** instance validator plugin, to ensure that a process which is a basepull uses
+ ** an extract step marked for basepulls.
+ *******************************************************************************/
+public class BasepullExtractStepValidator implements QInstanceValidatorPluginInterface<QProcessMetaData>
+{
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public void validate(QProcessMetaData process, QInstance qInstance, QInstanceValidator qInstanceValidator)
+   {
+      ///////////////////////////////////////////////////////////////////////////
+      // if there's no basepull config on the process, don't do any validation //
+      ///////////////////////////////////////////////////////////////////////////
+      if(process.getBasepullConfiguration() == null)
+      {
+         return;
+      }
+
+      //////////////////////////////////////////////////////////////////////////////////////////
+      // try to find an input field in the process, w/ a defaultValue that's a QCodeReference //
+      // and is an instance of BasepullExtractStepInterface                                   //
+      //////////////////////////////////////////////////////////////////////////////////////////
+      boolean foundBasepullExtractStep = false;
+      for(QFieldMetaData field : process.getInputFields())
+      {
+         if(field.getDefaultValue() != null && field.getDefaultValue() instanceof QCodeReference codeReference)
+         {
+            try
+            {
+               BasepullExtractStepInterface extractStep = QCodeLoader.getAdHoc(BasepullExtractStepInterface.class, codeReference);
+               if(extractStep != null)
+               {
+                  foundBasepullExtractStep = true;
+               }
+            }
+            catch(Exception e)
+            {
+               //////////////////////////////////////////////////////
+               // ok, just means we haven't found our extract step //
+               //////////////////////////////////////////////////////
+            }
+         }
+      }
+
+      ///////////////////////////////////////////////////////////
+      // validate we could find a BasepullExtractStepInterface //
+      ///////////////////////////////////////////////////////////
+      qInstanceValidator.assertCondition(foundBasepullExtractStep, "Process [" + process.getName() + "] has a basepullConfiguration, but does not have a field with a default value that is a BasepullExtractStepInterface CodeReference");
+   }
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/validation/plugins/QInstanceValidatorPluginInterface.java

@@ -0,0 +1,41 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.instances.validation.plugins;
+
+
+import com.kingsrook.qqq.backend.core.instances.QInstanceValidator;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+
+
+/*******************************************************************************
+ ** Interface for additional / optional q instance validators.  Some will be
+ ** provided by QQQ - others can be defined by applications.
+ *******************************************************************************/
+public interface QInstanceValidatorPluginInterface<T>
+{
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   void validate(T object, QInstance qInstance, QInstanceValidator qInstanceValidator);
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/logging/CollectedLogMessage.java

@@ -0,0 +1,164 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.logging;
+
+
+import org.apache.logging.log4j.Level;
+import org.json.JSONException;
+import org.json.JSONObject;
+
+
+/*******************************************************************************
+ ** A log message, which can be "collected" by the QCollectingLogger.
+ *******************************************************************************/
+public class CollectedLogMessage
+{
+   private Level     level;
+   private String    message;
+   private Throwable exception;
+
+
+
+   /*******************************************************************************
+    ** Constructor
+    **
+    *******************************************************************************/
+   public CollectedLogMessage()
+   {
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public String toString()
+   {
+      return "CollectedLogMessage{level=" + level + ", message='" + message + '\'' + ", exception=" + exception + '}';
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for message
+    *******************************************************************************/
+   public String getMessage()
+   {
+      return (this.message);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for message
+    *******************************************************************************/
+   public void setMessage(String message)
+   {
+      this.message = message;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for message
+    *******************************************************************************/
+   public CollectedLogMessage withMessage(String message)
+   {
+      this.message = message;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for exception
+    *******************************************************************************/
+   public Throwable getException()
+   {
+      return (this.exception);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for exception
+    *******************************************************************************/
+   public void setException(Throwable exception)
+   {
+      this.exception = exception;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for exception
+    *******************************************************************************/
+   public CollectedLogMessage withException(Throwable exception)
+   {
+      this.exception = exception;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for level
+    **
+    *******************************************************************************/
+   public Level getLevel()
+   {
+      return level;
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for level
+    **
+    *******************************************************************************/
+   public void setLevel(Level level)
+   {
+      this.level = level;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for level
+    **
+    *******************************************************************************/
+   public CollectedLogMessage withLevel(Level level)
+   {
+      this.level = level;
+      return (this);
+   }
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public JSONObject getMessageAsJSONObject() throws JSONException
+   {
+      return (new JSONObject(getMessage()));
+   }
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/logging/LogPair.java

@@ -79,7 +79,7 @@ public class LogPair
       }
       else if(value instanceof LogPair[] subLogPairs)
       {
-         String subLogPairsString = Arrays.stream(subLogPairs).map(LogPair::toString).collect(Collectors.joining(","));
+         String subLogPairsString = Arrays.stream(subLogPairs).filter(Objects::nonNull).map(LogPair::toString).collect(Collectors.joining(","));
          valueString = '{' + subLogPairsString + '}';
       }
       else if(value instanceof UnsafeSupplier<?, ?> us)

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/logging/QCollectingLogger.java

@@ -0,0 +1,155 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.logging;
+
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Properties;
+import org.apache.logging.log4j.Level;
+import org.apache.logging.log4j.Logger;
+import org.apache.logging.log4j.Marker;
+import org.apache.logging.log4j.message.Message;
+import org.apache.logging.log4j.message.ObjectMessage;
+import org.apache.logging.log4j.message.SimpleMessageFactory;
+import org.apache.logging.log4j.simple.SimpleLogger;
+import org.apache.logging.log4j.util.PropertiesUtil;
+
+
+/*******************************************************************************
+ ** QQQ log4j implementation, used within a QLogger, to "collect" log messages
+ ** in an internal list - the idea being - for tests, to assert that logs happened.
+ *******************************************************************************/
+public class QCollectingLogger extends SimpleLogger
+{
+   private List<CollectedLogMessage> collectedMessages = new ArrayList<>();
+
+   ///////////////////////////////////////////////////////////////////////////////////////////////////////////
+   // just in case one of these gets activated, and left on, put a limit on how many messages we'll collect //
+   ///////////////////////////////////////////////////////////////////////////////////////////////////////////
+   private int capacity = 100;
+
+   private Logger logger;
+
+
+
+   /*******************************************************************************
+    ** Constructor
+    **
+    *******************************************************************************/
+   public QCollectingLogger(Logger logger)
+   {
+      super(logger.getName(), logger.getLevel(), false, false, true, false, "", new SimpleMessageFactory(), new PropertiesUtil(new Properties()), System.out);
+      this.logger = logger;
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public void logMessage(String fqcn, Level level, Marker marker, Message message, Throwable throwable)
+   {
+      ////////////////////////////////////////////
+      // add this log message to our collection //
+      ////////////////////////////////////////////
+      collectedMessages.add(new CollectedLogMessage()
+         .withLevel(level)
+         .withMessage(message.getFormattedMessage())
+         .withException(throwable));
+
+      ////////////////////////////////////////////////////////////////////////////////////////
+      // if we've gone over our capacity, remove the 1st entry until we're back at capacity //
+      ////////////////////////////////////////////////////////////////////////////////////////
+      while(collectedMessages.size() > capacity)
+      {
+         collectedMessages.remove(0);
+      }
+
+      //////////////////////////////////////////////////////////////////////
+      // update the message that we log to indicate that we collected it. //
+      // if it looks like JSON, insert as a name:value pair; else text.   //
+      //////////////////////////////////////////////////////////////////////
+      String formattedMessage = message.getFormattedMessage();
+      String updatedMessage;
+      if(formattedMessage.startsWith("{"))
+      {
+         updatedMessage = """
+            {"collected":true,""" + formattedMessage.substring(1);
+      }
+      else
+      {
+         updatedMessage = "[Collected] " + formattedMessage;
+      }
+      ObjectMessage myMessage = new ObjectMessage(updatedMessage);
+
+      ///////////////////////////////////////////////////////////////////////////////////////
+      // log the message with the original log4j logger, with our slightly updated message //
+      ///////////////////////////////////////////////////////////////////////////////////////
+      logger.logMessage(level, marker, fqcn, null, myMessage, throwable);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for logger
+    **
+    *******************************************************************************/
+   public void setLogger(Logger logger)
+   {
+      this.logger = logger;
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for collectedMessages
+    **
+    *******************************************************************************/
+   public List<CollectedLogMessage> getCollectedMessages()
+   {
+      return collectedMessages;
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public void clear()
+   {
+      this.collectedMessages.clear();
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for capacity
+    **
+    *******************************************************************************/
+   public void setCapacity(int capacity)
+   {
+      this.capacity = capacity;
+   }
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/logging/QLogger.java

@@ -22,6 +22,7 @@
 package com.kingsrook.qqq.backend.core.logging;
 
 
+import java.io.Serializable;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
@@ -119,6 +120,56 @@ public class QLogger
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public static QCollectingLogger activateCollectingLoggerForClass(Class<?> c)
+   {
+      Logger            loggerFromLogManager = LogManager.getLogger(c);
+      QCollectingLogger collectingLogger     = new QCollectingLogger(loggerFromLogManager);
+
+      QLogger qLogger = getLogger(c);
+      qLogger.setLogger(collectingLogger);
+
+      return collectingLogger;
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public static void deactivateCollectingLoggerForClass(Class<?> c)
+   {
+      Logger  loggerFromLogManager = LogManager.getLogger(c);
+      QLogger qLogger              = getLogger(c);
+      qLogger.setLogger(loggerFromLogManager);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public <T extends Throwable> T warnAndThrow(T t, LogPair... logPairs) throws T
+   {
+      warn(t.getMessage(), t, logPairs);
+      throw (t);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public <T extends Throwable> T errorAndThrow(T t, LogPair... logPairs) throws T
+   {
+      error(t.getMessage(), t, logPairs);
+      throw (t);
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/
@@ -518,7 +569,7 @@ public class QLogger
    /*******************************************************************************
     **
     *******************************************************************************/
-   private String makeJsonString(String message, Throwable t, List<LogPair> logPairList)
+   protected String makeJsonString(String message, Throwable t, List<LogPair> logPairList)
    {
       if(logPairList == null)
       {
@@ -567,7 +618,10 @@ public class QLogger
             {
                user = session.getUser().getIdReference();
             }
-            sessionLogPair = logPair("session", logPair("id", session.getUuid()), logPair("user", user));
+
+            LogPair variantsLogPair = getVariantsLogPair(session);
+
+            sessionLogPair = logPair("session", logPair("id", session.getUuid()), logPair("user", user), variantsLogPair);
          }
 
          try
@@ -587,6 +641,38 @@ public class QLogger
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private static LogPair getVariantsLogPair(QSession session)
+   {
+      LogPair variantsLogPair = null;
+      try
+      {
+         if(session.getBackendVariants() != null)
+         {
+            LogPair[] variants = new LogPair[session.getBackendVariants().size()];
+
+            int i = 0;
+            for(Map.Entry<String, Serializable> entry : session.getBackendVariants().entrySet())
+            {
+               variants[i] = new LogPair(entry.getKey(), entry.getValue());
+            }
+
+            variantsLogPair = new LogPair("variants", variants);
+         }
+      }
+      catch(Exception e)
+      {
+         ////////////////
+         // leave null //
+         ////////////////
+      }
+      return variantsLogPair;
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/
@@ -620,4 +706,15 @@ public class QLogger
       exceptionList.get(0).setHasLoggedLevel(level);
       return (level);
    }
+
+
+
+   /*******************************************************************************
+    ** Setter for logger
+    **
+    *******************************************************************************/
+   private void setLogger(Logger logger)
+   {
+      this.logger = logger;
+   }
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/AbstractActionInput.java

@@ -26,6 +26,7 @@ import java.util.UUID;
 import com.fasterxml.jackson.annotation.JsonIgnore;
 import com.kingsrook.qqq.backend.core.actions.async.AsyncJobCallback;
 import com.kingsrook.qqq.backend.core.actions.async.AsyncJobStatus;
+import com.kingsrook.qqq.backend.core.actions.async.NonPersistedAsyncJobCallback;
 import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.exceptions.QInstanceValidationException;
 import com.kingsrook.qqq.backend.core.instances.QInstanceValidator;
@@ -139,7 +140,7 @@ public class AbstractActionInput
          ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
          // don't return null here (too easy to NPE).  instead, if someone wants one of these, create one and give it to them. //
          ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-         asyncJobCallback = new AsyncJobCallback(UUID.randomUUID(), new AsyncJobStatus());
+         asyncJobCallback = new NonPersistedAsyncJobCallback(UUID.randomUUID(), new AsyncJobStatus().withJobName(getClass().getSimpleName()));
       }
       return asyncJobCallback;
    }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/audits/AuditDetailAccumulator.java

@@ -0,0 +1,156 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.model.actions.audits;
+
+
+import java.io.Serializable;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Optional;
+import com.kingsrook.qqq.backend.core.context.QContext;
+import com.kingsrook.qqq.backend.core.logging.QLogger;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.data.QRecordEntity;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
+
+
+/*******************************************************************************
+ ** Object to accumulate multiple audit-details to be recorded under a single
+ ** audit per-record, within a process step.  Especially useful if/when the
+ ** process step spreads its work out through multiple classes.
+ **
+ ** Pattern of usage looks like:
+ **
+ ** <pre>
+ ** // declare as a field (or local) w/ message for the audit headers
+ ** private AuditDetailAccumulator auditDetailAccumulator = new AuditDetailAccumulator("Audit header message");
+ **
+ **  // put into thread context
+ **  AuditDetailAccumulator.setInContext(auditDetailAccumulator);
+ **
+ **  // add a detail message for a record
+ **  auditDetailAccumulator.addAuditDetail(tableName, record, "Detail message");
+ **
+ **  // in another class, get the accumulator from context and safely add a detail message
+ **  AuditDetailAccumulator.getFromContext().ifPresent(ada -> ada.addAuditDetail(tableName, record, "More Details"));
+ **
+ **  // at the end of a step run/runOnePage method, add the accumulated audit details to step output
+ **  auditDetailAccumulator.getAccumulatedAuditSingleInputs().forEach(runBackendStepOutput::addAuditSingleInput);
+ **  auditDetailAccumulator.clear();
+ ** </pre>
+ *******************************************************************************/
+public class AuditDetailAccumulator implements Serializable
+{
+   private static final QLogger LOG = QLogger.getLogger(AuditDetailAccumulator.class);
+
+   private static final String objectKey = AuditDetailAccumulator.class.getSimpleName();
+
+   private String header;
+
+   private Map<TableNameAndPrimaryKey, AuditSingleInput> recordAuditInputMap = new HashMap<>();
+
+
+
+   /*******************************************************************************
+    ** Constructor
+    **
+    *******************************************************************************/
+   public AuditDetailAccumulator(String header)
+   {
+      this.header = header;
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public void setInContext()
+   {
+      QContext.setObject(objectKey, this);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public static Optional<AuditDetailAccumulator> getFromContext()
+   {
+      return QContext.getObject(objectKey, AuditDetailAccumulator.class);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public void addAuditDetail(String tableName, QRecordEntity entity, String message)
+   {
+      if(entity != null)
+      {
+         addAuditDetail(tableName, entity.toQRecord(), message);
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public void addAuditDetail(String tableName, QRecord record, String message)
+   {
+      QTableMetaData table      = QContext.getQInstance().getTable(tableName);
+      Serializable   primaryKey = record.getValue(table.getPrimaryKeyField());
+      if(primaryKey == null)
+      {
+         LOG.info("Missing primary key in input record - audit detail message will not be recorded.", logPair("message", message));
+         return;
+      }
+
+      AuditSingleInput auditSingleInput = recordAuditInputMap.computeIfAbsent(new TableNameAndPrimaryKey(tableName, primaryKey), (key) -> new AuditSingleInput(table, record, header));
+      auditSingleInput.addDetail(message);
+   }
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public Collection<AuditSingleInput> getAccumulatedAuditSingleInputs()
+   {
+      return (recordAuditInputMap.values());
+   }
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public void clear()
+   {
+      recordAuditInputMap.clear();
+   }
+
+
+   private record TableNameAndPrimaryKey(String tableName, Serializable primaryKey) {}
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/audits/AuditSingleInput.java

@@ -41,7 +41,7 @@ import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
 /*******************************************************************************
  ** Input data to insert a single audit record (with optional child record)..
  *******************************************************************************/
-public class AuditSingleInput
+public class AuditSingleInput implements Serializable
 {
    private String  auditTableName;
    private String  auditUserName;

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/messaging/Attachment.java

@@ -0,0 +1,95 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.model.actions.messaging;
+
+
+/*******************************************************************************
+ **
+ *******************************************************************************/
+public class Attachment
+{
+   private byte[] contents;
+   private String name;
+
+
+
+   /*******************************************************************************
+    ** Getter for contents
+    *******************************************************************************/
+   public byte[] getContents()
+   {
+      return (this.contents);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for contents
+    *******************************************************************************/
+   public void setContents(byte[] contents)
+   {
+      this.contents = contents;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for contents
+    *******************************************************************************/
+   public Attachment withContents(byte[] contents)
+   {
+      this.contents = contents;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for name
+    *******************************************************************************/
+   public String getName()
+   {
+      return (this.name);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for name
+    *******************************************************************************/
+   public void setName(String name)
+   {
+      this.name = name;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for name
+    *******************************************************************************/
+   public Attachment withName(String name)
+   {
+      this.name = name;
+      return (this);
+   }
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/messaging/Content.java

@@ -0,0 +1,95 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.model.actions.messaging;
+
+
+/*******************************************************************************
+ **
+ *******************************************************************************/
+public class Content
+{
+   private String      body;
+   private ContentRole contentRole;
+
+
+
+   /*******************************************************************************
+    ** Getter for body
+    *******************************************************************************/
+   public String getBody()
+   {
+      return (this.body);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for body
+    *******************************************************************************/
+   public void setBody(String body)
+   {
+      this.body = body;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for body
+    *******************************************************************************/
+   public Content withBody(String body)
+   {
+      this.body = body;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for contentRole
+    *******************************************************************************/
+   public ContentRole getContentRole()
+   {
+      return (this.contentRole);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for contentRole
+    *******************************************************************************/
+   public void setContentRole(ContentRole contentRole)
+   {
+      this.contentRole = contentRole;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for contentRole
+    *******************************************************************************/
+   public Content withContentRole(ContentRole contentRole)
+   {
+      this.contentRole = contentRole;
+      return (this);
+   }
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/messaging/ContentRole.java

@@ -0,0 +1,39 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.model.actions.messaging;
+
+
+/*******************************************************************************
+ **
+ *******************************************************************************/
+public interface ContentRole
+{
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   enum Default implements ContentRole
+   {
+      DEFAULT
+   }
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/messaging/MultiParty.java

@@ -0,0 +1,92 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.model.actions.messaging;
+
+
+import java.util.ArrayList;
+import java.util.List;
+
+
+/*******************************************************************************
+ **
+ *******************************************************************************/
+public class MultiParty extends Party
+{
+   private List<Party> partyList;
+
+
+
+   /*******************************************************************************
+    ** Getter for partyList
+    *******************************************************************************/
+   public List<Party> getPartyList()
+   {
+      return (this.partyList);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for partyList
+    *******************************************************************************/
+   public void setPartyList(List<Party> partyList)
+   {
+      this.partyList = partyList;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for partyList
+    *******************************************************************************/
+   public MultiParty withPartyList(List<Party> partyList)
+   {
+      this.partyList = partyList;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public MultiParty withParty(Party party)
+   {
+      addParty(party);
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public void addParty(Party party)
+   {
+      if(this.partyList == null)
+      {
+         this.partyList = new ArrayList<>();
+      }
+      this.partyList.add(party);
+   }
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/messaging/Party.java

@@ -0,0 +1,127 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.model.actions.messaging;
+
+
+/*******************************************************************************
+ **
+ *******************************************************************************/
+public class Party
+{
+   private String    label;
+   private String    address;
+   private PartyRole role;
+
+
+
+   /*******************************************************************************
+    ** Getter for label
+    *******************************************************************************/
+   public String getLabel()
+   {
+      return (this.label);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for label
+    *******************************************************************************/
+   public void setLabel(String label)
+   {
+      this.label = label;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for label
+    *******************************************************************************/
+   public Party withLabel(String label)
+   {
+      this.label = label;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for address
+    *******************************************************************************/
+   public String getAddress()
+   {
+      return (this.address);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for address
+    *******************************************************************************/
+   public void setAddress(String address)
+   {
+      this.address = address;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for address
+    *******************************************************************************/
+   public Party withAddress(String address)
+   {
+      this.address = address;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for role
+    *******************************************************************************/
+   public PartyRole getRole()
+   {
+      return (this.role);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for role
+    *******************************************************************************/
+   public void setRole(PartyRole role)
+   {
+      this.role = role;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for role
+    *******************************************************************************/
+   public Party withRole(PartyRole role)
+   {
+      this.role = role;
+      return (this);
+   }
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/messaging/PartyRole.java

@@ -0,0 +1,39 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.model.actions.messaging;
+
+
+/*******************************************************************************
+ **
+ *******************************************************************************/
+public interface PartyRole
+{
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   enum Default implements PartyRole
+   {
+      DEFAULT
+   }
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/messaging/SendMessageInput.java

@@ -0,0 +1,278 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.model.actions.messaging;
+
+
+import java.util.ArrayList;
+import java.util.List;
+import com.kingsrook.qqq.backend.core.model.actions.AbstractActionInput;
+
+
+/*******************************************************************************
+ **
+ *******************************************************************************/
+public class SendMessageInput extends AbstractActionInput
+{
+   private String           messagingProviderName;
+   private Party            to;
+   private Party            from;
+   private String           subject;
+   private List<Content>    contentList;
+   private List<Attachment> attachmentList;
+
+
+
+   /*******************************************************************************
+    ** Getter for to
+    *******************************************************************************/
+   public Party getTo()
+   {
+      return (this.to);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for to
+    *******************************************************************************/
+   public void setTo(Party to)
+   {
+      this.to = to;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for to
+    *******************************************************************************/
+   public SendMessageInput withTo(Party to)
+   {
+      this.to = to;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for from
+    *******************************************************************************/
+   public Party getFrom()
+   {
+      return (this.from);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for from
+    *******************************************************************************/
+   public void setFrom(Party from)
+   {
+      this.from = from;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for from
+    *******************************************************************************/
+   public SendMessageInput withFrom(Party from)
+   {
+      this.from = from;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for subject
+    *******************************************************************************/
+   public String getSubject()
+   {
+      return (this.subject);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for subject
+    *******************************************************************************/
+   public void setSubject(String subject)
+   {
+      this.subject = subject;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for subject
+    *******************************************************************************/
+   public SendMessageInput withSubject(String subject)
+   {
+      this.subject = subject;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for contentList
+    *******************************************************************************/
+   public List<Content> getContentList()
+   {
+      return (this.contentList);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for contentList
+    *******************************************************************************/
+   public void setContentList(List<Content> contentList)
+   {
+      this.contentList = contentList;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for contentList
+    *******************************************************************************/
+   public SendMessageInput withContentList(List<Content> contentList)
+   {
+      this.contentList = contentList;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for attachmentList
+    *******************************************************************************/
+   public List<Attachment> getAttachmentList()
+   {
+      return (this.attachmentList);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for attachmentList
+    *******************************************************************************/
+   public void setAttachmentList(List<Attachment> attachmentList)
+   {
+      this.attachmentList = attachmentList;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for attachmentList
+    *******************************************************************************/
+   public SendMessageInput withAttachmentList(List<Attachment> attachmentList)
+   {
+      this.attachmentList = attachmentList;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public SendMessageInput withContent(Content content)
+   {
+      addContent(content);
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public void addContent(Content content)
+   {
+      if(this.contentList == null)
+      {
+         this.contentList = new ArrayList<>();
+      }
+      this.contentList.add(content);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public SendMessageInput withAttachment(Attachment attachment)
+   {
+      addAttachment(attachment);
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public void addAttachment(Attachment attachment)
+   {
+      if(this.attachmentList == null)
+      {
+         this.attachmentList = new ArrayList<>();
+      }
+      this.attachmentList.add(attachment);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for messagingProviderName
+    *******************************************************************************/
+   public String getMessagingProviderName()
+   {
+      return (this.messagingProviderName);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for messagingProviderName
+    *******************************************************************************/
+   public void setMessagingProviderName(String messagingProviderName)
+   {
+      this.messagingProviderName = messagingProviderName;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for messagingProviderName
+    *******************************************************************************/
+   public SendMessageInput withMessagingProviderName(String messagingProviderName)
+   {
+      this.messagingProviderName = messagingProviderName;
+      return (this);
+   }
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/messaging/SendMessageOutput.java

@@ -0,0 +1,33 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.model.actions.messaging;
+
+
+import com.kingsrook.qqq.backend.core.model.actions.AbstractActionOutput;
+
+
+/*******************************************************************************
+ **
+ *******************************************************************************/
+public class SendMessageOutput extends AbstractActionOutput
+{
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/messaging/email/EmailContentRole.java

@@ -0,0 +1,35 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.model.actions.messaging.email;
+
+
+import com.kingsrook.qqq.backend.core.model.actions.messaging.ContentRole;
+
+
+/*******************************************************************************
+ **
+ *******************************************************************************/
+public enum EmailContentRole implements ContentRole
+{
+   TEXT,
+   HTML
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/messaging/email/EmailPartyRole.java

@@ -0,0 +1,38 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.model.actions.messaging.email;
+
+
+import com.kingsrook.qqq.backend.core.model.actions.messaging.PartyRole;
+
+
+/*******************************************************************************
+ **
+ *******************************************************************************/
+public enum EmailPartyRole implements PartyRole
+{
+   TO,
+   CC,
+   BCC,
+   FROM,
+   REPLY_TO
+}