CE-1107: added ability to use replaceAction specifying that null key values be treated as equal

Feature/ce 978 crashing nodes

.circleci/adjust-pom-version.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+
+if [ -z "$CIRCLE_BRANCH" ] && [ -z "$CIRCLE_TAG" ]; then
+   echo "Error:  env vars CIRCLE_BRANCH and CIRCLE_TAG were not set."
+   exit 1;
+fi
+
+if [ "$CIRCLE_BRANCH" == "dev" ] || [ "$CIRCLE_BRANCH" == "staging" ] || [ "$CIRCLE_BRANCH" == "main" ] || [ \! -z $(echo "$CIRCLE_TAG" | grep "^version-") ]; then
+   echo "On a primary branch or tag [${CIRCLE_BRANCH}${CIRCLE_TAG}] - will not edit the pom version.";
+   exit 0;
+fi
+
+if [ -n "$CIRCLE_BRANCH" ]; then
+   SLUG=$(echo $CIRCLE_BRANCH | sed 's/[^a-zA-Z0-9]/-/g')
+else
+   SLUG=$(echo $CIRCLE_TAG | sed 's/^snapshot-//g')
+fi
+
+POM=$(dirname $0)/../pom.xml
+
+echo "Updating $POM <revision> to: $SLUG-SNAPSHOT"
+sed -i "s/<revision>.*/<revision>$SLUG-SNAPSHOT<\/revision>/" $POM
+git diff $POM

.circleci/config.yml

@@ -1,7 +1,7 @@
 version: 2.1
 
 orbs:
-  localstack: localstack/platform@1.0
+  localstack: localstack/platform@2.1
 
 commands:
   store_jacoco_site:
@@ -26,6 +26,7 @@ commands:
             sudo rm /etc/alternatives/java
             sudo ln -s /usr/lib/jvm/java-17-openjdk-amd64/bin/java /etc/alternatives/java
       - run:
+          ## used by jacoco uncovered class reporting in pom.xml
           name: Install html2text
           command: |
             sudo apt-get update
@@ -51,10 +52,18 @@ commands:
           module: qqq-backend-module-filesystem
       - store_jacoco_site:
           module: qqq-backend-module-rdbms
+      - store_jacoco_site:
+          module: qqq-backend-module-api
+      - store_jacoco_site:
+          module: qqq-middleware-api
       - store_jacoco_site:
           module: qqq-middleware-javalin
       - store_jacoco_site:
           module: qqq-middleware-picocli
+      - store_jacoco_site:
+          module: qqq-middleware-slack
+      - store_jacoco_site:
+          module: qqq-language-support-javascript
       - store_jacoco_site:
           module: qqq-sample-project
       - run:
@@ -65,11 +74,6 @@ commands:
           when: always
       - store_test_results:
           path: ~/test-results
-      - run:
-          name: Find Un-tested Classes
-          command: |
-            set +o pipefail && for i in */target/site/jacoco/*/index.html; do html2text -width 500 -nobs $i | sed '1,/^Total/d;' | grep -v Created | sed 's/ \+/ /g' | sed 's/ [[:digit:]]$//' | grep -v 0$ | cut -d' ' -f1; done
-          when: always
       - save_cache:
           paths:
             - ~/.m2
@@ -78,6 +82,10 @@ commands:
   mvn_jar_deploy:
     steps:
       - checkout
+      - run:
+          name: Adjust pom version
+          command: |
+            .circleci/adjust-pom-version.sh
       - restore_cache:
           keys:
             - v1-dependencies-{{ checksum "pom.xml" }}
@@ -90,22 +98,54 @@ commands:
             - ~/.m2
           key: v1-dependencies-{{ checksum "pom.xml" }}
 
+  install_asciidoctor:
+    steps:
+      - checkout
+      - run:
+          name: Install asciidoctor
+          command: |
+            sudo apt-get update
+            sudo apt install -y asciidoctor
+
+  run_asciidoctor:
+    steps:
+      - run:
+          name: Run asciidoctor
+          command: |
+            cd docs
+            asciidoctor -a docinfo=shared index.adoc
+
+  upload_docs_site:
+    steps:
+      - run:
+          name: scp html to justinsgotskinnylegs.com
+          command: |
+            cd docs
+            scp index.html dkelkhoff@45.79.44.221:/mnt/first-volume/dkelkhoff/nginx/html/justinsgotskinnylegs.com/qqq-docs.html
+
 jobs:
   mvn_test:
     executor: localstack/default
     steps:
-      - localstack/startup
+      ## - localstack/startup
       - install_java17
       - mvn_verify
 
   mvn_deploy:
     executor: localstack/default
     steps:
-      - localstack/startup
+      ## - localstack/startup
       - install_java17
       - mvn_verify
       - mvn_jar_deploy
 
+  publish_asciidoc:
+    executor: localstack/default
+    steps:
+      - install_asciidoctor
+      - run_asciidoctor
+      - upload_docs_site
+
 workflows:
   test_only:
     jobs:
@@ -126,4 +166,7 @@ workflows:
               only: /dev/
             tags:
               only: /(version|snapshot)-.*/
-
+      - publish_asciidoc:
+          filters:
+            branches:
+              only: /dev/

README.md

@@ -10,7 +10,7 @@ The bundle contains all of the sub-jars.  It is named:
 
 ```qqq-${version}.jar```
 
-You can also use fine grained jars:
+You can also use fine-grained jars:
 - `qqq-backend-core`: The core module.  Useful if you're developing other modules.
 - `qqq-backend-module-rdbms`: Backend module for working with Relational Databases.
 - `qqq-backend-module-filesystem`: Backend module for working with Filesystems (including AWS S3).
@@ -35,4 +35,3 @@ GNU Affero General Public License for more details.
 
 You should have received a copy of the GNU Affero General Public License
 along with this program.  If not, see <https://www.gnu.org/licenses/>.
-

docs/.gitignore

@@ -0,0 +1 @@
+*.html

docs/Introduction.adoc

@@ -0,0 +1,176 @@
+= Introduction
+include::variables.adoc[]
+
+QQQ is a Low-code Application Framework for Engineers.
+Its purpose is to provide the basic structural elements of an application - things that every application needs - so that the engineers building an application on top of QQQ don't need to worry about those pieces, and can instead focus on the unique needs of the application that they are building.
+
+== What makes QQQ special?
+
+The scope of what QQQ provides is far-reaching, and most likely goes beyond what you may initially be thinking.
+That is to say, QQQ includes code all the way from the backend of an application, through its middleware layer, and including its frontend.
+For example, a common set of modules deployed in a QQQ application will provide:
+
+* Backend RDBMS/Database connectivity and access.
+* Frontend web UI (e.g., a React application)
+* A java web server acting as middleware between the frontend web UI and the backend
+
+That is to say - as an engineer deploying a QQQ application - you do not need to write a single line of code that is concerned with any of those things.
+
+* You do not need to write code to connect to your database.
+* You do not write any web UI code.
+* You do not write any middleware code to tie together the frontend and backend.
+
+Instead, QQQ includes *all* of these pieces.
+QQQ knows how to connect to databases (and actually, several other kinds of backend systems - but ignore that for now).
+Plus it knows how to do most of what an application needs to do with a database (single-record lookups, complex queries, joins, aggregates, bulk inserts, updates, deletes).
+QQQ also knows how to present the data from a database - in table views, or single-record views, or exports, or reports or widgets.
+And it knows how to present a powerful ad-hoc query interface to users, and how to show screens where users can create, update, and delete records.
+It also provides the connective tissue (middleware) between those backend layers where data is stored, and frontend layers were users interact with data.
+
+== What makes your application special?
+
+I've said a lot about what does QQQ knows - but let's dig a little deeper.
+What does QQQ know, and what does it not know?
+Well - what it doesn't know is, it doesn't know the special or unique aspects of the application that you are building.
+
+So, what makes your application unique?
+
+Is your application unique because it needs to have screens where users can search for records in a table?
+
+No!
+
+QQQ assumes (as does the author of this document) that all applications (at least of the nature that QQQ supports) need what we call Query Screens.
+So QQQ gives you a Query Screen for all of your tables - with zero code from you.
+
+Is your application unique because you want users to be able to view, create, edit, and delete records from tables?
+
+No!
+
+QQQ assumes that all applications need these basic https://en.wikipedia.org/wiki/Create,_read,_update_and_delete[CRUD] capabilities.
+So QQQ provides all of these UI screens - for view, create, edit, delete - along with the supporting middleware and backend code - all the way down to the SQL that selects & manages the data.
+You get it all for free - zero code.
+
+Is your application unique because you have a `fiz_bar` table, with 47 columns, and a `whoz_zat` table, with 42 columns of its own, that joins to `fiz_bar` on the `zizzy_ziz_id` field?
+
+Yes!
+
+OK - we found some of it - what makes your application unique is the data that you're working with.
+Your tables - their fields - the connection info for your database.
+QQQ doesn't know those details.
+So - that's your first job as a QQQ application engineer - to describe your data to QQQ.
+
+For the example above - you need to tell QQQ that you have a `fiz_bar` table, and that you have a `whoz_zat` table - and you need to tell QQQ what the fields or columns in thsoe tables are.
+You can even tell QQQ how to join those tables (on that `zizzy_ziz_id` field).
+And then that's it.
+Once QQQ has this {link-table} meta data, it can then provide its Query screens, and full CRUD screens to your tables, in your database, with your fields.
+
+And at the risk of repeating myself - you can do this (get a full Query & CRUD application) with zero lines of actual procedural code.
+You only need to supply meta-data (which, at the time of this writing, is done in Java, but it's just creating objects - and in the future could be done in YAML files, for example).
+
+== Beyond Basics
+Going beyond the basic wiring as described above, QQQ also provides some of the more advanced elements needed in a modern data-driven web application, including:
+
+* Authentication & Authorization
+* ad-hoc Query engine for access to data tables
+* Full CRUD (Create, Read, Update, Delete) capabilities
+* Multistep custom workflows ("Processes" in QQQ parlance)
+* Scheduled jobs
+* Enterprise Service Bus
+
+So what do we mean by all of this?
+We said that basically every application needs, for example, Authentication & Authorization.
+Login screens.
+User & Role tables.
+Permissions.
+So, when it's time for you to build a new application for your _Big Tall Floor Lamp_ manufacturing business, do you need to start by writing a login screen?
+And a Permissions scheme?
+And throwing HTTP 401 errors?
+And managing user-role relationships?
+And then having a bug in the check permission logic on the _Light Bulb Inventory Edit_ screen, so Jim is always keying in bad quantities, even though he isn't supposed to have permission there?
+
+No!
+
+All of the (really important, even though application developers hate doing it) aspects of security - you don't need to write ANY code for dealing with that.
+Just tell QQQ what Authentication provider you want to use (e.g., https://auth0.com/[Auth0]), and - to paraphrase the old https://www.youtube.com/watch?v=YHzM4avGrKI[iMac ad] - there's no step 2.
+QQQ just does it.
+
+''''
+
+QQQ can provide this type of application using a variety and/or combination of backend data storage types.
+And, whichever type of backend is used, QQQ gives a common interface (both user-facing and programmer-facing).
+Backend types include:
+
+* Relational Databases (RDBMS)
+* File Systems
+* Web APIs
+* NoSQL/Document Databases (_Future_)
+
+In addition, out-of-the-box, QQQ also goes beyond these basics, delivering:
+
+* Bulk versions of all CRUD operations.
+* Automatically generated JSON APIs.
+* Auditing of data changes.
+* End-user (e.g., non-engineer) customization via dynamic scripting capabilities
+
+#todo say much more#
+
+== QQQ Architecture
+
+Like a house!
+
+== Developing a QQQ Application
+In developing an application with QQQ, engineers will generally have to define two types of code:
+
+. *Meta Data* - This is the code that you use to tell QQQ the shape of your application - your unique Tables, Processes, Apps, Reports, Widgets, etc.
+In general, this code is 100% declarative in nature (as opposed to procedural or functional).
+That is to say - it has no logic.
+It is just a definition of what exists - but it has no instructions, algorithms, or business logic.
+
+* _In a future version of QQQ, we anticipate being able to define meta-data in a format such as YAML or JSON, or to even load it from a relational or document database.
+This speaks to the fact that this "code" is not executable code - but rather is a simple declaration of (meta) data._
+* A key function of QQQ then is to drive all of its layers of functionality - frontend UIs, middleware, and core backend actions (e.g., ORM operations) - based on this meta-data.
+** For example:
+... You define the meta-data for a table in your application - including its fields and their data types, as well as what backend system the table exists within.
+... Then, the QQQ Frontend Material Dashboard UI's Query Screen loads that table's meta-data, and uses it to control the screen that is presented.  Including:
+**** The data grid shown on the screen will have columns for each field in the table.
+**** The Filter button in the Query Screen will present a menu listing all fields from the table for the user to build ad-hoc queries against the table.
+The data-types specified for the fields (in the meta-data) dictate what operators QQQ allows the user to use against fields (e.g., Strings offer "contains" vs Numbers offer "greater than").
+**** Values for records from the table will be formatted for presentation based on the meta-data (such as a numeric field being shown with commas if it represents a quantity, or formatted as currency).
+...
+
+[start=2]
+. *Meta Data* - declarative code - java object instances (potentially which could be read from `.yaml` files or other data sources in a future version of QQQ), which tell QQQ about the backend systems, tables, processes, reports, widgets, etc, that make up the application.
+For example:
+* Details about the database you are using, and how to connect to it.
+* A database table's name, fields, their types, its keys, and basic business rules (required fields, read-only fields, field lengths).
+* The description of web API - its URL and authentication mechanism.
+* A table/path within a web API, and the fields returned in the JSON at that endpoint.
+* The specification of a custom workflow (process), including what screens are needed, with input & output values, and references to the custom application code for processing the data.
+* Details about a chart that summarizes data from a table for presentation as a dashboard widget.
+// the section below is kinda dumb.  like, it says you have to write application code, but
+// then it just talks about how your app code gets for-free the same shit that QQQ does.
+// it should instead say more about what your custom app code is or does.
+// 2. *Application code* - to customize beyond what the framework does out-of-the box, and to provide application-specific business-logic.
+// QQQ provides its programmers the same classes that it internally uses for record access, resulting in a unified application model.
+// For example:
+// * The same record-security model that is enforced for ad-hoc user queries through the frontend is applied to custom application code.
+// ** So if your table has a security key defined, which says that users can only see Order records that are associated with the user's assigned Store, then QQQ's order Query Screen will enforce that rule.
+// ** And at the same time - any custom processes ran by a user will have the same security applied to any queries that they run against the Order table.
+// ** And any custom dashboard widgets - will only include data that the user is allowed to see.
+// * Record audits are performed in custom code the same as they are in framework-driven actions.
+// ** So if a custom process edits a record, details of the changed fields show up in the record's audit, the same as if the record was edited using the standard QQQ edit action.
+// * Changed records are sent through the ESB automatically regardless of whether they are updated by custom application code or standard framework code.
+// ** Meaning record automations are triggered regardless of how a record is created or edited - without you, as an application engineering, needing to send records through the bus.
+// * The multi-threaded, paged producer/consumer pattern used in standard framework actions is how all custom application actions are also invoked.
+// ** For example, the standard QQQ Bulk Edit action uses the same streamed-ETL process that custom application processes can use.
+// Meaning your custom processes can take full advantage of the same complex frontend, middleware, and backend structural pieces, and you can just focus on your unique busines logic needs.
+2. *Application code* - to customize beyond what the QQQ framework does out-of-the box, and to provide application-specific business-logic.
+QQQ provides its programmers the same classes that it internally uses for record access, resulting in a unified application model.
+For example:
+
+== Lifecycle?
+* define meta data
+** enrichment
+** validation
+* start application
+* for dev - hotSwap

docs/actions/GetAction.adoc

@@ -0,0 +1,64 @@
+== GetAction
+include::../variables.adoc[]
+
+The `*GetAction*` is essentially a subset of the <<QueryAction>>, only specifically meant to get just a single record from a {link-table}.
+In SQL/RDBMS terms, it is analogous to a `SELECT` statement, where a single record may be found - or - it may not be found.
+
+For all tables, `GetAction` can do a lookup by primary key.
+In addition, for tables that define a `UniqueKey`, name/value pairs (in the form of a `Map`) can be used as input for `GetAction`.
+
+=== Examples
+[source,java]
+.Basic form - by primary key
+----
+GetInput input = new GetInput();
+input.setTableName("orders");
+input.setPrimaryKey(1);
+GetOutput output = new GetAction.execute(input);
+QRecord record = output.getRecord();
+----
+
+[source,java]
+.Secondary form - by Unique Key
+----
+GetInput input = new GetInput();
+input.setTableName("products");
+input.setUniqueKey(Map.of("storeId", 1701, "sku", "ABCD"));
+GetOutput output = new GetAction.execute(input);
+QRecord record = output.getRecord();
+----
+
+=== GetInput
+* `table` - *String, Required* - Name of the table being queried against.
+* `primaryKey` - *Serializable, Conditional* - Value for the primary key field of the table being queried.
+Type should match the table's primary key field's type.
+If a `primaryKey` is not given, then a `uniqueKey` must be given.
+* `uniqueKey` - *Map of String -> Serializable, Conditional* - Map of name-value pairs that define the record to be fetcheed.
+Keys in the map must be field names from the table being queried.
+Values in the map should should be of types that correspond to the fields.
+If a `primaryKey` is not given, then a `uniqueKey` must be given.
+If both `primaryKey` and `uniqueKey` are given, then `uniqueKey` is ignored.
+* `transaction` - *QBackendTransaction object* - Optional transaction object.
+** Behavior for this object is backend-dependant.
+In an RDBMS backend, this object is generally needed if you want your query to see data that may have been modified within the same transaction.
+
+* `shouldTranslatePossibleValues` - *boolean, default: false* - Controls whether any fields in the table with a *possibleValueSource* assigned to them should have those possible values looked up
+(e.g., to provide text translations in the generated records' `displayValues` map).
+** For example, if getting a record to present to a user, this would generally need to be *true*.
+But if getting a record as part of a process, then this can generally be left as *false*.
+* `shouldGenerateDisplayValues` - *boolean, default: false* - Controls whether field level *displayFormats* should be used to populate the generated records' `displayValues` map.
+** For example, if getting a record to present to a user, this would generally need to be *true*.
+But if getting a record as part of a process, then this can generally be left as *false*.
+* `shouldFetchHeavyFields` - *boolean, default: true* - Controls whether or not fields marked as `isHeavy` should be fetched & returned or not.
+* `shouldOmitHiddenFields` - *boolean, default: true* - Controls whether or not fields marked as `isHidden` should be included in the result or not.
+* `shouldMaskPassword` - *boolean, default: true* - Controls whether or not fields with `type` = `PASSWORD` should be masked, or if their actual values should be returned.
+* `queryJoins` - *List of <<QueryJoin>> objects* - Optional list of tables to be joined with the main table being queried.
+See QueryJoin under <<QueryAction>> for further details.
+* `includeAssociations` - *boolean, default: false* - Control whether or not records from tables defined as `associations` under the table being queried should be included in the result or not.
+* `associationNamesToInclude` - *Collection of String* - If `includeAssociations` is true, then this field can be used to limit which associated tables are included.
+If this field is null, then all associated tables are included.
+Otherwise, a table is only included if its name is in this collection.
+
+=== GetOutput
+* `record` - *QRecord* - The record that was specified by the input `primaryKey` or `uniqueKey`.
+Will be null if the record was not found.

docs/actions/InsertAction.adoc

@@ -0,0 +1,86 @@
+== InsertAction
+include::../variables.adoc[]
+
+To insert (add, create) new records into any {link-table}, the `*InsertAction*` is used.
+In SQL/RDBMS terms, it is analogous to a `INSERT` statement, where one or more records can be provided as input.
+
+=== Examples
+[source,java]
+.Canonical InsertAction invocation
+----
+InsertInput insertInput = new InsertInput();
+insertInput.setTableName("person");
+insertInput.setRecords(personRecordList);
+InsertOutput insertOutput = new InsertAction().execute(insertInput);
+List<QRecord> insertedPersonRecords = insertOutput.getRecords();
+----
+
+=== Details
+`InsertAction` does several things beyond just inserting records into the specified table.
+A high-level flow of its internal logic is:
+
+. For tables using an automation status field, set its value to `PENDING_INSERT_AUTOMATIONS` for all `records` that are going to be inserted.
+. Perform the following validations, which include running the table's `PRE_INSERT_CUSTOMIZER`, if one is defined, at the time that is specified by the customizer's `WhenToRun` property (default is `AFTER_ALL_VALIDATIONS`):
+.. Ensure that default values specified in the table's fields are present if needed.
+.. Apply per-field behaviors, as defined in {link-field} meta-data, such as truncating strings that are longer than their specified max.
+.. Check for unique key violations (done here instead of in the backend, to provide better error messaging, and to allow a subset of records to be stored while some fail).
+_We might want to make an input control in the future to specify that either the full input set should succeed or fail..._
+.. Validate that required fields (again, per {link-field} meta-data) are set, generating per-record errors if they are not.
+.. Validate any security fields in the records - e.g., ensure that the user has permission to insert records with the values they are attempting to insert.
+. Send the records to the table's backend module to actually insert them into the backend storage.
+. If the table has any associations defined, and if associated records are present, then recursively run `InsertAction` on the associated records.
+.. In particular, before these recursive `InsertAction` calls are made, values that were generated by the original insert may need to be propagated down into the associated records.
+*** For example, if inserting `order` and `lineItem` records, where a {link-join} exists between the two tables on `order.id` and `lineItem.orderId`, and `order.id` values were generated in the first `InsertAction`, then those values are propagated down into the associated `lineItem.orderId` fields.
+. If the {link-instance} has an `audit` table, then based on the {link-table}'s audit rules, audits about the inserted records are created.
+. If the table has a `POST_INSERT_CUSTOMIZER`, it is executed.
+
+=== Overloads
+
+`InsertAction` can be called in a few alternate forms, mostly just for convenience:
+
+[source,java]
+.If inserting a single record, get that record back instead of the InsertOutput:
+----
+InsertInput insertInput = new InsertInput();
+insertInput.setTableName("person");
+insertInput.setRecords(List.of(personRecord));
+QRecord insertedRecord = new InsertAction().executeForRecord(insertInput);
+
+// or more compactly, using InsertInput.withRecord (instead of withRecords)
+QRecord insertedRecord = new InsertAction()
+   .executeForRecord(new InsertInput("person").withRecord(personRecord));
+----
+
+[source,java]
+.Taking QRecordEntity objects as inputs instead of QRecords:
+----
+// insert a list of person entities:
+InsertInput insertInput = new InsertInput("person").withRecordEntities(personList);
+InsertOutput insertOutput = new InsertAction().execute(insertInput);
+
+// or for a single person entity (also mapping the output record back to an entity):
+Person insertedPerson = new Person(new InsertAction()
+   .executeForRecord(new InsertInput("person").withRecordEntity(person)));
+
+----
+
+=== InsertInput
+* `table` - *String, Required* - Name of the table that records are to be inserted into.
+* `records` - *List of QRecord, Required* - List of records to be inserted into the table.
+If the list is empty, the insert action does a `noop`.
+
+.Less common options
+* `inputSource` - *InputSource object, default: QInputSource.SYSTEM* - an indicator of what the source of the action is - generally, a `SYSTEM` triggered action, or a `USER` triggered action.
+** `InsertAction` will call the `shouldValidateRequiredFields()` method on this object to determine if it should validate that required fields on the records have values.
+Both `QInputSource.SYSTEM` and `QInputSource.USER` return `true` from this method, but application can define their own `InputSource` objects with different behavior.
+** In addition, this field can be used in pre- and post-insert customizers to drive further custom logic.
+* `skipUniqueKeyCheck` - *boolean, default: false* - control whether or not `InsertAction` should check for unique key violations before attempting to insert its records.
+** In a context where one has previously done this validation, or is okay letting the backend provide such checks, they may wish to avoid re-doing this work, and thus may set this property to `true`.
+* `omitDmlAudit` - *boolean, default: false* - control if the automatic DML audit that `InsertAction` generally performs should be omitted.
+* `auditContext` - *String* - optional message which can be included in DML audit messages, to give users more context about why the insert occurred.
+
+
+=== InsertOutput
+* `records` - *List of QRecord* - Copy of the input list of records, with details added based on the results of the input action.
+** If there were warnings or errors, the corresponding field (`warnings` or `errors`) will be set in the records.
+** If the insert action generated any values (such as a serial id or a default value), those values will be in the record's `fields` map.

docs/actions/QueryAction.adoc

@@ -0,0 +1,192 @@
+== QueryAction
+include::../variables.adoc[]
+
+The `*QueryAction*` is the basic action that is used to get records from a {link-table}, generally according to a <<QQueryFilter,Filter>>.
+In SQL/RDBMS terms, it is analogous to a `SELECT` statement, where 0 or more records may be found and returned.
+
+=== Examples
+==== Basic Form
+[source,java]
+----
+QueryInput input = new QueryInput();
+input.setTableName("orders");
+input.setFilter(new QQueryFilter()
+   .withCriteria(new QFilterCriteria("total", GREATER_THAN, new BigDecimal("3.50")))
+   .withOrderBy(new QFilterOrderBy("orderDate", false)));
+QueryOutput output = new QueryAction.execute(input);
+List<QRecord> records = output.getRecords();
+----
+
+=== Details
+`QueryAction`, in general, can be called in two different modes:
+
+. The most common use-case case, and default, fetches all records synchronously, does any post-processing (as requested in the <<QueryInput>>), and returns all records as a list in the <<QueryOutput>>).
+. The alternative use-case is meant for larger operations, where one wouldn't want all records matching a query in-memory.
+For this scenario, a `RecordPipe` object can be passed in to the <<QueryInput>>.
+This causes `QueryAction` to run its post-processing action on records as they are placed into the pipe, and to potentially block (per the pipe's settings).
+This method of usage needs to be done on a separate thread from another thread which would be consuming records from the pipe.
+QQQ's `AsyncRecordPipeLoop` class provides an implementation of doing such a dual-threaded job.
+
+If the {link-table} has a `POST_QUERY_CUSTOMIZER` defined, then after records are fetched from the backend, that code is executed on the records before they leave the `QueryAction` (either through its `QueryOutput` or `RecordPipe`).
+
+=== QueryInput
+* `table` - *String, Required* - Name of the table being queried against.
+* `filter` - *<<QQueryFilter>> object* - Specification for what records should be returned, based on *<<QFilterCriteria>>* objects, and how they should be sorted, based on *<<QFilterOrderBy>>* objects.
+If a `filter` is not given, then all rows in the table will be returned by the query.
+* `skip` - *Integer* - Optional number of records to be skipped at the beginning of the result set.
+e.g., for implementing pagination.
+* `limit` - *Integer* - Optional maximum number of records to be returned by the query.
+* `transaction` - *QBackendTransaction object* - Optional transaction object.
+** Behavior for this object is backend-dependant.
+In an RDBMS backend, this object is generally needed if you want your query to see data that may have been modified within the same transaction.
+* `recordPipe` - *RecordPipe object* - Optional pipe object that records are placed into, for asynchronous processing.
+** If a *recordPipe* is used, then records cannot be retrieved from the *QueryOutput*.
+Rather, such records must be read from the pipe's `consumeAvailableRecords()` method.
+** A *recordPipe* should only be used when a *QueryAction* is running in a separate Thread from the record's consumer.
+* `shouldTranslatePossibleValues` - *boolean, default: false* - Controls whether any fields in the table with a *possibleValueSource* assigned to them should have those possible values looked up
+(e.g., to provide text translations in the generated records' `displayValues` map).
+** For example, if running a query to present results to a user, this would generally need to be *true*.
+But if running a query to provide data as part of a process, then this can generally be left as *false*.
+* `shouldGenerateDisplayValues` - *boolean, default: false* - Controls whether field level *displayFormats* should be used to populate the generated records' `displayValues` map.
+** For example, if running a query to present results to a user, this would generally need to be *true*.
+But if running a query to provide data as part of a process, then this can generally be left as *false*.
+* `shouldFetchHeavyFields` - *boolean, default: true* - Controls whether or not fields marked as `isHeavy` should be fetched & returned or not.
+* `shouldOmitHiddenFields` - *boolean, default: true* - Controls whether or not fields marked as `isHidden` should be included in the result or not.
+* `shouldMaskPassword` - *boolean, default: true* - Controls whether or not fields with `type` = `PASSWORD` should be masked, or if their actual values should be returned.
+* `queryJoins` - *List of <<QueryJoin>> objects* - Optional list of tables to be joined with the main table being queried.
+See QueryJoin below for further details.
+
+==== QQueryFilter
+A key component of *<<QueryInput>>*, a *QQueryFilter* defines both what records should be included in a query's results (e.g., an SQL `WHERE`), as well as how those results should be sorted (SQL `ORDER BY`).
+
+* `criteria` - *List of <<QFilterCriteria>>* - Individual conditions or clauses to filter records.
+They are combined using the *booleanOperator* specified in the *QQueryFilter*.  See below for further details.
+* `orderBys` - *List of <<QFilterOrderBy>>* - List of fields (and directions) to control the sorting of query results.
+In general, multiple *orderBys* can be given (depending on backend implementations).
+* `booleanOperator` - *Enum of AND, OR, default: AND* - Specifies the logical joining operator used among individual criteria.
+* `subFilters` - *List of QQueryFilter* - To build arbitrarily complex queries, with nested boolean logic, 0 or more *subFilters* may be provided.
+** Each *subFilter* can include its own additional *subFilters*.
+** Each *subFilter* can specify a different *booleanOperator*.
+** For example, consider the following *QQueryFilter*, that uses two *subFilters*, and a mix of *booleanOperators*
+
+[source,java]
+----
+ queryInput.setFilter(new QQueryFilter()
+    .withBooleanOperator(OR)
+    .withSubFilters(List.of(
+       new QQueryFilter().withBooleanOperator(AND)
+          .withCriteria(new QFilterCriteria("firstName", EQUALS, "James"))
+          .withCriteria(new QFilterCriteria("lastName", EQUALS, "Maes")),
+       new QQueryFilter().withBooleanOperator(AND)
+          .withCriteria(new QFilterCriteria("firstName", EQUALS, "Darin"))
+          .withCriteria(new QFilterCriteria("lastName", EQUALS, "Kelkhoff"))
+    )));
+
+// which would generate the following WHERE clause in an RDBMS backend:
+// WHERE (first_name='James' AND last_name='Maes') OR (first_name='Darin' AND last_name='Kelkhoff')
+----
+
+===== QFilterCriteria
+* `fieldName` - *String, required* - Reference to a field on the table being queried.
+** Or, in the case of a query with *queryJoins*, a qualified name of a field from a join-table (where the qualifier would be the joined table's name or alias, followed by a dot)
+*** For example:  `orderLine.sku` or `orderBillToCustomer.firstName`
+* `operator` - *Enum of QCriteriaOperator, required* - Comparison operation to be applied to the field specified as *fieldName* and the *values* or *otherFieldName*.
+** e.g., `EQUALS`, `NOT_IN`, `GREATER_THAN`, `BETWEEN`, `IS_BLANK`, etc.
+* `values` - *List of values, conditional* - Provides the value(s) that the field is compared against.
+The number of values (0, 1, 2, or more) required are based on the *operator* being used.
+If an *otherFieldName* is given, and the *operator* expects 1 value, then *values* is ignored, and *otherFieldName* is used.
+* `otherFieldName` - *String, conditional* - Specifies that the *fieldName* should be compared against another field in the records, rather than the values in the *values* property.
+Only used for *operators* that expect 1 value (e.g., `EQUALS` or `LESS_THAN_OR_EQUALS` - not `IS_NOT_BLANK` or `IN`).
+
+[source,java]
+.QFilterCriteria definition examples:
+----
+// in-line, via constructors that take (List<Serializable> values) or (Serializable... values) as 3rd arg
+new QFilterCriteria("id", IN, 1, 2, 3)
+new QFilterCriteria("name", IS_BLANK)
+new QFilterCriteria("orderNo", IN, orderNoList)
+new QFilterCriteria("state", EQUALS, "MO")
+
+// long-form, with fluent setters
+new QFilterCriteria()
+   .withFieldName("quantity")
+   .withOpeartor(QCriteriaOperator.GREATER_THAN)
+   .withValues(List.of(47));
+
+// to use otherFieldName, long-form must be used
+new QFilterCriteria()
+   .withFieldName("firstName")
+   .withOpeartor(QCriteriaOperator.EQUALS)
+   .withOtherFieldName("lastName");
+
+// using otherFieldName to build a criterion that looks at two fields from two different join tables
+new QFilterCriteria()
+   .withFieldName("billToCustomer.lastName")
+   .withOpeartor(QCriteriaOperator.NOT_EQUALS)
+   .withOtherFieldName("shipToCustomer.lastName");
+
+----
+
+===== QFilterOrderBy
+* `fieldName` - *String, required* - Reference to a field on the table being queried.
+** Or, in the case of a query with *queryJoins*, a qualified name of a field from a join-table (where the qualifier would be the joined table's name or alias, followed by a dot)
+* `isAscending` - *boolean, default: true* - Specify if the sort is ascending or descending.
+
+
+[source,java]
+.QFilterOrderBy definition examples:
+----
+// short-form, via constructors
+new QFilterOrderBy("id") // isAscending defaults to true.
+new QFilterOrderBy("name", false)
+
+// long-form, with fluent setters
+new QFilterOrderBy()
+   .withFieldName("birthDate")
+   .withIsAscending(false);
+----
+
+==== QueryJoin
+* `joinTable` - *String, required* - Name of the table that is being joined in to the existing query.
+** Will be inferred from *joinMetaData*, if *joinTable* is not set when *joinMetaData* gets set.
+* `baseTableOrAlias` - *String, required* - Name of a table (or an alias) already defined in the query, to which the *joinTable* will be joined.
+** Will be inferred from *joinMetaData*, if *baseTableOrAlias* is not set when *joinMetaData* gets set (which will only use the leftTableName from the joinMetaData - never an alias).
+* `joinMetaData` - *QJoinMetaData object* - Optional specification of a {link-join} in the current QInstance.
+If not set, will be looked up at runtime based on *baseTableOrAlias* and *joinTable*.
+** If set before *baseTableOrAlias* and *joinTable*, then they will be set based on the *leftTable* and *rightTable* in this object.
+* `alias` - *String* - Optional (unless multiple instances of the same table are being joined together, when it becomes required).
+Behavior based on SQL `FROM` clause aliases.
+If given, must be used as the part before the dot in field name specifications throughout the rest of the query input.
+* `select` - *boolean, default: false* - Specify whether fields from the *rightTable* should be selected by the query.
+If *true*, then the `QRecord` objects returned by this query will have values with corresponding to the (table-or-alias `.` field-name) form.
+* `type` - *Enum of INNER, LEFT, RIGHT, FULL, default: INNER* - specifies the SQL-style type of join being performed.
+
+[source,java]
+.QueryJoin definition examples:
+----
+// selecting from an "orderLine" table - then join to its corresponding "order" table
+queryInput.withTableName("orderLine");
+queryInput.withQueryJoin(new QueryJoin("order").withSelect(true));
+...
+queryOutput.getRecords().get(0).getValueBigDecimal("order.grandTotal");
+
+// given an "order" table with 2 foreign keys to a customer table (billToCustomerId and shipToCustomerId)
+// Note, we must supply the JoinMetaData to the QueryJoin, to drive what fields to join on in each case.
+queryInput.withTableName("order");
+queryInput.withQueryJoins(List.of(
+   new QueryJoin(instance.getJoin("orderJoinShipToCustomer")
+       .withAlias("shipToCustomer")
+       .withSelect(true)),
+   new QueryJoin(instance.getJoin("orderJoinBillToCustomer")
+       .withAlias("billToCustomer")
+       .withSelect(true))));
+...
+record.getValueString("billToCustomer.firstName")
+   + " placed an order for "
+   + record.getValueString("shipToCustomer.firstName")
+
+----
+
+=== QueryOutput
+* `records` - *List of QRecord* - List of 0 or more records that match the query filter.
+** _Note:  If a *recordPipe* was supplied to the QueryInput, then calling `queryOutput.getRecords()` will result in an `IllegalStateException` being thrown - as the records were placed into the pipe as they were fetched, and cannot all be accessed as a single list._

docs/actions/RenderTemplateAction.adoc

@@ -0,0 +1,33 @@
+== RenderTemplateAction
+include::../variables.adoc[]
+
+The `*RenderTemplateAction*` performs the job of taking a template - that is, a string of code, in a templating language, such as https://velocity.apache.org/engine/1.7/user-guide.html[Velocity], and merging it with a set of data (known as a context), to produce some using-facing output, such as a String of HTML.
+
+=== Examples
+==== Canonical Form
+[source,java]
+----
+RenderTemplateInput input = new RenderTemplateInput(qInstance);
+input.setSession(session);
+input.setCode("Hello, ${name}");
+input.setTemplateType(TemplateType.VELOCITY);
+input.setContext(Map.of("name", "Darin"));
+RenderTemplateOutput output = new RenderTemplateAction.execute(input);
+String result = output.getResult();
+assertEquals("Hello, Darin", result);
+----
+
+==== Convenient Form
+[source,java]
+----
+String result = RenderTemplateAction.renderVelocity(input, Map.of("name", "Darin"), "Hello, ${name}");
+assertEquals("Hello, Darin", result);
+----
+
+=== RenderTemplateInput
+* `code` - *String, Required* - String of template code to be rendered, in the templating language specified by the `type` parameter.
+* `type` - *Enum of VELOCITY, Required* - Specifies the language of the template code.
+* `context` - *Map of String → Object* - Data to be made available to the template during rendering.
+
+=== RenderTemplateOutput
+* `result` - *String* - Result of rendering the input template and context.

docs/docinfo.html

@@ -0,0 +1,6 @@
+<style>
+   .sect2 + .sect2
+   {
+      border-top: 1px solid #e7e7e9;
+   }
+</style>

docs/implementations/TableSync.adoc

@@ -0,0 +1,406 @@
+== TableSyncProcess
+include::../variables.adoc[]
+
+The `TableSyncProcess` is designed to help solve the common use-case where you have a set of records in one table,
+you want to apply a transformation to them, and then you want them to result in records in a different table.
+
+=== Sample Scenario
+For example, you may be receiving an import file feed from a partner - say, a CSV file of order records
+- that you need to synchronize into your local database's orders table.
+
+=== High Level Steps
+At a high-level, the steps of this task are:
+
+1. Get the records from the partner's feed.
+2. Decide Insert/Update - e.g., if you already have any of these orders in your database table, in which case, they need updated, versus ones
+you don't have, which need inserted.
+3. Map fields from the partner's order record to your own fields.
+4. Store the necessary records in your database (insert or update).
+
+=== What you need to tell QQQ
+`TableSyncProcess`, as defined by QQQ, knows how to do everything for a job like this, other than the part that's unique to your business.
+Those unique parts that you need to tell QQQ are:
+
+* What are the source & destination tables?
+* What are the criteria to identify source records to be processed?
+* What are the key fields are that "link" together records from the source & destination tables?
+** For the example above, imagine you have a "partnerOrderNo" field in your database's order table - then you'd need
+to tell QQQ what field in the partner's table provides the value for that field in your table.
+* How do fields / values from the source table map to fields & values in the destination table?
+
+=== Detailed Steps
+To get more specific, in a QQQ `TableSyncProcess`, here's what happens for each of the high-level steps given above:
+
+* Getting records from the partner's feed (done by QQQ):
+** Records from the source will be fetched via an Extract step, which runs a query to find the records that need processing.
+** Depending on your use-case, you may use an `ExtractViaQueryStep` (maybe with a Table Automation) or `ExtractViaBasepullQueryStep`
+(e.g., if you are polling a remote data source).
+* Deciding Insert/Update (done by QQQ):
+** Given  a set of records from the source table (e.g., output of the Extract step mentioned above), get values from the "key" field in that table.
+** Do a lookup in the destination table, where its corresponding "key" field has the values extracted from the source records.
+** For each source record, if its "key" was found in the destination table, then plan an update to that existing corresponding
+destination record; else, plan to insert a new record in the destination table.
+* Mapping values (done by your custom Application code):
+** Specifically, mapping is done in a subclass of `AbstractTableSyncTransformStep`, in the `populateRecordToStore` method.
+Of particular interest are these two parameters to that method:
+*** `QRecord destinationRecord` - as determined by the insert/update logic above, this will either be a
+new empty record (e.g., for inserting), or a fully populated record from the destination table (for updating).
+*** `QRecord sourceRecord` - this is the record being processed, from the source table.
+** This method is responsible for setting values in the `destinationRecord`, and returning that record
+(unless it has decided that for some reason the record should _not_ be stored, in which case it may return `null`).
+* Storing the records (done by QQQ):
+** This is typically done with the `LoadViaInsertOrUpdateStep`, though it is customizable if additional work is needed (e.g., via a
+subclass of `LoadViaInsertOrUpdateStep`, or a more custom subclass of `AbstractLoadStep`).
+
+=== Bare-bones Example
+For this example, let's assume we're setting up a partner-order-feed as described above, with the following details:
+
+* We have records from a partner in a table named `"partnerOrderImport"` (let's assume the records may have been created
+using the QQQ `FilesystemImporter` process).
+These records have the following fields:
+** `orderNo, date, city, state, postal, whseNo`
+* We need to synchronize those records with table in our database named `"order"`, with the following fields corresponding to those from the partner:
+** `partnerOrderNo, orderDate, shipToCity, shipToState, shipToZipCode, warehouseId`
+* The same conceptual order may appear in the `"partnerOrderImport"` multiple times, e.g., if they update some data on the order and re-transmit it to us.
+Meaning, we need to update our `"order"` records when we receive a new version an existing order.
+
+To use `*TableSyncProcess*` for solving this use-case, we'll need to create 2 things:
+
+1. A `QProcessMetaData` object, which we can create using the builder object provided by class `TableSyncProcess`.
+Note that this type of process is specialization of the standard QQQ `StreamedETLWithFrontendProcess`, as described elsewhere in this documentation.
+2. A subclass of `AbstractTableSyncTransformStep`, where we implement our mapping logic.
+Again, note that `AbstractTableSyncTransformStep` is a subclass of `AbstractTransformStep`, as used by `StreamedETLWithFrontendProcess`.
+
+And to be good programmers, we'll actually create a 3rd thing:
+
+[start=3]
+. A unit test for our Transform step.
+
+Here are examples of these pieces of code:
+
+[source,java]
+.Example of building process using the TableSyncProcess builder:
+----
+// the false argument below tells the build we are not a basepull-style process
+QProcessMetaData processMetaData = TableSyncProcess.processMetaDataBuilder(false)
+
+    // give our process a unique name within our QInstance
+    .withName("partnerOrderToLocalOrderProcess")
+
+    // tell the process to what class to use for transforming records from source to destination
+    .withSyncTransformStepClass(PartnerOrderToOrderTransformStep.class)
+
+    .getProcessMetaData();
+----
+
+[source,java]
+.Example implementation of an AbstractTableSyncTransformStep
+----
+public class PartnerOrderToOrderTransformStep extends AbstractTableSyncTransformStep
+{
+   @Override
+   protected SyncProcessConfig getSyncProcessConfig()
+   {
+      return (new SyncProcessConfig(
+         "partnerOrderImport", // source tableName
+         "orderNo",            // source table key fieldName
+         "order",              // destination tableName
+         "partnerOrderNo"      // destination table foreign key fieldName
+      ));
+   }
+
+   @Override
+   public QRecord populateRecordToStore(RunBackendStepInput runBackendStepInput, QRecord destinationRecord, QRecord sourceRecord) throws QException
+   {
+      // map simple values from source table to destination table
+      destinationRecord.setValue("orderDate", sourceRecord.get("date"));
+      destinationRecord.setValue("shipToAddressCity", sourceRecord.get("city"));
+      destinationRecord.setValue("shipToAddressState", sourceRecord.get("state"));
+      destinationRecord.setValue("shipToAddressZipCode", sourceRecord.get("postal"));
+      return (destinationRecord);
+   }
+
+}
+----
+
+[source,java]
+.Example Unit Test for a transform step
+----
+   @Test
+   void testTransformStep()
+   {
+      // insert 1 test order, that will be updated by the transform step
+      Integer existingId = new InsertAction().execute(new InsertInput("order").withRecords(List.of(
+         new QRecord().withValue("partnerOrderNo", 101).withValue("shipToState", "IL")
+      ))).getRecords().get(0).getValueInteger("id");
+
+      // set up input for the step - a list of 2 of the partner's orders
+      RunBackendStepInput input = new RunBackendStepInput();
+      input.setRecords(List.of(
+         new QRecord().withValue("orderNo", 101).withValue("state", "NY"), // will update the order above
+         new QRecord().withValue("orderNo", 102).withValue("state", "CA")  // will insert a new order
+      ));
+      RunBackendStepOutput output = new RunBackendStepOutput();
+
+      // run the code under test - our transform step
+      new PartnerOrderToOrderTransformStep().run(input, output);
+
+      // Note that by just running the transform step, no records have been stored.
+      // We can assert against the output of this step.
+
+      assertEquals(existingId, output.getRecords().get(0).getValue("id"));
+      assertEquals(101, output.getRecords().get(0).getValue("partnerOrderNo"));
+      assertEquals("NY", output.getRecords().get(0).getValue("shipToState"));
+
+      assertNull(output.getRecords().get(1).getValue("id"));
+      assertEquals(102, output.getRecords().get(1).getValue("partnerOrderNo"));
+      assertEquals("CA", output.getRecords().get(1).getValue("shipToState"));
+   }
+
+   @Test
+   void testFullProcess()
+   {
+      // todo! :)
+   }
+----
+
+=== Pseudocode process flow
+Now that we've seen the bare-bones example, let's see an even more detailed breakdown of how a full `TableSyncProcess` works,
+by looking at its 3 "ETL" steps in pseudocode:
+
+==== ExtractStep (Producer Thread)
+
+* Queries source table for records
+** Often based on Table Automations (e.g., for all newly inserted records) or Basepull pattern (polling for new/updated records).
+
+==== TransformStep (Consumer Thread)
+
+* Receives pages of records from `ExtractStep` in the `run` method.
+* Makes `sourceKeyList` by getting `sourceTableKeyField` values from the records.
+* Calls `initializeRecordLookupHelper(runBackendStepInput, sourceRecordList)`
+** Calls `getLookupsToPreLoad` to control which lookups are performed.
+* Calls `getExistingRecordsByForeignKey(runBackendStepInput, destinationTableForeignKeyField, destinationTableName, sourceKeyList);`
+** Calls `getExistingRecordQueryFilter(runBackendStepInput, sourceKeyList)` as part of querying the `destinationTable`
+** Returns the output of `buildExistingRecordsMap(destinationTableForeignKeyField, queryOutput.getRecords())`
+* foreach input record (from `sourceTable`):
+** Calls `getExistingRecord(existingRecordsByForeignKey, destinationForeignKeyField, sourceKeyValue)`
+** if an existing record was returned (and if the syncConfig says `performUpdates`), this record is set as `recordToStore`
+** else if no existing record was returned (and if the syncConfig says `performInserts`), a new record is set as `recordToStore`
+** else continue the foreach.
+** call `populateRecordToStore(runBackendStepInput, recordToStore, sourceRecord)`
+** if a record is returned it is added to the process step output (to be stored in the LoadStep)
+
+==== LoadStep (Consumer Thread)
+
+* Receives records from the output of the `TransformStep`.
+* Inserts and/or Updates `destinationTable`, with records returned by `populateRecordToStore`
+
+=== Additional Process Configuration Examples
+The following examples show how to use additional settings in the `TableSyncProcess` builder.
+
+==== UI
+While a `TableSyncProcess` will often run via a schedule and/or automation, we may also want to allow users
+to manually run it in a UI.
+
+[source,java]
+.Making our process available for a UI
+----
+QProcessMetaData processMetaData = TableSyncProcess.processMetaDataBuilder(true)
+    .withName("partnerOrderToLocalOrderProcess")
+    .withSyncTransformStepClass(PartnerOrderToOrderTransformStep.class)
+
+    // attach our process to its source table, to show up in UI
+    .withTableName("partnerOrderImport")
+
+    // add some fields to display on the review screen, in UI
+    .withReviewStepRecordFields(List.of(
+       new QFieldMetaData("clientId", QFieldType.STRING).withLabel("Client"),
+       new QFieldMetaData("warehouseId", QFieldType.STRING).withLabel("Warehouse"),
+       new QFieldMetaData("partnerOrderNo", QFieldType.STRING)))
+    .getProcessMetaData();
+----
+
+==== Basepull
+The previous example would work as a Table Automation (e.g., where the list of records identified in the
+Extract step were determined by the Automation system).
+However, a second common pattern is to use `Basepull` (e.g., if polling for updated records from a partner API endpoint).
+
+[source,java]
+.Configuring our process as a Basepull
+----
+// the true argument below tells the build we ARE a basepull-style process
+// this changes the default extract-step.
+QProcessMetaData processMetaData = TableSyncProcess.processMetaDataBuilder(false)
+    .withName("partnerOrderToLocalOrderProcess")
+    .withSyncTransformStepClass(PartnerOrderToOrderTransformStep.class)
+
+    // See Basepull documentation for details
+    .withBasepullConfiguration(new BasepullConfiguration())
+
+    // schedule our process to run automatically every minute
+    .withSchedule(new QScheduleMetaData().withRepeatSeconds(60))
+
+    .getProcessMetaData();
+----
+
+=== Additional Options in the Transform Step
+
+==== Specifying to not perform Inserts or not perform Updates
+We may have a scenario where we want our sync process to never update records if the key is already found in the destination table.
+We can configure this with an additional optional parameter to the `SyncProcessConfig` constructor:
+
+[source,java]
+.Specifying to not do updates in a TableSyncProcess
+----
+   @Override
+   protected SyncProcessConfig getSyncProcessConfig()
+   {
+      return (new SyncProcessConfig("partnerOrderImport", "orderNo", "order", "partnerOrderNo",
+         true,  // performInserts
+         false  // performUpdates
+      ));
+   }
+----
+
+Similarly, we may want to disallow inserts from a particular sync process.
+The `performInserts` argument to the `SyncProcessConfig` constructor lets us do that:
+
+[source,java]
+.Specifying to not do inserts in a TableSyncProcess
+----
+   @Override
+   protected SyncProcessConfig getSyncProcessConfig()
+   {
+      return (new SyncProcessConfig("partnerOrderImport", "orderNo", "order", "partnerOrderNo",
+         false, // performInserts
+         true   // performUpdates
+      ));
+   }
+----
+
+==== Customizing the query for existing records
+
+In some cases, a specific Table Sync process may need to refine the query filter that is used
+to lookup existing records in the destination table (e.g. for determining insert vs. update).
+
+For example, in our orders-from-a-partner scenario, if we have more than 1 partner sending us orders,
+where there could be overlapping orderNo values among them - we may have an additional field in our
+orders table to identify which partner an order came from.
+So then when we're looking up orders by `partnerOrderNo`, we would need to also include the `partnerId` field
+in our query, so that we only update orders from the specific partner that we're dealing with.
+
+To do this (to customize the existing record query filter), we need can just override the method `getExistingRecordQueryFilter`.
+Generally we would start by calling the `super` version of the method, and then add to it additional criteria.
+
+[source,java]
+.Customizing the query filter used to look for existing records
+----
+   /*******************************************************************************
+    ** Define the query filter to find existing records.  e.g., for determining
+    ** insert vs. update.  Subclasses may override this to customize the behavior,
+    ** e.g., in case an additional field is needed in the query.
+    *******************************************************************************/
+   protected QQueryFilter getExistingRecordQueryFilter(RunBackendStepInput runBackendStepInput, List<Serializable> sourceKeyList)
+   {
+      QQueryFilter filter = super.getExistingRecordQueryFilter(runBackendStepInput, sourceKeyList);
+      filter.addCriteria(new QFilterCriteria("partnerId", EQUALS, PARTNER_ID));
+      return (filter);
+   }
+----
+
+==== More efficient additional record lookups
+
+It is a common use-case to need to map various ids from a partner's system to ids in your own system.
+For the orders example, we might need to know what warehouse the order is shipping from.
+The customer may send their identifier for the warehouse, and we may need to map those identifiers to our own warehouse ids.
+
+The QQQ-provided class `RecordLookupHelper` exists to help with performing lookups like this,
+and in particular, it can be used to execute one query to fetch a full table, storing records
+by a key field, then returning those records without performing additional queries.
+
+`AbstractTableSyncTransformStep` has a protected `recordLookupHelper` member.
+If we override the method `getLookupsToPreLoad()`, then this object is
+populated by calling its `preloadRecords` method with each specified pair of tableNames and fieldNames.
+
+[source,java]
+.Specifying tables to pre-load using a RecordLookupHelper
+----
+   /*******************************************************************************
+    ** Specify a list of tableName/keyColumnName pairs to run through
+    ** the preloadRecords method of the recordLookupHelper.
+    *******************************************************************************/
+   @Override
+   protected List<Pair<String, String>> getLookupsToPreLoad()
+   {
+      return (List.of(
+         Pair.of("warehouse", "partnerWarehouseNo")
+      ));
+   }
+----
+
+If we have preloaded some lookups, we can then use them in our `populateRecordToStore` method as follows:
+[source,java]
+.Using the recordLookupHelper in populateRecordToStore
+----
+   // lookup warehouse with partnerWarehouseNo=whseNo from partner, and use our id in destination record
+   String partnerWarehouseNo = sourceRecord.getValue("whseNo");
+   Integer warehouseId = recordLookupHelper.getRecordId("warehouse", "partnerWarehouseNo", whseNo, Integer.class);
+   destinationRecord.setValue("warehouseId", warehouseId);
+----
+
+==== Additional override points
+
+There are more methods which can be overridden in your `AbstractTableSyncTransformStep` subclass,
+to provide further customizations of behaviors, specifically in the area of dealing with existing
+records (e.g., the insert/update use-case).
+[source,java]
+.Additional AbstractTableSyncTransformStep overrides
+----
+
+   /*******************************************************************************
+    ** Run the existingRecordQueryFilter - to look in the destinationTable for
+    ** any records that may need an update (rather than an insert).
+    **
+    ** Generally returns a Map, keyed by a Pair of the destinationTableForeignKeyField
+    ** and the value in that field.  But, for more complex use-cases, one can override
+    ** the buildExistingRecordsMap method, to make different keys (e.g., if there are
+    ** two possible destinationTableForeignKeyFields).
+    *******************************************************************************/
+   protected Map<Pair<String, Serializable>, QRecord> getExistingRecordsByForeignKey
+   (
+      RunBackendStepInput runBackendStepInput,
+      String destinationTableForeignKeyField,
+      String destinationTableName,
+      List<Serializable> sourceKeyList
+   ) throws QException;
+
+
+   /*******************************************************************************
+    ** Overridable point where you can, for example, keys in the existingRecordsMap
+    ** with different fieldNames from the destinationTable.
+    **
+    ** Note, if you're overriding this method, you'll likely also want & need to
+    ** override getExistingRecord.
+    *******************************************************************************/
+   protected Map<Pair<String, Serializable>, QRecord> buildExistingRecordsMap
+   (
+      String destinationTableForeignKeyField,
+      List<QRecord> existingRecordList
+   );
+
+   /*******************************************************************************
+    ** Given the map of existingRecordsByForeignKey (as built by
+    ** getExistingRecordsByForeignKey which calls buildExistingRecordsMap),
+    ** get one record from that map, for a given key-value from a source record.
+    **
+    ** The destinationForeignKeyField is given as advice if needed (e.g., to see its type)
+    *******************************************************************************/
+   protected QRecord getExistingRecord
+   (
+      Map<Pair<String, Serializable>, QRecord> existingRecordsByForeignKey,
+      QFieldMetaData destinationForeignKeyField,
+      Serializable sourceKeyValue
+   );
+
+----
+

docs/index.adoc

@@ -0,0 +1,76 @@
+= QQQ
+:doctype: article
+:toc: left
+:toclevels: 2
+:source-highlighter: coderay
+
+include::Introduction.adoc[leveloffset=+1]
+
+== Meta Data
+// Organizational units
+include::metaData/QInstance.adoc[leveloffset=+1]
+include::metaData/Backends.adoc[leveloffset=+1]
+include::metaData/Apps.adoc[leveloffset=+1]
+
+// Primary meta-data types
+include::metaData/Tables.adoc[leveloffset=+1]
+include::metaData/Processes.adoc[leveloffset=+1]
+include::metaData/Widgets.adoc[leveloffset=+1]
+
+// Helper meta-data types
+include::metaData/Fields.adoc[leveloffset=+1]
+include::metaData/PossibleValueSources.adoc[leveloffset=+1]
+include::metaData/Joins.adoc[leveloffset=+1]
+include::metaData/SecurtiyKeyTypes.adoc[leveloffset=+1]
+include::metaData/Reports.adoc[leveloffset=+1]
+include::metaData/Icons.adoc[leveloffset=+1]
+include::metaData/PermissionRules.adoc[leveloffset=+1]
+
+== Services
+
+include::misc/ScheduledJobs.adoc[leveloffset=+1]
+
+=== Web server (Javalin)
+#todo#
+
+=== API server (OpenAPI)
+#todo#
+
+== Custom Application Code
+include::misc/QContext.adoc[leveloffset=+1]
+include::misc/QRecords.adoc[leveloffset=+1]
+include::misc/QRecordEntities.adoc[leveloffset=+1]
+include::misc/ProcessBackendSteps.adoc[leveloffset=+1]
+include::misc/RenderingWidgets.adoc[leveloffset=+1]
+
+=== Table Customizers
+#todo#
+
+== QQQ Core Actions
+include::actions/QueryAction.adoc[leveloffset=+1]
+
+include::actions/GetAction.adoc[leveloffset=+1]
+
+=== CountAction
+#todo#
+
+=== AggregateAction
+#todo#
+
+include::actions/InsertAction.adoc[leveloffset=+1]
+
+=== UpdateAction
+#todo#
+
+=== DeleteAction
+#todo#
+
+=== AuditAction
+#todo#
+
+== QQQ Default Implementations
+include::implementations/TableSync.adoc[leveloffset=+1]
+// later... include::actions/RenderTemplateAction.adoc[leveloffset=+1]
+
+== QQQ Utility Classes
+include::utilities/RecordLookupHelper.adoc[leveloffset=+1]

docs/justfile

@@ -0,0 +1,12 @@
+## https://github.com/casey/just
+
+default:
+   just --list
+
+build-index-html:
+   asciidoctor -a docinfo=shared index.adoc
+   line=$(grep 'Last updated' index.html) && sed -i "s/id=\"content\">/&$line/" index.html
+
+build-and-publish-index-html: build-index-html
+   scp index.html first-node:/mnt/first-volume/dkelkhoff/nginx/html/justinsgotskinnylegs.com/qqq-docs.html
+   @echo "Updated: https://justinsgotskinnylegs.com/qqq-docs.html"

docs/metaData/Apps.adoc

@@ -0,0 +1,92 @@
+[#Apps]
+== Apps
+include::../variables.adoc[]
+
+QQQ User Interfaces (e.g., Material Dashboard) generally organize their contents via *Apps*.
+Apps are a lightweight construct in QQQ - basically just containers for other objects.
+
+Specifically, Apps can contain:
+
+* {link-widgets}
+* {link-tables}
+* {link-process}
+* {link-reports}
+* Other {link-apps} - to create a multi-tiered navigational hierarchy.
+
+=== QAppMetaData
+Apps are defined in a QQQ Instance in `*QAppMetaData*` objects.  Apps must consist of either 1 or more {link-widgets}, or 1 or more *Sections*, which are expected to contain 1 or more {link-tables}, {link-processes}, or {link-reports}.
+
+*QAppMetaData Properties:*
+
+* `name` - *String, Required* - Unique name for the app within the QQQ Instance.
+* `label` - *String* - User-facing label for the app, presented in User Interfaces.
+Inferred from `name` if not set.
+* `permissionRules` - *QPermissionRules* - Permissions to apply to the app.  See {link-permissionRules} for details.
+* `children` - *List of QAppChildMetaData* - Objects contained within the app. These can be {link-tables}, {link-processes}, {link-reports} or other {link-apps}.
+** See the example below for some common patterns for how these child-meta data objects are added to an App.
+* `parentAppName` - *String* - For an app which is a child of another app, the parent app's name is referenced in this field.
+** Note that this is generally automatically set when the child is added to its parent, in the `addChild` method.
+* `icon` - *QIcon* - An icon to display in a UI for the app.  See {link-icons}.
+* `widgets` - *List of String* - A list of names of {link-widgets} to include in the app.
+* `sections` - *List of <<QAppSection>>* - A list of `QAppSection` objects, to create organizational subdivisions within the app.
+** As shown in the example below, the method `withSectionOfChildren` can be used to fluently add a new `QAppSection`, along with its child objects, to both an app and a section all at once.
+
+==== QAppSection
+A `QAppSection` is an organizational subsection of a {link-app}.
+
+* `name` - *String, Required* - Unique name for the section within its app.
+* `label` - *String* - User-facing label for the section, presented in User Interfaces.
+Inferred from `name` if not set.
+* `icon` - *QIcon* - An icon to display in a UI for the section.  See {link-icons}.
+* `tables` - *List of String* - A list of names of {link-tables} to include in the section.
+* `processes` - *List of String* - A list of names of {link-processes} to include in the section.
+* `reports` - *List of String* - A list of names of {link-reports} to include in the section.
+
+*Examples*
+[source,java]
+----
+/*******************************************************************************
+** Full example of constructing a QAppMetaData object.
+*******************************************************************************/
+public class ExampleAppMetaDataProducer extends MetaDataProducer<QAppMetaData>
+{
+
+   /*******************************************************************************
+    ** Produce the QAppMetaData
+    *******************************************************************************/
+   @Override
+   public QAppMetaData produce(QInstance qInstance) throws QException
+   {
+      return (new QAppMetaData()
+         .withName("sample")
+         .withLabel("My Sample App")
+         .withIcon(new QIcon().withName("thumb_up"))
+         .withWidgets(List.of(
+            UserWelcomeWidget.NAME,
+            SystemHealthLineChartWidget.NAME))
+         .withSectionOfChildren(new QAppSection().withName("peoplePlacesAndThings"),
+            qInstance.getTable(People.TABLE_NAME),
+            qInstance.getTable(Places.TABLE_NAME),
+            qInstance.getTable(Things.TABLE_NAME),
+            qInstance.getProcess(AssociatePeopleWithPlacesProcess.NAME))
+         .withSectionOfChildren(new QAppSection().withName("math").withLabel("Mathematics"),
+            qInstance.getProcess(ComputePiProcess.NAME),
+            qInstance.getReport(PrimeNumbersReport.NAME),
+            qInstance.getReport(PolygonReport.NAME)));
+   }
+
+
+   /*******************************************************************************
+    ** Since this meta-data producer expects to find other meta-data objects in the
+    ** QInstance, give it a sortOrder higher than the default (which we'll expect
+    ** the other objects used).
+    *******************************************************************************/
+   @Override
+   public int getSortOrder()
+   {
+      return (Integer.MAX_VALUE);
+   }
+
+}
+----
+

docs/metaData/Backends.adoc

@@ -0,0 +1,150 @@
+[#Backends]
+== Backends
+include::../variables.adoc[]
+
+A key component of QQQ is its ability to connect to various backend data stores, while providing the same interfaces to those backends - both User Interfaces, and Programming Interfaces.
+
+For example, out-of-the-box, QQQ can connect to:
+
+* <<RDBMSBackendMetaData,RDBMS>> (Relational Database Management Systems, such as MySQL)
+* File Systems (<<S3BackendMetaData,Amazon S3>> or <<FilesystemBackendMetaData,local disk>>)
+* <<APIBackendMetaData,JSON Web APIs>> (_using a custom mapping class per-API backend_).
+* In-Memory data stores
+
+All {link-tables} in a QQQ instance must belong to a backend. As such, any instance using tables (which would be almost all instances) must define 1 or more backends.
+
+=== QBackendMetaData
+Backends are defined in a QQQ Instance in a `*QBackendMetaData*` object.
+These objects will have some common attributes, but many different attributes based on the type of backend being used.
+
+*QBackendMetaData Properties:*
+
+* `name` - *String, Required* - Unique name for the backend within the QQQ Instance.
+* `backendType` - *String, Required* - Identifier for the backend type being defined.
+** This attribute is typically set in the constructor of a `QBackendMetaData` subclass, and as such does not need to be set when defining a backend meta data object.
+* `enabledCapabilities` and `disabledCapability` - *Sets*, containing *Capability* enum values.
+Basic rules that apply to all tables in the backend, describing what actions (such as Delete, or Count) are supported in the backend.
+** By default, QQQ assumes that a backend supports _most_ capabilities, with one exception being `QUERY_STATS`.
+** #TODO# fully explain rules here
+* `usesVariants` - *Boolean, default false* - Control whether or not the backend supports the concept of "Variants".
+** Supporting variants means that tables within the backend can exist in alternative "variants" of the backend.
+For example, this might mean a sharded or multi-tenant database backend (perhaps a different server or database name per-client).
+Or this might mean using more than one set of credentials for connecting to an API backend - each of those credential sets would be a "variant".
+** A backend that uses variants requires additional properties to be set.  #TODO complete variant documentation#
+
+In a QQQ application, one will typically not create an instance of `QBackendMetaData` directly, but instead will create an instance of one of its subclasses, specific to the type of backend being used.
+The currently available list of such classes are:
+
+==== RDBMSBackendMetaData
+The meta data required for working with tables in an RDBMS (relational database) backend are defined in an instance of the `*RDBMSBackendMetaData*` class.
+
+*RDBMSBackendMetaData Properties:*
+
+* `vendor` - *String, Required* - Database vendor.  Currently supported values are: `aurora`, `mysql`, `h2`.
+* `jdbcUrl` - *String, Optional* - Full JDBC URL for connecting to the database.
+** If this property is provided, then following properties (which are the components of a JDBC URL) are ignored.
+In other words, you can either provide the `jdbcUrl`, or the individual components that make it up.
+* `hostName` - *String, Conditionally Required* - Hostname or ip address of the RDBMS server.
+* `port` - *Integer, Conditionally Required* - Port used to connect to the RDBMS server.
+* `databaseName` - *String, Conditionally Required* - Name of the database being connected to, within the RDBMS server.
+* `username` - *String, Conditionally Required* - Username for authenticating in the database server.
+* `password` - *String, Conditionally Required* - Password for authenticating in the database server.
+
+*Examples*
+[source,java]
+----
+/*******************************************************************************
+** Full example of constructing an RDBMSBackendMetaData
+*******************************************************************************/
+public class ExampleDatabaseBackendMetaDataProducer extends MetaDataProducer<QBackendMetaData>
+{
+   public static final String NAME = "rdbmsBackend";
+
+   /*******************************************************************************
+    ** Produce the QBackendMetaData
+    *******************************************************************************/
+   @Override
+   public QBackendMetaData produce(QInstance qInstance)
+   {
+      ///////////////////////////////////////////////////////////////////////
+      // read settings from either a .env file or the system's environment //
+      ///////////////////////////////////////////////////////////////////////
+      QMetaDataVariableInterpreter interpreter = new QMetaDataVariableInterpreter();
+      String vendor       = interpreter.interpret("${env.RDBMS_VENDOR}");
+      String hostname     = interpreter.interpret("${env.RDBMS_HOSTNAME}");
+      String port         = interpreter.interpret("${env.RDBMS_PORT}");
+      String databaseName = interpreter.interpret("${env.RDBMS_DATABASE_NAME}");
+      String username     = interpreter.interpret("${env.RDBMS_USERNAME}");
+      String password     = interpreter.interpret("${env.RDBMS_PASSWORD}");
+
+      return (new RDBMSBackendMetaData()
+         .withName(NAME)
+         .withVendor(vendor)
+         .withHostName(hostname)
+         .withPort(ValueUtils.getValueAsInteger(port))
+         .withDatabaseName(databaseName)
+         .withUsername(username)
+         .withPassword(password)
+         .withCapability(Capability.QUERY_STATS));
+   }
+}
+----
+
+==== S3BackendMetaData
+The meta data required for working with tables in an Amazon S3 backend are defined in an instance of the `*S3BackendMetaData*` class.
+
+*S3BackendMetaData Properties:*
+
+* `bucketName` - *String, Required* - Bucket name to connect to inside AWS S3.
+* `accessKey` - *String, Required* - Access key for connecting to S3 inside AWS S3.
+* `secretKey` - *String, Required* - Secret key for connecting to S3 inside AWS S3.
+* `region` - *String, Required* - AWS region containing the Bucket in S3.
+* `basePath` - *String, Required* - Base path to the files within the S3 bucket.
+
+==== FilesystemBackendMetaData
+The meta data required for working with tables in a (local) filesystem backend are defined in an instance of the `*FilesystemBackendMetaData*` class.
+
+*FilesystemBackendMetaData Properties:*
+
+* `basePath` - *String, Required* - Base path to the backend's files.
+
+==== APIBackendMetaData
+The meta data required for working with tables in a web API are defined in an instance of the `*APIBackendMetaData*` class.
+
+QQQ provides a minimal, reasonable default implementation for working with web APIs, making assumptions such as using `POST` to insert records, and `GET` with a primary key in the URL to get a single record.
+However, in our experience, almost all APIs are implemented differently enough, that a layer of custom code is required.
+For example, query/search endpoints are almost always unique in how they take their search parameters, and how they wrap their output.
+
+To deal with this, a QQQ API Backend can define a custom class in the `actionUtil` property of the backend meta-data, as a subclass of `BaseAPIActionUtil`, which ideally can override methods at the level where unique functionality is needed.
+For example, an application need not define the full method for executing a Query against an API backend (which would need to make the HTTP request (actually multiple, to deal with pagination)).
+Instead, one can just override the `buildQueryStringForGet` method, where the unique details of making the request are defined, and maybe the `jsonObjectToRecord` method, where records are mapped from the API's response to a QRecord.
+
+#todo - full reference and examples for `BaseAPIActionUtil`#
+
+*APIBackendMetaData Properties:*
+
+* `baseUrl` - *String, Required* - Base URL for connecting to the API.
+* `contentType` - *String, Required* - value of `Content-type` header included in all requests to the API.
+* `actionUtil` - *QCodeReference, Required* - Reference to a class that extends `BaseAPIActionUtil`, where custom code for working with this API backend is defined.
+* `customValues` - *Map of String → Serializable* - Application-defined additional name=value pairs that can
+* `authorizationType` - *Enum, Required* - Specifies how authentication is provided for the API.
+The value here, dictates which other authentication-related properties are required.
+Possible values are:
+** `API_KEY_HEADER` - Uses the `apiKey` property in an HTTP header named `API-Key`.
+_In the future, when needed, QQQ will add a property, tentatively named `apiKeyHeaderName`, to allow customization of this header name._
+** `API_TOKEN` - Uses the `apiKey` property in an `Authroization` header, as: `"Token " + apiKey`
+** `BASIC_AUTH_API_KEY` - Uses the `apiKey` property, Base64-encoded, in an `Authroization` header, as `"Basic " + base64(apiKey)`
+** `BASIC_AUTH_USERNAME_PASSWORD` - Combines the `username` and `password` properties, Base64-encoded, in an `Authroization` header, as `"Basic " + base64(username + ":" + password)`
+** `OAUTH2` - Implements an OAuth2 client credentials token grant flow, using the properties `clientId` and `clientSecret`.
+By default, the id & secret are sent as query-string parameters to the API's `oauth/token` endpoint.
+Alternatively, if the meta-data has a `customValue` of `setCredentialsInHeader=true`, then the id & secret are posted in an `Authorization` header (base-64 encoded, and concatenated with `":"`).
+** `API_KEY_QUERY_PARAM` - Uses the `apiKey` property as a query-string parameter, with its name taken from the `apiKeyQueryParamName` property.
+** `CUSTOM` - Has a no-op implementation at the QQQ layer.
+Assumes that an override of `protected void handleCustomAuthorization(HttpRequestBase request)` be implemented in the backend's `actionUtil` class.
+This would be
+* `apiKey` - *String, Conditional* - See `authorizationType` above for details.
+* `clientId` - *String, Conditional* - See `authorizationType` above for details.
+* `clientSecret` - *String, Conditional* - See `authorizationType` above for details.
+* `username` - *String, Conditional* - See `authorizationType` above for details.
+* `password` - *String, Conditional* - See `authorizationType` above for details.
+* `apiKeyQueryParamName` - *String, Conditional* - See `authorizationType` above for details.

docs/metaData/Fields.adoc

@@ -0,0 +1,119 @@
+[#Fields]
+== Fields
+include::../variables.adoc[]
+
+QQQ Fields define
+
+=== QFieldMetaData
+*QFieldMetaData Properties:*
+
+* `name` - *String, Required* - Unique name for the field within its container (table, process, etc).
+* `label` - *String* - User-facing label for the field, presented in User Interfaces.
+* `type` - *enum of QFieldType, Required* - Data type for values in the field.
+* `backendName` - *String* - Name of the field within its backend.
+** For example, in an RDBMS-backed table, a field's `name` may be written in camel case, but its `backendName` written with underscores.
+* `isRequired` - *boolean, default false* - Indicator that a value is required in this field.
+* `isEditable` - *boolean, default true* - Indicator that users may edit values in this field.
+* `displayFormat` - *String, default `%s`* - Java Format Specifier string, used to format values in the field for display in user interfaces.
+Used to set values in the `displayValues` map within a `QRecord`.
+** Recommended values for `displayFormat` come from the `DisplayFormat` interface, such as `DisplayFormat.CURRENCY`, `DisplayFormat.COMMAS`, or `DisplayFormat.DECIMAL2_COMMAS`.
+* `defaultValue` - Value to use for the field if no other value is given.  Type is based on the field's `type`.
+* `possibleValueSourceName` - *String* - Reference to a {link-pvs} to be used for this field.
+Values in this field should correspond to ids from the referenced Possible Value Source.
+* `maxLength` - *Integer* - Maximum length (number of characters) allowed for values in this field.
+Only applicable for fields with `type=STRING`.  Needs to be used with a `FieldBehavior` of type `ValueTooLongBehavior`.
+
+==== Field Behaviors
+Additional behaviors can be attached to fields through the use of the `behaviors` attribute,
+which is a `Set` of 0 or more instances of implementations of the `FieldBehavior` interface.
+Note that in some cases, these instances may be `enum` constants,
+but other times may be regular Objects.
+
+QQQ provides a set of common field behaviors.
+Applications can also define their own field behaviors by implementing the `FieldBehavior` interface,
+and attaching instances of their custom behavior classes to fields.
+
+===== ValueTooLongBehavior
+Used on String fields.  Requires the field to have a `maxLength` set.
+Depending on the chosen instance of this enum, before a record is Inserted or Updated,
+if the value in the field is longer than the `maxLength`, then one of the following actions can occur:
+
+* `TRUNCATE` - The value will be simply truncated to the `maxLength`.
+* `TRUNCATE_ELLIPSIS` - The value will be truncated to 3 characters less than the `maxLength`, and three periods (an ellipsis) will be placed at the end.
+* `ERROR` - An error will be reported, and the record will not be inserted or updated.
+* `PASS_THROUGH` - Nothing will happen.  This is the same as not having a `ValueTooLongBehavior` on the field.
+
+[source,java]
+.Examples of using ValueTooLongBehavior
+----
+   new QFieldMetaData("sku", QFieldType.STRING)
+      .withMaxLength(40),
+      .withBehavior(ValueTooLongBehavior.ERROR),
+
+   new QFieldMetaData("reason", QFieldType.STRING)
+      .withMaxLength(250),
+      .withBehavior(ValueTooLongBehavior.TRUNCATE_ELLIPSIS),
+
+----
+
+===== DynamicDefaultValueBehavior
+Used to set a dynamic default value to a field when it is being inserted or updated.
+For example, instead of having a hard-coded `defaultValue` specified in the field meta-data,
+and instead of having to add, for example, a pre-insert custom action.
+
+* `CREATE_DATE` - On inserts, sets the field's value to the current time.
+* `MODIFY_DATE` - On inserts and updates, sets the field's value to the current time.
+* `USER_ID` - On inserts and updates, sets the field's value to the current user's id (but only if the value is currently null).
+
+_Note that the `QInstanceEnricher` will, by default, add the `CREATE_DATE` and `MODIFY_DATE` `DynamicDefaultValueBehavior`
+options to any fields named `"createDate"` or `"modifyDate"`.
+This behavior can be disabled by setting the `configAddDynamicDefaultValuesToFieldsNamedCreateDateAndModifyDate` property
+on the `QInstanceEnricher` instance used by the application to `false`._
+
+[source,java]
+.Examples of using DynamicDefaultValueBehavior
+----
+   new QFieldMetaData("createDate", QFieldType.DATE_TIME)
+      .withBehavior(DynamicDefaultValueBehavior.CREATE_DATE),
+
+   new QFieldMetaData("modifyDate", QFieldType.DATE_TIME)
+      .withBehavior(DynamicDefaultValueBehavior.MODIFY_DATE),
+
+   new QFieldMetaData("createdByUserId", QFieldType.STRING)
+      .withBehavior(DynamicDefaultValueBehavior.USER_ID),
+----
+
+===== DateTimeDisplayValueBehavior
+By default, in QQQ, fields of type `DATE_TIME` are stored in UTC,
+and their values in a QRecord is a java `Instant` instance, which is always UTC.
+However, frontends will prefer to display date-time values in the user's local Time Zone whenever possible.
+
+Using `DateTimeDisplayValueBehavior` allows a `DATE_TIME` field to be displayed in a different Time Zone.
+An example use-case for this would be displaying airplane flight times,
+where you would want a flight from California to New York to display Pacific Time for its departure time,
+and Eastern Time for its arrival.
+
+An instance of `DateTimeDisplayValueBehavior` can be configured to either use a hard-coded time `ZoneId`
+(for example, to always show users UTC, or a business's home-office time zone).
+Or, it can be set up to get the time zone to use from another field in the table.
+
+[source,java]
+.Examples of using DateTimeDisplayValueBehavior
+----
+new QTableMetaData().withName("flights").withFields(List.of(
+   ...
+   new QFieldMetaData("departureTimeZoneId", QFieldType.STRING),
+   new QFieldMetaData("arrivaTimeZoneId", QFieldType.STRING),
+
+   new QFieldMetaData("departureTime", QFieldType.DATE_TIME)
+      .withBehavior(new DateTimeDisplayValueBehavior()
+         .withZoneIdFromFieldName("departureTimeZoneId")),
+
+   new QFieldMetaData("arrivalTime", QFieldType.DATE_TIME)
+      .withBehavior(new DateTimeDisplayValueBehavior()
+         .withZoneIdFromFieldName("arrivalTimeZoneId"))
+
+   new QFieldMetaData("ticketSaleStartDateTime", QFieldType.DATE_TIME)
+      .withBehavior(new DateTimeDisplayValueBehavior()
+         .withDefaultZoneId("UTC"))
+----

docs/metaData/Icons.adoc

@@ -0,0 +1,20 @@
+[#Icons]
+== Icons
+include::../variables.adoc[]
+
+#TODO#
+
+=== QIcon
+Icons are defined in a QQQ Instance in a `*QIcon*` object.
+
+#TODO#
+
+*QIcon Properties:*
+
+* `name` - *String* - Name of an icon from the https://mui.com/material-ui/material-icons/[Material UI Icon set]
+** Note that icon names from the link above need to be converted from _CamelCase_ to _underscore_case_...
+* `path` - *String* - Path to a file served under the application's web root, to be used as the icon.
+
+_Either `name` or `path` must be specified. If both are given, then name is used._
+
+

docs/metaData/Joins.adoc

@@ -0,0 +1,17 @@
+[#Joins]
+== Joins
+include::../variables.adoc[]
+
+#TODO#
+
+=== QJoinMetaData
+Joins are defined in a QQQ Instance in a `*QJoinMetaData*` object.
+
+#TODO#
+
+*QJoinMetaData Properties:*
+
+* `name` - *String, Required* - Unique name for the join within the QQQ Instance. #todo infererences or conventions?#
+
+#TODO#
+

docs/metaData/PermissionRules.adoc

@@ -0,0 +1,13 @@
+[#PermissionRules]
+== Permission Rules
+include::../variables.adoc[]
+
+#TODO#
+
+=== PermissionRule
+#TODO#
+
+*PermissionRule Properties:*
+
+#TODO#
+

docs/metaData/PossibleValueSources.adoc

@@ -0,0 +1,17 @@
+[#PossibleValueSources]
+== Possible Value Sources
+include::../variables.adoc[]
+
+#TODO#
+
+=== QPossibleValueSource
+A Possible Value Source is defined in a QQQ Instance in a `*QPossibleValueSource*` object.
+
+#TODO#
+
+*QPossibleValueSource Properties:*
+
+* `name` - *String, Required* - Unique name for the possible value source within the QQQ Instance.
+
+#TODO#
+

docs/metaData/Processes.adoc

@@ -0,0 +1,216 @@
+[#Processes]
+== Processes
+
+include::../variables.adoc[]
+
+Besides {link-tables}, the other most common type of object in a QQQ Instance is the Process.
+Processes are "custom" actions (e.g., defined by the application developers, rather than QQQ) that users and/or the system can execute against records.
+Processes generally are made up of two types of sub-objects:
+
+* *Screens* - i.e., User Interfaces (e.g., for gathering input and/or showing output).
+* *BackendSteps* - Java classes (of type `BackendStep`) that execute the logic of the process.
+
+=== QProcessMetaData
+
+Processes are defined in a QQQ Instance in a `*QProcessMetaData*` object.
+In addition to directly building a `QProcessMetaData` object setting its properties directly, there are a few common process patterns that provide *Builder* objects for ease-of-use.
+See StreamedETLWithFrontendProcess below for a common example
+
+*QProcessMetaData Properties:*
+
+* `name` - *String, Required* - Unique name for the process within the QQQ Instance.
+* `label` - *String* - User-facing label for the process, presented in User Interfaces.
+Inferred from `name` if not set.
+* `icon` - *QIcon* - Icon associated with this process in certain user interfaces.
+See {link-icons}.
+* `tableName` - *String* - Name of a {link-table} that the process is associated with in User Interfaces (e.g., Action menu).
+* `isHidden` - *Boolean, default false* - Option to hide the process from all User Interfaces.
+* `basepullConfiguration` - *<<BasepullConfiguration>>* - config for the common "Basepull" pattern, of identifying records with a timestamp greater than the last time the process was ran.
+See below for details.
+* `permissionRules` - *QPermissionRules object* - define the permission/access rules for the process.
+See {link-permissionRules} for details.
+* `steps` and `stepList` - *Map of String → <<QStepMetaData>>* and *List of QStepMetaData* - Defines the <<QFrontendStepMetaData,screens>> and <<QBackendStepMetaData,backend code>> that makes up the process.
+** `stepList` is the list of steps in the order that they will by default be executed.
+** `steps` is a map, including all steps from `stepList`, but which may also include steps which can used by the process if its backend steps make the decision to do so, at run-time.
+** A process's steps are normally defined in one of two was:
+*** 1) by a single call to `.withStepList(List<QStepMetaData>)`, which internally adds each step into the `steps` map.
+*** 2) by multiple calls to `.addStep(QStepMetaData)`, which adds a step to both the `stepList` and `steps` map.
+** If a process also needs optional steps, they should be added by a call to `.addOptionalStep(QStepMetaData)`, which only places them in the `steps` map.
+* `schedule` - *<<QScheduleMetaData>>* - set up the process to run automatically on the specified schedule.
+See below for details.
+* `minInputRecords` - *Integer* - #not used...#
+* `maxInputRecords` - *Integer* - #not used...#
+
+#todo: supplementalMetaData (API)#
+
+==== QStepMetaData
+
+This is the base class for the two types of steps in a process - <<QFrontendStepMetaData,screens>> and <<QBackendStepMetaData,backend code>>.
+There are some shared attributes of both of them, defined here.
+
+*QStepMetaData Properties:*
+
+* `name` - *String, Required* - Unique name for the step within the process.
+* `label` - *String* - User-facing label for the step, presented in User Interfaces.
+Inferred from `name` if not set.
+* `stepType` - *String* - _Deprecated._
+
+==== QFrontendStepMetaData
+
+For processes with a user-interface, they must define one or more "screens" in the form of `QFrontendStepMetaData` objects.
+
+*QFrontendStepMetaData Properties:*
+
+* `components` - *List of <<QFrontendComponentMetaData>>* - a list of components to be rendered on the screen.
+* `formFields` - *List of String* - list of field names used by the screen as form-inputs.
+* `viewFields` - *List of String* - list of field names used by the screen as visible outputs.
+* `recordListFields` - *List of String* - list of field names used by the screen in a record listing.
+
+==== QFrontendComponentMetaData
+
+A screen in a process may consist of multiple "components" - e.g., help text, and a form, and a list of records.
+Each of these components are defined in a `QFrontendComponentMetaData`.
+
+*QFrontendComponentMetaData Properties:*
+
+* `type` - *enum, Required* - The type of component to display.
+Each component type works with different keys in the `values` map.
+Possible values for `type` are:
+** `EDIT_FORM` - Displays a list of fields for editing, similar to a record edit screen.
+Requires that `formFields` be populated in the step.
+** `VIEW_FORM` - Displays a list of fields for viewing (not editing), similar to a record view screen.
+Requires that `viewFields` be populated in the step.
+** `HELP_TEXT` - Block of help text to be display on the screen.
+Requires an entry in the component's `values` map named `"text"`.
+** `HTML` - Block of custom HTML, generated by the process backend.
+Expects a process value named `html`.
+** `DOWNLOAD_FORM` - Presentation of a link to download a file generated by the process.
+Expects process values named `downloadFileName` and `serverFilePath`.
+** `GOOGLE_DRIVE_SELECT_FOLDER` - Special form that presents a UI from Google Drive, where the user can select a folder (e.g., as a target for uploading files in a subsequent backend step).
+** `BULK_EDIT_FORM` - For use by the standard QQQ Bulk Edit process.
+** `VALIDATION_REVIEW_SCREEN` - For use by the QQQ Streamed ETL With Frontend process family of processes.
+Displays a component prompting the user to run full validation or to skip it, or, if full validation has been ran, then showing the results of that validation.
+** `PROCESS_SUMMARY_RESULTS` - For use by the QQQ Streamed ETL With Frontend process family of processes.
+Displays the summary results of running the process.
+** `RECORD_LIST` - _Deprecated.
+Showed a grid with a list of records as populated by the process._
+* `values` - *Map of String → Serializable* - Key=value pairs, with different expectations based on the component's `type`.
+See above for details.
+
+==== QBackendStepMetaData
+
+Process Backend Steps are where custom (at this time, Java, but in the future, potentially, from any supported language) code is executed, to provide the logic of the process.
+QQQ comes with several common backend steps, such as for extracting lists of records, storing lists of records, etc.
+
+*QBackendStepMetaData Properties:*
+
+* `code` - *QCodeReference, Required* - Reference to the code to be executed for the step.
+The referenced code must implement the `BackendStep` interface.
+* `inputMetaData` - *QFunctionInputMetaData* - Definition of the data that the backend step expects and/or requires.
+Sub-properties are:
+** `fieldList` - *List of {link-fields}* - Optional list of fields used by the process step.
+In general, a process does not _have to_ specify the fields that its steps use.
+It can be used, however, for example, to cause a `defaultValue` to be applied to a field if it isn't specified in the process's input.
+It can also be used to cause the process to throw an error, if a field is marked as `isRequired`, but a value is not present.
+** `recordListMetaData` - *RecordListMetaData object* - _Not used at this time._
+
+==== BasepullConfiguration
+
+A "Basepull" process is a common pattern where an application needs to perform some action on all new (or updated) records from a particular data source.
+To implement this pattern, one needs to store the timestamp of when the action runs, then query the source for records where a date-time field is after that timestamp.
+
+QQQ helps facilitate this pattern by automatically retrieving and updating that timestamp field, and by building a default query filter based on that timestamp.
+
+This is done by adding a `BasepullConfiguration` object to a process's meta-data.
+
+*BasepullConfiguration Properties:*
+
+* `tableName` - *String, Required* - Name of a {link-table} in the QQQ Instance where the basepull timestamps are stored.
+* `keyField` - *String, Required* - Name of a {link-field} in the basepull table that stores a unique identifier for the process.
+* `keyValue` - *String* - Optional value to be stored in the `keyField` of the basepull table as the unique identifier for the process.
+If not set, then the process's `name` is used.
+* `lastRunTimeFieldName` - *String, Required* - Name of a {link-field} in the basepull table that stores the last-run time for the process.
+* `hoursBackForInitialTimestamp` - *Integer* - Optional number of hours to go back in time (from `now`) for the first time that the process is executed (i.e., if there is no timestamp stored in the basepull table).
+* `timestampField` - *String, Required* - Name of a {link-field} in the table being queried against the last-run timestamp.
+
+==== QScheduleMetaData
+
+QQQ can automatically run processes using an internal scheduler, if they are configured with a `QScheduleMetaData` object.
+
+*QScheduleMetaData Properties*
+
+* `repeatSeconds` - *Integer, Conditional* - How often the process should be executed, in seconds.
+* `repeatMillis` - *Integer, Conditional* - How often the process should be executed, in milliseconds.
+Mutually exclusive with `repeatSeconds`.
+* `initialDelaySeconds` - *Integer, Conditional* - How long between when the scheduler starts and the process should first run, in seconds.
+* `initialDelayMillis` - *Integer, Conditional* - How long between when the scheduler starts and the process should first run, in milliseconds.
+Mutually exclusive with `initialDelaySeconds`.
+* `variantRunStrategory` - *enum, Conditional* - For processes than run against {link-tables} that use a {link-backend} with Variants enabled, this property controls if the various instances of the process should run in `PARALLEL` or in `SERIAL`.
+* `variantBackend` - *enum, Conditional* - For processes than run against {link-tables} that use a {link-backend} with Variants enabled, this property specifies the name of the {link-backend}.
+
+==== StreamedETLWithFrontendProcess
+
+A common pattern for QQQ processes to exhibit is called the "Streamed ETL With Frontend" process pattern.
+This pattern is to do an "Extract, Transform, Load" job on a potentially large set of records.
+
+The records are Streamed through the process's steps, meaning, QQQ runs multiple threads - a producer, which is selecting records, and a consumer, which is processing records.
+As such, in general, an unlimited number of records can be processed by a process, without worrying about exhausting server resources (e.g., OutOfMemory).
+
+These processes also have a standard user-interface for displaying a summary of what the process will do (and has done), with a small number of records as a preview.
+The goal of the summary is to give the user the big-picture of what the process will do (e.g., X records will be inserted; Y records will be updated), along with a small view of some details on the records that will be stored (e.g., on record A field B will be set to C).
+
+This type of process uses 3 backend steps, and 2 frontend steps, as follows:
+
+* *preview* (backend) - does just a little work (limited # of rows), to give the user a preview of what the final result will be - e.g., some data to seed the review screen.
+* *review* (frontend) - a review screen, which after the preview step does not have a full process summary, but can generally tell the user how many records are input to the process, and can show a preview of a small number of the records.
+* *validate* (backend) - optionally (per input on review screen), does like the preview step, but does it for all records from the extract step.
+* *review* (frontend) - a second view of the review screen, if the validate step was executed.
+Now that the full validation was performed, a full process summary can be shown, along with a some preview records.
+* *execute* (backend) - processes all the rows - does all the work - stores data in the backend.
+* *result* (frontend) - a result screen, showing a "past-tense" version of the process summary.
+
+These backend steps are defined within QQQ, meaning they themselves do not execute any application-defined custom code.
+Instead, these steps use the following secondary <<QBackendStepMetaData,backend steps>>:
+
+* *Extract* - Fetch the rows to be processed.
+Used by preview (but only for a limited number of rows), validate (without limit), and execute (without limit).
+* *Transform* - Do whatever transformation is needed to the rows.
+Done on preview, validate, and execute.
+Always works with a page of records at a time.
+Since it is called on the preview & validate steps, it should *NOT* ever store any data (unless it does a specific check to confirm that it is being used on an *execute* step).
+* *Load* - Store the records into the backend, as appropriate.
+Only called by the execute step.
+Always works with a page of records at a time.
+
+The meta-data for a `StreamedETLWithFrontendProcess` uses several input fields on its steps.
+As such, it can be somewhat clumsy and error-prone to fully define a `StreamedETLWithFrontendProcess`.
+To improve this programmer-interface, an inner `Builder` class exists within `StreamedETLWithFrontendProcess` (generated by a call to `StreamedETLWithFrontendProcess.processMetaDataBuilder()`).
+
+*StreamedETLWithFrontendProcess.Builder methods:*
+
+* `withName(String name) - Set the name for the process.
+* `withLabel(String label) - Set the label for the process.
+* `withIcon(QIcon icon)` - Set an {link-icon} to be display with the process in the UI.
+* `withExtractStepClass(Class<? extends AbstractExtractStep>)` - Define the Extract step for the process.
+If no special extraction logic is needed, `ExtractViaQuery.class` is often a reasonable default.
+In other cases, `ExtractViaQuery` can be a reasonable class to extend for a custom extract step.
+* `withTransformStepClass(Class<? extends AbstractTransformStep>)` - Define the Transform step for the process.
+If no transformation logic is needed, `NoopTransformStep.class` can be used (though this is not very common).
+* `withLoadStepClass(Class<? extends AbstractLoadStep>)` - Define the Load step for the process.
+Several standard implementations exist, such as: `LoadViaInsertStep.class`, `LoadViaUpdateStep.class`, and `LoadViaDeleteStep.class`.
+* `withTableName(String tableName)` - Specify the name of the {link-table} that the process should be associated with in the UI.
+* `withSourceTable(String sourceTable)` - Specify the name of the {link-table} to be used as the source of records for the process.
+* `withDestinationTable(String destinationTable)` - Specify the name of the {link-table} to be used as the destination for records from the process.
+* `withSupportsFullValidation(Boolean supportsFullValidation)` - By default, all StreamedETLWithFrontendProcesses do allow the user to choose to run the full validation step.
+However, in case cases it may not make sense to do so - so this method can be used to turn off that option.
+* `withDoFullValidation(Boolean doFullValidation)` - By default, all StreamedETLWithFrontendProcesses will prompt the user if they want to run the full validation step or not.
+However, in case cases you may want to enforce that the validation step always be executed.
+Calling this method will remove the option from the user, and always run a full validation.
+* `withTransactionLevelAutoCommit()`, `withTransactionLevelPage()`, and `withTransactionLevelProcess()` - Change the transaction-level used by the process.
+By default, these processes are ran with a single transaction for all pages of their execute step.
+But for some cases, doing page-level transactions can reduce long-transactions and locking within the system.
+* `withPreviewMessage(String previewMessage)` - Sets the message shown on the validation review screen(s) above the preview records.
+* `withReviewStepRecordFields(List<QFieldMetaData> fieldList)` -
+* `withFields(List<QFieldMetaData> fieldList)` - Adds additional input fields to the preview step of the process.
+* `withBasepullConfiguration(BasepullConfiguration basepullConfiguration)` - Add a <<BasepullConfiguration>> to the process.
+* `withSchedule(QScheduleMetaData schedule)` - Add a <<QScheduleMetaData>> to the process.

docs/metaData/QInstance.adoc

@@ -0,0 +1,39 @@
+[#QInstance]
+== QInstance
+include::../variables.adoc[]
+
+An application in QQQ is defined as a set of Meta Data objects.  These objects are all stored together in an object called a `*QInstance*`.
+
+Currently, a `QInstance` must be programmatically constructed in java code - e.g., by constructing objects which get added to the QInstance, for example:
+
+[source,java]
+.Adding meta-data for two tables and one process to a QInstance
+----
+QInstance qInstance = new QInstance();
+qInstance.addTable(definePersonTable());
+qInstance.addTable(defineHomeTable());
+qInstance.addProcess(defineSendPersonHomeProcess());
+----
+
+It is on the QQQ roadmap to allow meta-data to be defined in a non-programmatic way, for example, in YAML or JSON files, or even from a dynamic data source (e.g. a document or relational database).
+
+The middleware and/or frontends being used in an application will drive how the `QInstance` is connected to the running server/process.
+For example, using the `qqq-middleware-javalin` module, a the `QJavalinImplementation` class () has a constructor which takes a `QInstance` as an argument:
+
+[source,java]
+.Starting a QQQ Javalin middleware server - passing a QInstance as a parameter to the new QJavalinImplementation
+----
+QJavalinImplementation qJavalinImplementation = new QJavalinImplementation(qInstance);
+Javalin service = Javalin.create();
+service.routes(qJavalinImplementation.getRoutes());
+service.start();
+----
+
+*QBackendMetaData Setup Methods:*
+
+These are the methods that one is most likely to use when setting up (defining) a `QInstance` object:
+
+* asdf
+
+*QBackendMetaData Usage Methods:*
+

docs/metaData/Reports.adoc

@@ -0,0 +1,174 @@
+[#Reports]
+== Reports
+include::../variables.adoc[]
+
+QQQ can generate reports based on {link-tables} defined within a QQQ Instance.
+Users can run reports, providing input values.
+Alternatively, application code can run reports as needed, supplying input values.
+
+=== QReportMetaData
+Reports are defined in a QQQ Instance with a `*QReportMetaData*` object.
+Reports are defined in terms of their sources of data (`QReportDataSource`), and their view(s) of that data (`QReportView`).
+
+*QReportMetaData Properties:*
+
+* `name` - *String, Required* - Unique name for the report within the QQQ Instance.
+* `label` - *String* - User-facing label for the report, presented in User Interfaces.
+Inferred from `name` if not set.
+* `processName` - *String* - Name of a {link-process} used to run the report in a User Interface.
+* `inputFields` - *List of {link-field}* - Optional list of fields used as input to the report.
+** The values in these fields can be used via the syntax `${input.NAME}`, where `NAME` is the `name` attribute of the `inputField`.
+** For example:
+
+[source,java]
+----
+// given this inputField:
+new QFieldMetaData("storeId", QFieldType.INTEGER)
+
+// its run-time value can be accessed, e.g., in a query filter under a data source:
+new QFilterCriteria("storeId", QCriteriaOperator.EQUALS, List.of("${input.storeId}"))
+
+// or in a report view's title or field formulas:
+.withTitleFields(List.of("${input.storeId}"))
+new QReportField().withName("storeId").withFormula("${input.storeId}")
+----
+
+* `dataSources` - *List of QReportDataSource, Required* - Definitions of the sources of data for the report.
+At least one is required.
+
+==== QReportDataSource
+Data sources for QQQ Reports can either reference {link-tables} within the QQQ Instance, or they can provide custom code in the form of a `CodeReference` to a  `Supplier`, for use cases such as a static data tab in an Excel report.
+
+*QReportDataSource Properties:*
+
+* `name` - *String, Required* - Unique name for the data source within its containing Report.
+* `sourceTable` - *String, Conditional* - Reference to a {link-table} in the QQQ Instance, which the data source queries data from.
+* `queryFilter` - *QQueryFilter* - If a `sourceTable` is defined, then the filter specified here is used to filter and sort the records queried from that table when generating the report.
+* `staticDataSupplier` - *QCodeReference, Conditional* - Reference to custom code which can be used to supply the data for the data source, as an alternative to querying a `sourceTable`.
+** Must be a `JAVA` code type
+** Must be a `REPORT_STATIC_DATA_SUPPLIER` code usage.
+** The referenced class must implement the interface: `Supplier<List<List<Serializable>>>`.
+
+==== QReportView
+Report Views control how the source data for a report is organized and presented to the user in the output report file.
+If a DataSource describes the rows for a report (e.g., what table provides what records), then a View may be thought of as describing the columns in the report.
+A single report can have multiple views, specifically, for the use-case where an Excel file is being generated, in which case each View creates a tab or sheet within the `xlsx` file.
+
+*QReportView Properties:*
+
+* `name` - *String, Required* - Unique name for the view within its containing Report.
+* `label` - *String* - Used as a sheet (tab) label in Excel formatted reports.
+* `type` - *enum of TABLE, SUMMARY, PIVOT. Required* - Defines the type of view being defined.
+** *TABLE* views are a simple listing of the records from the data source.
+** *SUMMARY* views are essentially pre-computed Pivot Tables.
+That is to say, the aggregation done by a Pivot Table in a spreadsheet file is done by QQQ while generating the report.
+In this way, a non-spreadsheet report (e.g., PDF or CSV) can have summarized data, as though it were a Pivot Table in a live spreadsheet.
+** *PIVOT* views produce actual Pivot Tables, and are only supported in Excel files _(and are not supported at the time of this writing)_.
+* `dataSourceName` - *String, Required* - Reference to a DataSource within the report, that is used to provide the rows for the view.
+* `varianceDataSourceName` - *String* - Optional reference to a second DataSource within the report, that is used in `*SUMMARY*` type views for computing variances.
+** For example, given a Data Source with a filter that selects all sales records for a given year, a Variance Data Source may have a filter that selects the previous year, for doing comparissons.
+* `pivotFields` - *List of String, Conditional* - For *SUMMARY* or *PIVOT* type views, specify the field(s) used as pivot rows.
+** For example, in a summary view of orders, you may "pivot" on the *customerId* field, to produce one row per-customer, with aggregate data for that customer.
+* `titleFormat` - *String* - Java Format String, used with `titleFields` (if given), to produce a title row, e.g., first row in the view (before any rows from the data source).
+* `titleFields` - *List of String, Conditional* - Used with `titleFormat`, to provide values for any format specifiers in the format string.
+Syntax to reference a field (e.g., from a report input field) is: `${input.NAME}`, where `NAME` is the `name` attribute of the inputField.
+** Example of using `titleFormat` and `titleFields`:
+
+[source,java]
+----
+// given these inputFields:
+new QFieldMetaData("startDate", QFieldType.DATE)
+new QFieldMetaData("endDate", QFieldType.DATE)
+
+// a view can have a title row like this:
+.withTitleFormat("Weekly Sales Report - %s - %s")
+.withTitleFields(List.of("${input.startDate}", "${input.endDate}"))
+----
+
+* `includeHeaderRow` - *boolean, default true* - Indication that first row of the view should be the column labels.
+** If true, then header row is put in the view.
+** If false, then no header row is put in the view.
+* `includeTotalRow` - *boolean, default false* - Indication that a totals row should be added to the view.
+All numeric columns are summed to produce values in the totals row.
+** If true, then totals row is put in the view.
+** If false, then no totals row is put in the view.
+* `includePivotSubTotals` - *boolean, default false* - For a *SUMMARY* or *PIVOT* type view, if there are more than 1 *pivotFields* being used, this field is an indication that each higher-level pivot should include sub-totals.
+** #TODO - provide example#
+* `columns` - *List of QReportField, required* - Definition of the columns to appear in the view.  See section on QReportField for details.
+* `orderByFields` - *List of QFilterOrderBy, optional* - For a *SUMMARY* or *PIVOT* type view, how to sort the rows.
+* `recordTransformStep` - *QCodeReference, subclass of `AbstractTransformStep`* - Custom code reference that can be used to transform records after they are queried from the data source, and before they are placed into the view.
+Can be used to transform or customize values, or to look up additional values to add to the report.
+** #TODO - provide example#
+* `viewCustomizer` - *QCodeReference, implementation of interface `Function<QReportView, QReportView>`* - Custom code reference that can be used to customize the report view, at runtime.
+Can be used, for example, to dynamically define the report's *columns*.
+** #TODO - provide example#
+
+===== QReportField
+Report Fields define the fields (AKA columns) of data that appear in a report view.
+These fields can either be direct references to fields from the report's data sources, or values computed using formula defined in the QReportField.
+
+*QReportField Properties:*
+
+* `name` - *String, required* - Unique identifier for the field within its ReportView.
+In general, will be a reference to a field from the ReportView's DataSource *unless a *formula* is given (for *SUMMARY* type views), the field is marked as *isVirtual*, or the field is marked as *showPossibleValueLabel*).
+* `label` - *String* - Optional text label to identify the field, for example, in a header row.
+If not given, may be derived from field, where possible.
+* `type` - *QFieldType*
+* `formula` - *String, conditional* - Required for *SUMMARY* type views.
+Defines the formula to be used for computing the value in this field.
+** For example:
+
+[source,java]
+----
+.withName("reportEndDate").withFormula("${input.endDate}")
+
+.withName("count").withFormula("${pivot.count.id}")
+
+.withName("percentOfTotal").withFormula("=DIVIDE(${pivot.count.id},${total.count.id})")
+
+.withName("sumCost").withFormula("${pivot.sum.cost}")
+
+.withName("sumCharge").withFormula("${pivot.sum.charge}")
+
+.withName("profit").withFormula("=MINUS(${pivot.sum.charge},${pivot.sum.cost})")
+
+.withName("totalCost").withFormula("=DIVIDE_SCALE(${pivot.sum.cost},${pivot.count.id},2)")
+
+.withName("revenuePer").withFormula("=DIVIDE_SCALE(${pivot.sum.charge},${pivot.count.id},2)")
+
+.withName("marginPer").withFormula("=MINUS(DIVIDE_SCALE(${pivot.sum.charge},${pivot.count.id},2),DIVIDE_SCALE(${pivot.sum.cost},${pivot.count.id},2))")
+
+.withName("thisWeekMargin").withFormula("=SCALE(DIVIDE(${thisRow.profit},${pivot.sum.charge}),2)")
+
+.withName("previousWeekProfit").withFormula("=MINUS(${variancePivot.sum.charge},${variancePivot.sum.cost})")
+
+.withName("previousWeekMargin").withFormula("=SCALE(DIVIDE(${thisRow.previousWeekProfit},${variancePivot.sum.charge}),2)")
+
+.withName("marginThisVsPrevious").withFormula("=SCALE(MINUS(${thisRow.margin},${thisRow.marginPrevious}),3)")
+
+.withName("exception").withFormula("""
+   =IF(LT(${thisRow.margin},0),Negative Margin,IF(LT(${thisRow.marginThisVsPrevious},0),Margin Decreased,""))""")
+----
+
+* `displayFormat` *String*
+* `isVirtual` *Boolean, default false* - (needs reviewed - may only be required for report views using a data source with a *staticDataSupplier*)
+* `showPossibleValueLabel` *Boolean, default false* - To show a translated value for a Possible Value field (e.g., a name or other value meaningful to a user, instead of a foreign key).
+* `sourceFieldName` *String* - Used for the scenario where a possibleValue field is included in a report both as the foreign key (raw, id value), and the translated "label" value.
+In that case, the field marked with *showPossibleValueLabel* = true should be given a different name, and should use *sourceFieldName* to indicate the field that has the id value.
+** For example:
+
+[source,java]
+----
+// this field would have the "raw" warehouseId values
+// e.g., integers - foreign keys to a warehouse table.  Generally useful for machines to know.
+new QReportField("warehouseId")
+   .withLabel("Warehouse Id"),
+
+// this field would have the translated values from the warehouse PossibleValueSource
+// for example, maybe the name field from the warehouse table.  A string, useful for humans to read.
+new QReportField("warehouseName")
+   .withSourceFieldName("warehouseId")
+   .withShowPossibleValueLabel(true)
+   .withLabel("Warehouse Name"),
+----
+

docs/metaData/SecurtiyKeyTypes.adoc

@@ -0,0 +1,80 @@
+[#SecurityKeyTypes]
+== Security Key Types
+include::../variables.adoc[]
+
+In QQQ, record-level security is provided by using a lock & key metaphor.
+
+The use-case being handled here is:
+
+* A user has full permission on a table (query, insert, update, and delete).
+* However, they should only be allowed to read a sub-set of the rows in the table.
+** e.g., maybe it's a multi-tenant system, or the table has user-specific records.
+
+The lock & key metaphor is realized by the user being associated with one or more "Keys"
+(as values in their session), and records in tables being associated with one or more "Locks"
+(as values in fields).
+A user is only allowed to access records where the user's key(s) match the record's security lock(s).
+
+For a practical example, picture a multi-tenant Order Management System,where all orders are assigned to a "client".
+Users (customers) should only be able to see orders associated with the client which that user works for.
+
+In this scenario, the `order` table would have a "lock" on its `clientId` field.
+Customer-users would have a `clientId` key in their session.
+When the QQQ backend did a search for records (e.g., an SQL query) it would implicitly
+(without any code being written by the application developer) filter the table to only
+allow the user to see records with their `clientId`.
+
+To implement this scenario, the application would define the following pieces of meta-data:
+
+* At the QQQ-Instance level, a `SecurityKeyType`,
+to define a domain of possible locks & keys within an application.
+** An application can define multiple Security Key Types.
+For example, maybe `clientId` and `userId` as key types.
+* At the per-table level, a `RecordSecurityLock`,
+which references a security key type, and how that key type should be applied to the table.
+** For example, what field stores the `clientId` value in the `order` table.
+* Finally, when a user's session is constructed via a QQQ Authentication provider,
+security key values are set, based on data from the authentication backend.
+
+=== Additional Scenarios
+
+==== All Access Key
+A "super-user" may be allowed to access all records in a table regardless of their record locks,
+if the Security Key Type specifies an `allAccessKeyName`,
+and if the user has a key in their session with that key name, and a value of `true`.
+Going back to the lock & key metaphor, this can be thought of as a "skeleton key",
+that can unlock any record lock (of the security key's type).
+
+==== Null Value Behaviors
+In a record security lock, different behaviors can be defined for handling rows with a null key value.
+
+For example:
+
+* Sometimes orders may be loaded into the OMS system described above, where the application doesn't yet know what client the order belongs to.
+In this case, the application may need to ensure that such records, with a `null` value in `clientId` are hidden from customer-users,
+to avoid potentially leaking a different client's data.
+** This can be accomplished with a record security lock on the `order` table, with a `nullValueBehavior` of `DENY`.
+** Furthermore, if internal/admin users _should_ be given access to such records, then the security key type can be
+configured with a `nullValueBehaviorKeyName` (e.g., `"clientIdNullValueBehavior"`), which can be set per-user to allow
+access to records, overriding the table lock's specified `nullValueBehavior`.
+*** This could also be done by giving internal/admin users an `allAccessKey`, but sometimes that is not what is required.
+
+* Maybe a warehouse locations table is assigned a `clientId` once inventory for a client is placed in the location,
+at which point in time, only the client's users should be allowed to see the record.
+But, if no client has been assigned to the location, and `clientId` is `null`,
+then you may want to allow any user to see such records.
+** This can be accomplished with a record security lock on the Warehouse Locations table, with a `nullValueBehavior` of `ALLOW`.
+
+=== QSecurityKeyType
+A Security Key Type is defined in a QQQ Instance in a `*QSecurityKeyType*` object.
+
+*QSecurityKeyType Properties:*
+
+* `name` - *String, Required* - Unique name for this security key type within the QQQ Instance.
+* `allAccessKeyName` - *String* - Optional name of the all-access security key associated with this key type.
+* `nullValueBehaviorKeyName` - *String* - Optional name of the null-value-behavior overriding security key associated with this key type.
+** Note, `name`, `allAccessKeyName`, and `nullValueBehaviorKeyName` are all checked against each other for uniqueness.
+A `QInstanceValidationException` will be thrown if any name collisions occur.
+* `possibleValueSourceName` - *String* - Optional reference to a possible value source from which value for the key can come.
+
+

docs/metaData/Tables.adoc

@@ -0,0 +1,284 @@
+[#Tables]
+== Tables
+include::../variables.adoc[]
+
+One of the most common types of object in a QQQ Instance is the Table.
+In the most common use-case, a QQQ Table may be the in-app representation of a Database table.
+That is, it is a collection of records (or rows) of data, each of which has a set of fields (or columns).
+
+QQQ also allows other types of data sources ({link-backends}) to be used as tables, such as File systems, API's, etc.
+All of these backend types present the same interfaces (both user-interfaces, and application programming interfaces), regardless of their backend type.
+
+=== QTableMetaData
+Tables are defined in a QQQ Instance in `*QTableMetaData*` objects.
+All tables must reference a {link-backend}, a list of fields that define the shape of records in the table, and additional data to describe how to work with the table within its backend.
+
+*QTableMetaData Properties:*
+
+* `name` - *String, Required* - Unique name for the table within the QQQ Instance.
+* `label` - *String* - User-facing label for the table, presented in User Interfaces.
+Inferred from `name` if not set.
+* `backendName` - *String, Required* - Name of a {link-backend} in which this table's data is managed.
+* `fields` - *Map of String → {link-field}, Required* - The columns of data that make up all records in this table.
+* `primaryKeyField` - *String, Conditional* - Name of a {link-field} that serves as the primary key (unique identifier) for records in this table.
+** Whether a primary key field is required or not depends on the backend type that the table belongs to.
+* `uniqueKeys` - *List of UniqueKey* - Definition of additional unique keys or constraints (from an RDBMS point of view) from the table.
+e.g., sets of columns which must have unique values for each record in the table.
+The properties of the `UniqueKey` object are:
+** `fieldNames` - *List of String, Required* - List of field names from this table.
+** `label` - *String* - Optional label to be shown to users with error messages (e.g., for violation of this unique key).
+* `backendDetails` - *QTableBackendDetails or subclass* - Additional data to configure the table within its {link-backend}.
+** For example, for an RDBMS-type backend, the name of the table within the database.
+** vs. a FileSystem backend, this may be the sub-path where files for the table are stored.
+** #todo - details on these#
+* `automationDetails` - *<<QTableAutomationDetails>>* - Configuration of automated jobs that run against records in the table, e.g., upon insert or update.
+* `customizers` - *Map of String → QCodeReference* - References to custom code that are injected into standard table actions, that allow applications to customize certain parts of how the table works.
+** Allowed values for keys in this map come from the `role` property of the `TableCustomizers` enum.
+** Based on the key in this map, the `QCodeReference` used as the value must be of the appropriate java type, as specified in the `expectedType` property of the `TableCustomizers` enum value corresponding to the key.
+** Example:
+
+[source,java]
+----
+// in defining a QTableMetaData, a customizer can be added as:
+.withCustomizer(TableCustomizers.PRE_INSERT_RECORD, new QCodeReference(MyPreInsCustomizer.class))
+
+// where MyPreInsCustomizer would be defined as:
+public class MyPreInsCustomizer extends AbstractPreInsertCustomizer
+----
+
+
+* `isHidden` - *Boolean, default false* - Option to hide the table from all User Interfaces.
+* `parentAppName` - *String* - Name of a {link-app} that this table exists within.
+** This field generally does not need to be set on the table when it is defined, but rather, is set when the table gets placed within an app.
+* `icon` - *QIcon* - Icon associated with this table in certain user interfaces.  See {link-icons}.
+* `recordLabelFormat` - *String* - Java Format String, used with `recordLabelFields` to produce a label shown for records from the table.
+* `recordLabelFields` - *List of String, Conditional* - Used with `recordLabelFormat` to provide values for any format specifiers in the format string.
+These strings must be field names within the table.
+** Example of using `recordLabelFormat` and `recordLabelFields`:
+
+[source,java]
+----
+// given these fields in the table:
+new QFieldMetaData("name", QFieldType.STRING)
+new QFieldMetaData("birthDate", QFieldType.DATE)
+
+// We can produce a record label such as "Darin Kelkhoff (1980-05-31)" via:
+.withRecordLabelFormat("%s (%s)")
+.withRecordLabelFields(List.of("name", "birthDate"))
+----
+
+* `sections` - *List of <<QFieldSection>>* - Mechanism to organize fields within user interfaces, into logical sections.
+If any sections are present in the table meta data, then all fields in the table must be listed in exactly 1 section.
+If no sections are defined, then instance enrichment will define default sections.
+* `associatedScripts` - *List of <<AssociatedScript>>* - Definition of user-defined scripts that can be associated with records within the table.
+* `enabledCapabilities` and `disabledCapabilities` - *Set of Capability enum values* - Overrides from the backend level, for capabilities that this table does or does not possess.
+* `associations` - *List of <<Association>>* - tables whose records can be managed along with records from this table.  See below for details.
+* `recordSecurityLocks` - *List of <<RecordSecurityLock>>* - locks that apply to records in the table - e.g., to control what users can or cannot access records in the table.
+See RecordSecurityLock below for details.
+* `permissionRules` - *QPermissionRules object* - define the permission/access rules for the table.
+See {link-permissionRules} for details.
+* `auditRules` - *<<QAuditRules>> object* - define the audit rules for the table.
+See QAuditRules below for details.
+* `cacheOf` - *<<CacheOf>> object* - specify that this table serves as a "cache of" another table.
+See CacheOf object below for details.
+* `exposedJoins` - *List of <<ExposedJoin>> objects* - optional list of joined tables that are to be exposed in User Interfaces.
+See ExposedJoin object below for details.
+
+#todo: supplementalMetaData (API)#
+
+
+==== QFieldSection
+When users view records from a QQQ Table in a UI, fields are organized on the screen based on the `QFieldSection` objects in the table's meta-data.
+
+*QFieldSection Properties:*
+
+* `name` - *String, Required* - unique identifier for the section within its table.
+* `label` - *String* - User-facing label for the section, presented in User Interfaces.
+Inferred from `name` if not set.
+* `tier` - *enum* - importance of the fields in section for the table.
+Different tiers may be presented differently in UI's.
+Only a single `T1` section is allowed per-table.  Possible values are: `T1`, `T2`, and `T3`.
+* `icon` - *QIcon* - Icon associated with this section in certain user interfaces.  See {link-icons}.
+* `isHidden` - *Boolean, default false* - Option to hide the table from all User Interfaces.
+* `gridColumns` - *Integer* - Option to specify how many columns in a grid layout the section should use.
+For the Material-Dashboard frontend, this is a grid of 12.
+* `fieldNames` - *List of String, Conditional* - List of names of {link-fields} from this table to be included in this section.
+* `widgetName` - *String, Conditional* - Name of a {link-widget} to be displayed in this section.
+** Note that exactly one of `fieldNames` or `widgetName` must be used.
+
+==== QTableAutomationDetails
+Records in QQQ can have application-defined custom actions automatically asynchronously executed against them after they are inserted or updated.
+The configuration to enable this functionality is assigned to a table in a `QTableAutomationDetails` object.
+
+*QTableAutomationDetails Properties:*
+
+* `statusTracking` - *AutomationStatusTracking object, Required* - define how QQQ should keep track, per record, its status (e.g., pending-insert-automations, running-update-automations, etc).
+Properties of `AutomationStatusTracking` object are:
+** `type` - *enum, Required* - what type of tracking is used for the table.
+Possible values are:
+*** `FIELD_IN_TABLE` - specifies that the table has a field which stores an `AutomationStatus` id.
+*** _Additional types may be defined in the future, such as ONE_TO_ONE_TABLE or SHARED_TABLE._
+** `fieldName` - *String, Conditional* - for `type=FIELD_IN_TABLE`, this property specifies the name of the {link-field} in the table that stores the `AutomationStatus` id.
+* `providerName` - *String, Required* - name of an Automation Provider within the QQQ Instance, which is responsible for running the automations on this table.
+* `overrideBatchSize` - *Integer* - optional control over how many records from the table are processed in a single batch/page.
+For tables with "slow" actions (e.g., one that may need to make an API call per-record), using a smaller batch size (say, 50) may be required to avoid timeout errors.
+* `actions` - *List of TableAutomationAction* - list of the actions to perform on new and updated records in the table.
+Properties are:
+** `name` - *String, Required* - unique identifier for the action within its table.
+** `triggerEvent` - *enum, Required* - indicate which event type (`POST_INSERT`, `POST_UPDATE`, or `PRE_DELETE` (which is not yet implemented)) the action applies to.
+** `priority` - *Integer, default 500* - mechanism to control the order in which actions on a table are executed, if there are more than one.
+Actions with a smaller value for `priority` are executed first.  Ties are broken in an undefined manner.
+** `filter` - *QQueryFilter* - optional filter that gets applied to records when they match the `triggerEvent`, to control which records have the action ran against them.
+** `includeRecordAssociations` - *Boolean, default false* - for tables that have associations, control whether or not a record's associated records are loaded when records are fetched and passed into the action's custom code.
+** `values` - *Map of String → Serializable* - optional application-defined map of name=value pairs that can be passed into the action's custom code.
+** `processName` - *String, Conditional* - name of a {link-processes} in the QQQ Instance which is executed as the custom-code of the action.
+** `codeReference` - *QCodeReference, Conditional* - reference to a class that extends `RecordAutomationHandler`, to be executed as the custom-code of the action.
+*** Note, exactly one of `processName` or `codeReference` must be provided.
+
+
+
+==== Association
+An `Association` is a way to define a relationship between tables, that facilitates, for example, a parent record having a list of its child records included in it when it is returned from a Query.
+Similarly, associated records can automatically be inserted/updated/deleted if they are included in a parent record when it is stored.
+
+*Association Properties:*
+
+* `name` - *String, Required* - unique name for the association within this table.
+Used as the key in the `associatedRecords` map within `QRecord` objects for this table.
+* `associatedTableName` - *String, Required* - name of a {link-table}, which is the associated table.
+* `joinName` - *String, Required* - name of a {link-join} in the instance, which defines how the tables are joined.
+
+
+
+==== RecordSecurityLock
+A `RecordSecurityLock` is the mechanism through which users can be allowed or denied access to read and/or write records, based on values in the record, and values in the user's session.
+Record security locks must correspond to a {link-securityKeyType}.
+
+For example:
+
+* An instance may have a security key type called `clientId`.
+* Users may have 1 or more `clientId` values in their Session, or, they may have an "All Clients" key in their session (e.g., for internal/admin users).
+* For some tables, it may be required to limit visibility to records based on a user's `clientId` key.
+To do this. a *RecordSecurityLock* would be applied to the table, specifying the `clientId` field corresponds to the `clientId` security key.
+* With these settings in place, QQQ will prevent users from viewing records from this table that do not have a matching key, and will similarly prevent users from writing records with an invalid key value.
+** For example, in an RDBMS backend, all `SELECT` statements generated against such a table will have an implicit filter, such as `AND client_id = ?` based on the user's security key values.
+
+*RecordSecurityLock Properties:*
+
+* `securityKeyType` - *String, Required* - name of a {link-securityKeyType} in the Instance.
+* `fieldName` - *String, Required* - name of a {link-field} in this table (or a joined table, if `joinNameChain` is set), where the value for the lock is stored.
+* `joinNameChain` - *List of String* - if the lock value is not stored in this table, but rather comes from a joined table, then this property defines the path of joins from this table to the table with the lock field.
+* `nullValueBehavior` - *enum, default: DENY* - control how records with a `null` value in the lock field should behave.
+Possible values are:
+** `DENY` - deny all users access to a record with a `null` value in the lock field (unless the user has an all-access key - see {link-securityKeyType})
+** `ALLOW` - allow all users access to a record with a `null` value in the lock field.
+** `ALLOW_WRITE_ONLY` - allow all users to write records with `null` in the lock field, but deny reads on records with `null` in the lock field (also excepted by all-access keys).
+* `lockScope` - *enum, default: READ_AND_WRITE* - control what types of operations the lock applies to.
+Possible values are:
+** `READ_AND_WRITE` - control both reading and writing records based on the user having an appropriate security key.
+** `WRITE` - allow all users to read the record, but limit writes to users with an appropriate security key.
+
+
+
+==== QAuditRules
+The audit rules on a table define the level of detail that is automatically stored in the audit table (if any) for DML actions (Insert, Update, Delete).
+
+*QAuditRules Properties:*
+
+* `auditLevel` - *enum, Required* - level of details that are audited.
+Possible values are:
+** `NONE` - no automatic audits are stored for the table.
+** `RECORD` - only record-level audits are stored for the table (e.g., a message such as "record was edited", but without field-level details)
+** `FIELD` - full field-level audits are stored (e.g., including all old & new values as audit details).
+
+
+
+==== CacheOf
+One QQQ Table can be defined as a "cache of" another QQQ Table by assigning a `CacheOf` object to the table which will function as the cache.
+_Note, at this time, only limited use-cases are supported._
+
+*CacheOf Properties:*
+
+* `sourceTable` - *String, Required* - name of the other QQQ Table that is the source of data in this cache.
+* `expirationSeconds` - *Integer* - optional number of seconds that a cached record is allowed to exist before it is considered expired, and must be re-fetched from the source table.
+* `cachedDateFieldName` - *String, Conditional* - used with `expirationSeconds` to define the field in this table that is used for storing the timestamp for when the record was cached.
+* `useCases` - *List of CacheUseCase* - what caching use-cases are to be implemented.
+
+Properties of *CacheUseCase* are:
+
+* `type` - *Enum, Required* - the type of use-case.  Possible values are:
+** `PRIMARY_KEY_TO_PRIMARY_KEY` - the primary key in the cache table equals the primary key in the source table.
+** `UNIQUE_KEY_TO_PRIMARY_KEY` - a unique key in the cache table equals the primary key in the source table.
+** `UNIQUE_KEY_TO_UNIQUE_KEY` - a unique key in the cache table equals a unique key in the source table.
+* `cacheSourceMisses` - *Boolean, default false* - whether or not, if a "miss" happens in the source, if that fact gets cached.
+* `cacheUniqueKey` - *UniqueKey, conditional* - define the fields in the cache table that define the unique key being used as the cache key.
+* `sourceUniqueKey` - *UniqueKey, conditional* - define the fields in the source table that define the unique key being used as the cache key.
+* `doCopySourcePrimaryKeyToCache` - *Boolean, default false* - specify whether or not the value of the primary key in the source table should be copied into records built in the cache table.
+* `excludeRecordsMatching` - *List of QQueryFilter* - optional filter to be applied to records before they are cached.
+If a record matches the filter, then it will not be cached.
+
+
+==== ExposedJoin
+Query screens in QQQ applications can potentially allow users to both display fields from joined tables, and filter by fields from joined tables, for any {link-join} explicitly defined as an *Exposed Join*.
+
+_The reasoning why not all joins are implicitly exposed is that in many applications, the full join-graph can sometimes be overwhelming, surprisingly broad, and not necessarily practically useful.
+This could be subject to change in the future, e.g., given a UI that allowed users to more explicitly add additional join tables..._
+
+*ExposedJoin Properties:*
+
+* `label` - *String, Required* - how the joined table should be presented in the UI.
+* `joinTable` - *String, Required* - name of the QQQ Table that is joined to this table, and is being exposed as a join in the UI.
+* `joinPath` - *List of String, Required* - names of 1 or more QQQ Joins that describe how to get from this table to the join table.
+
+
+
+==== AssociatedScript
+A QQQ Table can have end-user defined Script records associated with individual records in the table by use of the `associatedScripts` property of the table's meta-data.
+
+The "types" of these scripts (e.g., how they are used in an application) are wholly application-designed & managed.
+QQQ provides the mechanism for UI's to present and manage such scripts (e.g., the *Developer Mode* screen in the Material Dashboard), as well as an interface to load & execute such scripts `RunAssociatedScriptAction`).
+
+*AssociatedScript Properties:*
+
+* `fieldName` - *String, Required* - name of a {link-field} in the table which stores the id of the associated script record.
+* `scriptTypeId` - *Serializable (typically Integer), Required* - primary key value from the `"scriptType"` table in the instance, to designate the type of the Script.
+* `scriptTester` - *QCodeReference* - reference to a class which implements `TestScriptActionInterface`, that can be used by UI's for running an associated script to test it.
+
+
+
+=== Supplemental Meta Data
+==== QQQ Frontend Material Dashboard
+When running a QQQ application with the QQQ Frontend Material Dashboard module (QFMD),
+there are various pieces of supplemental meta-data which can be assigned to a Table,
+to modify some behaviors for the table in this UI.
+
+===== Default Quick Filter Field Names
+QFMD's table query has a "Basic" mode, which will always display a subset of the table's fields as quick-access filters.
+By default, the "Tier 1" fields on a table (e.g., fields in a Section that is marked as T1) will be used for this purpose.
+
+However, you can customize which fields are shown as the default quick-filter fields, by providing a list of field names in a
+`MaterialDashboardTableMetaData` object, placed in the table's `supplementalMetaData`.
+
+[source,java]
+----
+table.withSupplementalMetaData(new MaterialDashboardTableMetaData()
+    .withDefaultQuickFilterFieldNames(List.of("id", "warehouseId", "statusId", "orderDate")));
+----
+
+===== Go To Field Names
+QFMD has a feature where a table's query screen can include a "Go To" button,
+which a user can hit to open a modal popup, into which the user can enter a record's identifier,
+to be brought directly to the record matching that identifier.
+
+To use this feature, the table must have a List of `GotoFieldNames` set in its
+`MaterialDashboardTableMetaData` object in the table's `supplementalMetaData`.
+
+Each entry in this list is actually a list of fields, e.g., to account for a multi-value unique-key.
+
+[source,java]
+----
+table.withSupplementalMetaData(new MaterialDashboardTableMetaData()
+    .withGotoFieldNames(List.of(
+        List.of("id"),
+        List.of("partnerName", "partnerOrderId"))));
+----

docs/metaData/Widgets.adoc

@@ -0,0 +1,51 @@
+[#Widgets]
+== Widgets
+include::../variables.adoc[]
+
+Widgets are the most customizable UI components in QQQ.
+They can be used either on App Home Screens (e.g., as Dashboard screens),
+or they can be included into Record View screens.
+
+QQQ defines several types of widgets, such as charts (pie, bar, line),
+numeric displays, application-populated tables, or even fully custom HTML.
+
+=== QWidgetMetaData
+A Widget is defined in a QQQ Instance in a `*QWidgetMetaData*` object.
+
+*QWidgetMetaData Properties:*
+
+* `name` - *String, Required* - Unique name for the widget within the QQQ Instance.
+* `type` - *String, Required* - Specifies the UI & data type for the widget.
+* `label` - *String* - User-facing header or title for a widget.
+* `tooltip` - *String* - Text contents to be placed in a tooltip associated with the widget's label in the UI.
+** Values should come from the `WidgetType` enum's `getType()` method (e.g., `WidgetType.BAR_CHART.getType()`)
+* `gridColumns` - *Integer* - for a desktop-sized screen, in a 12-based grid,
+how many columns the widget should consume.
+* `codeReference` - *QCodeReference, Required* - Reference to the custom code,
+a subclass of `AbstractWidgetRenderer`, which is responsible for loading data to render the widget.
+* `footerHTML` - *String* - HTML String, which, if present, will be displayed in the
+footer of the widget (not supported by all widget types).
+* `isCard` - *boolean, default false* #TODO#
+* `showReloadButton` - *boolean, default true* #TODO#
+* `showExportButton` - *boolean, default false* #TODO#
+* `dropdowns` - #TODO#
+* `storeDropdownSelections` - *boolean* #TODO#
+* `icons` - *Map<String, QIcon>* #TODO#
+* `defaultValues` - *Map<String, Serializable>* #TODO#
+
+There are also some subclasses of `QWidgetMetaData`, for some specific widget types:
+
+*ParentWidgetMetaData Properties:*
+
+* `title` - *String* #TODO - how does this differ from label?#
+* `childWidgetNameList` - *List<String>, Required*
+* `childProcessNameList` - *List<String>* #TODO appears unused - check, and delete#
+* `laytoutType` - *enum of GRID or TABS, default GRID*
+
+*QNoCodeWidgetMetaData Properties:*
+
+* `values` - *List<AbstractWidgetValueSource>* #TODO#
+* `outputs` - *List<AbstractWidgetOutput>* #TODO#
+
+#TODO - Examples#
+

docs/misc/ProcessBackendSteps.adoc

@@ -0,0 +1,155 @@
+== Process Backend Steps
+include::../variables.adoc[]
+
+In many QQQ applications, much of the code that engineers write will take the form of Backend Steps for {link-processes}.
+Such code is defined in classes which implement the interface `BackendStep`.  
+This interface defines only a single method:
+[source,java]
+.BackendStep.java
+----
+public interface BackendStep
+{
+   /*******************************************************************************
+    ** Execute the backend step - using the request as input, and the result as output.
+    **
+    *******************************************************************************/
+   void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException;
+}
+
+----
+
+Process backend steps have access to state information - specifically, a list of records, and a map of name=value pairs - in the input & output objects.
+This state data is persisted by QQQ between steps (e.g., if a frontend step is presented to a user between backend steps).
+
+
+=== RunBackendStepInput
+All input data to the step is available in the `RunBackendStepInput` object.
+Key methods in this class are:
+
+* `getRecords()` - Returns the List of <<QRecords>> that are currently being acted on in the process.
+* `getValues()` - Returns a Map of String -> Serializable; name=value pairs that are the state-data of the process.
+** Values can be added to this state from a process's meta-data, from a screen, or from another backend step.
+* `getValue(String fieldName)` - Returns a specific value, by name, from the process's state.
+** This method has several variations that return the value as a specific type, such as `getValueString`, `getValueInteger`, `getValueBoolean`...
+* `getAsyncJobCallback()` - Accessor for an `AsyncJobCallback` object, which provides a way for the process backend step to communicate about its status or progress with a user running the process in a frontend.
+Provides methods:
+** `updateStatus(String message)` - Where general status messages can be given.
+For example, `"Loading census data"`
+** `updateStatus(int current, int total)` - For updating a progress meter.
+e.g., "47 of 1701" would be display by calling `.updateStatus(47, 1701)`
+* `getFrontendStepBehavior()` - Enum, indicating what should happen when a frontend step is encountered as the process's next step to run.
+Possible values are:
+** `BREAK` - Indicates that the process's execution should be suspended, so that the screen represented by the frontend step can be presented to a user.
+This would be the expected behavior if a process is being run by a user from a UI.
+** `SKIP` - Indicates that frontend steps should be skipped.
+This would be the expected behavior if a process is running from a scheduled job (without a user present to drive it), for example.
+** `FAIL` - Indicates that the process should end with an exception if a frontend step is encountered.
+** A backend step may want to act differently based on its frontendStepBehavior.
+For example, additional data may be looked up for displaying to a user if the behavior is `BREAK`.
+* `getBasepullLastRunTime()` - For <<BasepullConfiguration,Basepull>> processes, this is the `Instant` stored in the basepull table as the process's last run time.
+
+=== RunBackendStepOutput
+All output from a process step should be placed in its `RunBackendStepOutput` object (and/or stored to a backend, as appropriate).
+Key methods in this class are:
+
+* `addValue(String fieldName, Serializable value)` - Adds a single named value to the process's state, overwriting it the value if it already exists.
+* `addRecord(QRecord record)` - Add a `<<QRecord>>` to the process's output.
+* `addAuditSingleInput(AuditSingleInput auditSingleInput)` - Add a new entry to the process's list of audit inputs, to be stored at the completion of the process.
+** An `AuditSingleInput` object can most easily be built with the constructor: `AuditSingleInput(String tableName, QRecord record, String auditMessage)`.
+** Additional audit details messages (sub-bullets that accompany the primary `auditMessage`) can be added to an `AuditSingleInput` via the `addDetail(String message)` method.
+** _Note that at this time, the automatic storing of these audits is only provided by the execute step of a StreamedETLWithFrontendProcesses._
+
+=== Example
+
+[source,java]
+.Example of a BackendStep
+----
+/*******************************************************************************
+ ** For the "person" table's "Add Age" process -
+ ** For each input person record, add the specified yearsToAdd to their age.
+ *******************************************************************************/
+public class AddAge implements BackendStep
+{
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput)
+   {
+      /////////////////////////////////////////////////////////////////
+      // get the yearsToAdd input field value from the process input //
+      /////////////////////////////////////////////////////////////////
+      Integer yearsToAdd = runBackendStepInput.getValueInteger("yearsToAdd");
+      int totalYearsAdded = 0;
+
+      ///////////////////////////////////////////////////
+      // loop over the records passed into the process //
+      ///////////////////////////////////////////////////
+      for(QRecord record : runBackendStepInput.getRecords())
+      {
+         Integer age = record.getValueInteger("age");
+         age += yearsToAdd;
+         totalYearsAdded += yearsToAdd;
+
+         ////////////////////////////////////////////////////////////////////////////////////////////
+         // update the record with the new "age" value.                                            //
+         // note that this update record object will implicitly be available to the process's next //
+         // backend step, via the sharing of the processState object.                              //
+         ////////////////////////////////////////////////////////////////////////////////////////////
+         record.setValue("age", age);
+      }
+
+      /////////////////////////////////////////
+      // set an output value for the process //
+      /////////////////////////////////////////
+      runBackendStepOutput.addValue("totalYearsAdded", totalYearsAdded);
+   }
+}
+
+----
+
+
+=== Backend Steps for StreamedETLWithFrontendProcesses
+
+For <<StreamedETLWithFrontendProcess>> type processes, backend steps are defined a little bit differently than they are for other process types.
+In this type of process, the process meta-data defines 3 backend steps which are built-in to QQQ, and which do not have any custom application logic.
+These steps are:
+
+* `StreamedETLPreviewStep`
+* `StreamedETLValidateStep`
+* `StreamedETLExecuteStep`
+
+For custom application logic to be implemented in a StreamedETLWithFrontendProcesses, an application engineer must define (up to) 3 backend step classes which are loaded by the steps listed above.
+These application-defined steps must extend specific classes (which themselves implement the `BackendStep` interface), to provide the needed logic of this style of process.
+These steps are:
+
+* *Extract* - a subclass of `AbstractExtractStep` - is responsible for Extracting records from the source table.
+** For this step, we can often use the QQQ-provided class `ExtractViaQueryStep`, or sometimes a subclass of it.
+** The Extract step is called before the Preview, Validate, and Result screens, though for the Preview screen, it is set to only extract a small number of records (10).
+* *Transform* - a subclass of `AbstractTransformStep` - is responsible for applying the majority of the business logic of the process.
+In ETL terminology, this is the "Transform" action - which means applying some type of logical transformation an input record (found by the Extract step) to generate an output record (stored by the Load step).
+** A Transform step's `run` method will be called, potentially, multiple times, each time with a page of records in the `runBackendStepInput` parameter.
+** This method is responsible for adding records to the `runBackendStepOutput`, which will then be passed to the *Load* step.
+** This class is also responsible for implementing the method `getProcessSummary`, which provides the data to the *Validate* screen.
+** The run method will generally update ProcessSummaryLine objects to facilitate this functionality.
+** The Transform step is called before the Preview, Validate, and Result screens, consuming all records selected by the Extract step.
+* *Load* - a subclass of `AbstractLoadStep` - is responsible for the Load function of the ETL job.
+_A quick word on terminology - this step is actually doing what we are more likely to think of as storing data - which feels like the opposite of “loading” - but we use the name Load to keep in line with the ETL naming convention…_
+** The Load step is ONLY called before the Result screen is presented (possibly after Preview, if the user chose to skip validation, otherwise, after validation).
+** Similar to the Transform step, the Load step's `run` method will be called potentially multiple times, with pages of records in its input.
+** As such, the Load step is generally the only step where data writes should occur.
+*** e.g., a Transform step should not do any writes, as it will be called when the user is going to the Preview & Validate screens - e.g., before the user confirmed that they want to execute the action!
+** A common pattern is that the Load step just needs to insert or update the list of records output by the Transform step, in which case the QQQ-provided `LoadViaInsertStep` or `LoadViaUpdateStep` can be used, but custom use-cases can be built as well.
+
+Another distinction between StreamedELTWithFrontendProcess steps and general QQQ process backend steps, is that the list of records in the input & output objects is NOT shared for StreamedELTWithFrontendProcess steps.
+The direct implication of this is, that a Transform step MUST explicitly call `output.addRecord()` for any records that it wants to pass along to the Load step.
+
+==== Example
+
+[source,java]
+.Examples of a Transform and Load step for a StreamedELTWithFrontendProcess
+----
+   // todo!
+----
+
+#todo: more details on these 3 specialized types of process steps (e.g., method to overload, when stuff like pre-action is called; how summaries work).#

docs/misc/QContext.adoc

@@ -0,0 +1,30 @@
+== QContext
+include::../variables.adoc[]
+
+The class `QContext` contains a collection of thread-local variables, to define the current context of the QQQ code that is currently running.
+For example, what `QInstance` (meta-data container) is being used, what `QSession` (user attributes) is active, etc.
+
+Most of the time, main-line application code does not need to worry about setting up the `QContext` - although unit-test code can is a common exception to that rule.
+This is because all of QQQ's entry-points into execution (e.g., web context handlers, CLI startup methods, schedulers, and multi-threaded executors) take care of initializing the context.
+
+It is more common though, for application code to need to get data from the context, such as the current session or any piece of meta-data from the QInstance.
+The methods to access data from the `QContext` are fairly straightforward:
+
+=== Examples
+==== Get a QTableMetaData from the active QInstance
+[source,java]
+----
+QTableMeataData myTable = QContext.getQInstance().getTable("myTable");
+for(QFieldMeataData field : myTable.getFields().values())
+{
+   // ...
+}
+----
+
+==== Get a security key value for the current user session
+[source,java]
+----
+QSession           session   = QContext.getQSession();
+List<Serializable> clientIds = session.getSecurityKeyValues("clientId");
+----
+

docs/misc/QRecordEntities.adoc

@@ -0,0 +1,116 @@
+== QRecordEntities
+include::../variables.adoc[]
+
+While `<<QRecords>>` provide a flexible mechanism for passing around record data in a QQQ Application, they have one big disadvantage from the point-of-view of a Java Application:
+They do not provide a mechanism to ensure compile-time checks of field names or field types.
+
+As such, an alternative mechanism exists, which allows records in a QQQ application to be worked with following a more familiar Java Bean (Entity Bean) like pattern.
+This mechanism is known as a `QRecordEntity`.
+
+Specifically speaking, `QRecordEntity` is an abstract base class, whose purpose is to be the base class for entity-bean classes.
+Using reflection, `QRecordEntity` is able to convert objects back and forth from `QRecord` to specific entity-bean subtypes.
+
+For example, the method `QRecordEntity::toQRecord()` converts a record entity object into a `QRecord`.
+
+Inversely, `QRecordEntity::populateFromQRecord(QRecord record)` sets fields in a record entity object, based on the values in the supplied `QRecord`.
+It is conventional for a subclass of `QRecordEntity` to have both a no-arg (default) constructor, and a constructor that takes a `QRecord` as a parameter, and calls `populateFromQRecord`.
+
+In addition to these constructors, a `QRecordEntity` subclass will generally contain:
+
+* A `public static final String TABLE_NAME`, used throughout the application as a constant reference to the name for the {link-table}.
+* A series of `private` fields, corresponding to the fields in the table that the entity represents.
+** If these fields are annotated as `@QField()`, then the {link-table} meta-data for the table that the entity represents can in part be inferred by QQQ, by using the method `QTableMetaData::withFieldsFromEntity`.
+* `getX()`, `setX()`, and `withX()` methods for all of the entity's fields.
+
+=== Examples
+[source,java]
+.Example Definition of a QRecordEntity subclass:  Person.java
+----
+/*******************************************************************************
+ ** QRecordEntity for the person table.
+ *******************************************************************************/
+public class Person extends QRecordEntity
+{
+   public static final String TABLE_NAME = "person";
+
+   @QField(isEditable = false)
+   private Integer id;
+
+   @QField()
+   private String firstName;
+
+   @QField()
+   private String lastName;
+
+   @QField()
+   private Integer age;
+
+   // all other fields
+
+   /*******************************************************************************
+    ** Default constructor
+    *******************************************************************************/
+   public Person()
+   {
+   }
+
+
+   /*******************************************************************************
+    ** Constructor that takes a QRecord
+    *******************************************************************************/
+   public Person(QRecord record)
+   {
+      populateFromQRecord(record);
+   }
+
+
+   /*******************************************************************************
+    ** Custom method to concatenate firstName and lastName
+    *******************************************************************************/
+   public String getFullName()
+   {
+      //////////////////////////////////////////////////////////////////////
+      // if there were more than an example, we could do some null-checks //
+      // here to avoid silly output like "Bobby null" :)                  //
+      //////////////////////////////////////////////////////////////////////
+      return (firstName + " " + lastName);
+   }
+
+   // all getter, setter, and fluent setter (wither) methods
+}
+----
+
+The core ORM actions and process-execution actions of QQQ work with `QRecords`.
+However, application engineers may want to apply patterns like the following example, to gain the compile-time safety of `QRecordEntities`:
+
+[source,java]
+.Example Usage of a QRecordEntity
+----
+//////////////////////////////////////////////////////////////////////
+// assume we're working with the "person" table & entity from above //
+//////////////////////////////////////////////////////////////////////
+List<QRecord> recordsToUpdate = new ArrayList<>();
+for(QRecord record : inputRecordList)
+{
+   /////////////////////////////////////////////////////////////////////////////
+   // call the constructor that copies values from the record into the entity //
+   /////////////////////////////////////////////////////////////////////////////
+   Person person = new Person(record);
+
+   ////////////////////////////////////////////////
+   // call a custom method defined in the entity //
+   ////////////////////////////////////////////////
+   LOG.info("Processing: " + person.getFullName());
+
+   /////////////////////////////////////////////////////////////
+   // age is an Integer, so no type-conversion is needed here //
+   /////////////////////////////////////////////////////////////
+   person.setAge(person.getAge() + 1);
+
+   ///////////////////////////////////////////////////////////////////////////
+   // to pass the updated records to UpdateAction, convert them to QRecords //
+   ///////////////////////////////////////////////////////////////////////////
+   recordsToUpdate.add(person.toQRecord());
+}
+----
+

docs/misc/QRecords.adoc

@@ -0,0 +1,68 @@
+== QRecords
+include::../variables.adoc[]
+
+Almost all code inside a QQQ application will be dealing with *Records* (aka Tuples or Rows).
+That is: a collection of named values, representing a single Entity, Fact, or, Row from a {link-table}.
+
+The class that QQQ uses to work with records is called:  `QRecord`.
+
+=== Values
+At its core, a `QRecord` is a wrapper around a `Map<String, Serializable> values`.
+These are the *actual* values for the fields in the table for the record.
+That is, direct representations of the values as they are stored in the {link-backend}.
+
+The keys in the `values` map are names from the {link-fields} in the {link-table}.
+
+The values in `values` map are declared as `Serializable` (to help ensure the serializability of the `QRecord` as a whole).
+In practice, their types will be based on the `QFieldType` of the {link-field} that they correspond to.
+That will typically be one of: `String`, `Integer`, `Boolean`, `BigDecimal`, `Instant`, `LocalDate`, `LocalTime`, or `byte[]`.
+Be aware that `null` values may be in the `values` map, especially/per if the backend/table support `null`.
+
+To work with the `values` map, the following methods are provided:
+
+* `setValue(String fieldName, Serializable value)` - Sets a value for the specified field in the record.
+** Overloaded as `setValue(String fieldName, Object value)` - For some cases where the value may not be known to be `Serializable`.
+In this overload, if the value is `null` or `Serializable`, the primary version of `setValue` is called.
+Otherwise, the `value` is passed through `String::valueOf`, and the result is stored.
+** Overloaded as `setValue(QFieldMetaData field, Serializable value)` - which simply defers to the primary version of `setValue`, passing `field.getName()` as the first parameter.
+* `removeValue(String fieldName)` - Remove the given field from the `values` map.
+** Note that in some situations this is an important distinction from having a `null` value in the map (See <<UpdateAction)>>).
+* `setValues(Map<String, Serializable> values)` - Sets the full map of `values`.
+* `getValues()` - Returns the full map of `values`.
+* `getValue(String fieldName)` - Returns the value for the named field - possibly `null` - as a `Serializable`.
+* Several type-specific variations of `getValueX(String fieldName)`, where internally, values will be not exactly type-cast, but effectively converted (if possible) to the requested type.
+These conversions are done using the `ValueUtils.getValueAsX(Object)` methods.
+These methods are generally the preferred/cleanest way to get record values in application code, when it is needed in a type-specific way .
+** `getValueString(String fieldName)`
+** `getValueInteger(String fieldName)`
+** `getValueBoolean(String fieldName)`
+** `getValueBigDecimal(String fieldName)`
+** `getValueInstant(String fieldName)`
+** `getValueLocalDate(String fieldName)`
+** `getValueLocalTime(String fieldName)`
+** `getValueByteArray(String fieldName)`
+
+=== Display Values
+In addition to the `values` map, a `QRecord` contains another map called `displayValues`, which only stores `String` values.
+That is to say, values for other types are stringified, based on their {link-field}'s type and `displayFormat` property.
+In addition, fields which have a `possibleValueSource` property will have their translated values set in the `displayValues` map.
+
+By default, a `QRecord` will not have its `displayValues` populated.
+To populate `displayValues`, the <<QueryAction>> and <<GetAction>> classes take a property in their inputs called `shouldGenerateDisplayValues`, which must be set to `true` to generate `displayValues`.
+In addition, these two actions also have a property `shouldTranslatePossibleValues` in their inputs, which needs to be set to `true` if possible-value lookups are to be performed.
+
+As an alternative to the `shouldGenerateDisplayValues` and `shouldTranslatePossibleValues` inputs to <<QueryAction>> and <<GetAction>>, one can directly call the `QValueFormatter.setDisplayValuesInRecords` and/or `qPossibleValueTranslator.translatePossibleValuesInRecords` methods.
+Or, for special cases, `setDisplayValue(String fieldName, String displayValue)` or `setDisplayValues(Map<String, String> displayValues)` can be called directly.
+
+=== Backend Details
+Sometimes a backend may want to place additional data in a `QRecord` that doesn't exactly correspond to a field.
+To do this, the `Map<String, Serializable> backendDetails` member is used.
+
+For example, an API backend may store the full JSON `String` that came from the API as a backend detail in a `QRecord`.
+Or fields that are marked as `isHeavy`, if the full (heavy) value of the field hasn't been fetched, then the lengths of any such heavy fields may be stored in `backendDetails`.
+
+=== Errors and Warnings
+#todo#
+
+=== Associated Records
+#todo#

docs/misc/RenderingWidgets.adoc

@@ -0,0 +1,224 @@
+== Rendering Widgets
+include::../variables.adoc[]
+
+=== WidgetRenderer classes
+In general, to fully implement a Widget, you must define its `QWidgetMetaData`,
+and supply a subclass of `AbstractWidgetRenderer`, to provide the data to the widget.
+(Note the "No Code" category of widgets, which are an exception to this generalization).
+
+The only method required in a subclass of `AbstractWidgetRenderer` is:
+
+   public RenderWidgetOutput render(RenderWidgetInput input) throws QException
+
+The fields available in `RenderWidgetInput` are:
+
+- `Map<String, String> queryParams` - These are parameters supplied by the frontend, for example,
+if a user selected values from dropdowns to control a dimension of your widget, those name/value
+pairs would be in this map.  Similarly, if your widget is being included on a record view screen, then
+the record's primary key will be in this map.
+- `QWidgetMetaDataInterface widgetMetaData` - This is the meta-data for the widget being
+rendered.  This can be useful in case you are using the same renderer class for multiple widgets.
+
+The only field in `RenderWidgetOutput` is:
+
+- `QWidgetData widgetData` - This is a base class, with several attributes, and more importantly,
+several subclasses, specific to the type of widget that is being rendered.
+
+==== Widget-Type Specific Rendering Details
+Different widget types expect & require different types of values to be set in the `RenderWidgetOutput` by their renderers.
+
+===== Pie Chart
+The `WidgetType.PIE_CHART` requires an object of type `ChartData`.
+The fields on this type are:
+
+* `chartData` an instance of `ChartData.Data`, which has the following fields:
+** `labels` - *List<String>, required* - the labels for the slices of the pie.
+** `datasets` - *List<Dataset> required* - the data for each slice of the pie.
+For a Pie chart, only a single entry in this list is used
+(other chart types using `ChartData` may support more than 1 entry in this list).
+Fields in this object are:
+*** `label` - *String, required* - a label to describe the dataset as a whole.
+e.g., "Orders" for a pie showing orders of different statuses.
+*** `data` - *List<Number>, required* - the data points for each slice of the pie.
+*** `color` - *String* - HTML color for the slice
+*** `urls` - *List<String>* - Optional URLs for slices of the pie to link to when clicked.
+*** `backgroundColors` - *List<String>* - Optional HTML color codes for each slice of the pie.
+
+[source,java]
+.Pie chart widget example
+----
+// meta data
+new QWidgetMetaData()
+    .withName("pieChartExample")
+    .withType(WidgetType.PIE_CHART.getType())
+    .withGridColumns(4)
+    .withIsCard(true)
+    .withLabel("Pie Chart Example")
+    .withCodeReference(new QCodeReference(PieChartExampleRenderer.class));
+
+// renderer
+private List<String> labels = new ArrayList<>();
+private List<String> colors = new ArrayList<>();
+private List<Number> data   = new ArrayList<>();
+
+/*******************************************************************************
+** helper method - to add values for a slice to the lists
+*******************************************************************************/
+private void addSlice(String label, String color, Number datum)
+{
+    labels.add(label);
+    colors.add(color);
+    data.add(datum);
+}
+
+/*******************************************************************************
+** main method of the widget renderer
+*******************************************************************************/
+@Override
+public RenderWidgetOutput render(RenderWidgetInput input) throws QException
+{
+   addSlice("Apple", "#FF0000", 100);
+   addSlice("Orange", "#FF8000", 150);
+   addSlice("Banana", "#FFFF00", 75);
+   addSlice("Lime", "#00FF00", 100);
+   addSlice("Blueberry", "#0000FF", 200);
+
+   ChartData chartData = new ChartData()
+      .withChartData(new ChartData.Data()
+         .withLabels(labels)
+         .withDatasets(List.of(
+            new ChartData.Data.Dataset()
+               .withLabel("Flavor")
+               .withData(data)
+               .withBackgroundColors(colors)
+               .withUrls(urls))));
+
+   return (new RenderWidgetOutput(chartData));
+}
+
+----
+
+===== Bar Chart
+#todo#
+
+===== Stacked Bar Chart
+#todo#
+
+===== Horizontal Bar Chart
+#todo#
+
+===== Child Record List
+#todo#
+
+===== Line Chart
+#todo#
+
+===== Small Line Chart
+#todo#
+
+===== Statistics
+#todo#
+
+===== Parent Widget
+#todo#
+
+===== Composite
+A `WidgetType.COMPOSITE` is built by using one or more smaller elements, known as `Blocks`.
+Note that `Blocks` can also be used in the data of some other widget types
+(specifically, within a cell of a Table-type widget, or (in the future?) as a header above a pie or bar chart).
+
+A composite widget renderer must return data of type `CompositeWidgetData`,
+which has the following fields:
+
+* `blocks` - *List<AbstractBlockWidgetData>, required* - The blocks (1 or more) being composited together to make the widget.
+See below for details on the specific Block types.
+* `styleOverrides` - *Map<String, Serializable>* - Optional map of CSS attributes
+(named following javascriptStyleCamelCase) to apply to the `<div>` element that wraps the rendered blocks.
+* `layout` - *Layout enum* - Optional specifier for how the blocks should be laid out.
+e.g., predefined sets of CSS attributes to achieve specific layouts.
+** Note that some blocks are designed to work within composites with specific layouts.
+Look for matching names, such as `Layout.BADGES_WRAPPER` to go with `NumberIconBadgeBlock`.
+
+[source,java]
+.Composite widget example - consisting of 3 Progress Bar Blocks, and one Divider Block
+----
+// meta data
+new QWidgetMetaData()
+    .withName("compositeExample")
+    .withType(WidgetType.COMPOSITE.getType())
+    .withGridColumns(4)
+    .withIsCard(true)
+    .withLabel("Composite Example")
+    .withCodeReference(new QCodeReference(CompositeExampleRenderer.class));
+
+// renderer
+public RenderWidgetOutput render(RenderWidgetInput input) throws QException
+{
+   CompositeWidgetData data = new CompositeWidgetData();
+
+   data.addBlock(new ProgressBarBlockData()
+      .withValues(new ProgressBarValues()
+         .withHeading("Blocks")
+         .withPercent(new BigDecimal("78.5"))));
+
+   data.addBlock(new ProgressBarBlockData()
+      .withValues(new ProgressBarValues()
+         .withHeading("Progress")
+         .withPercent(new BigDecimal(0))));
+
+   data.addBlock(new DividerBlockData());
+
+   data.addBlock(new ProgressBarBlockData()
+      .withStyles(new ProgressBarStyles().withBarColor("#C0C000"))
+      .withValues(new ProgressBarValues()
+         .withHeading("Custom Color")
+         .withPercent(new BigDecimal("75.3"))));
+
+   return (new RenderWidgetOutput(data));
+}
+----
+
+
+===== Table
+#todo#
+
+===== HTML
+#todo#
+
+===== Divider
+#todo#
+
+===== Process
+#todo#
+
+===== Stepper
+#todo#
+
+===== Data Bag Viewer
+#todo#
+
+===== Script Viewer
+#todo#
+
+=== Block-type Specific Rendering Details
+For Composite-type widgets (or other widgets which can include blocks),
+there are specific data classes required to be returned by the widget renderer.
+
+Each block type defines a subclass of `AbstractBlockWidgetData`,
+which is a generic class with 3 type parameters:
+
+* `V` - an implementation of `BlockValuesInterface` - to define the type of values that the block uses.
+* `S` - an implementation of `BlockSlotsInterface` (expected to be an `enum`) - to define the "slots" in the block,
+that can have Tooltips and/or Links applied to them.
+* `SX` - an implementation of `BlockStylesInterface` - to define the types of style customizations that the block supports.
+
+These type parameters are designed to ensure type-safety for the application developer,
+to ensure that only
+
+=== Additional Tips
+
+* To make a Dashboard page (e.g., an App consisting of Widgets) with a parent widget use the parent widget's label as the page's label:
+** On the `QAppMetaData` that contains the Parent widget, call
+`.withSupplementalMetaData(new MaterialDashboardAppMetaData().withShowAppLabelOnHomeScreen(false))`.
+** In the Parent widget's renderer, on the `ParentWidgetData`, call `setLabel("My Label")` and
+`setIsLabelPageTitle(true)`.

docs/misc/ScheduledJobs.adoc

@@ -0,0 +1,141 @@
+== Schedulers and Scheduled Jobs
+include::../variables.adoc[]
+
+QQQ has the ability to automatically run various types of jobs on schedules,
+either defined in your instance's meta-data,
+or optionally via data in your application, in a `scheduledJob` table.
+
+=== Schedulers and QSchedulerMetaData
+2 types of schedulers are included in QQQ by default (though an application can define its own schedulers):
+
+* `SimpleScheduler` - is (as its name suggests) a simple class which uses java's `ScheduledExecutorService`
+to run jobs on repeating intervals.
+** Cannot run cron schedules - only repeating intervals.
+** If multiple servers are running, each will potentially run the same job concurrently
+** Has no configurations, e.g., to limit the number of threads.
+
+* `QuartzScheduler` - uses the 3rd party https://www.quartz-scheduler.org/[Quartz Scheduler] library to provide
+a much more capable, though admittedly more complex, scheduling solution.
+** Can run both cron schedules and repeating intervals.
+** By default, will not allow concurrent executions of the same job.
+** Supports multiple configurations, e.g., to limit the number of threads.
+
+An application can define its own scheduler by providing a class which implements the `QSchedulerInterface`.
+
+A `QInstance` can work with 0 or more schedulers, as defined by adding `QSchedulerMetaData` objects
+to the instance.
+
+This meta-data class is `abstract`, and is extended by the 2 built-in schedulers
+(e.g., `SimpleSchedulerMetaData` and `QuartzSchedulerMetaData`).  As such,
+these concrete subclasses are what you need to instantiate and add to your instance.
+
+To configure a QuartzScheduler, you can add a `Properties` object to the `QuartzSchedulerMetaData` object.
+See https://www.quartz-scheduler.org/documentation/[Quartz's documentation] for available configuration properties.
+
+[source,java]
+.Defining SchedulerMetaData
+----
+qInstance.addScheduler(new SimpleSchedulerMetaData().withName("mySimpleScheduler"));
+
+qInstance.addScheduler(new QuartzSchedulerMetaData()
+   .withName("myQuartzScheduler")
+   .withProperties(myQuartzProperties);
+----
+
+=== SchedulableTypes
+The types of jobs which can be scheduled in a QQQ application are defined in the `QInstance` by
+instances of the `SchedulableType` meta-data class.
+These objects contain a name, along with a `QCodeReference` to the `runner`,
+which must be a class that implements the `SchedulableRunner` interface.
+
+By default, (in the `QInstanceEnricher`), QQQ will make 3 `SchedulableType` options available:
+
+* `PROCESS` - Any Process defined in the `QInstance` can be scheduled.
+* `QUEUE_PROCESSOR` - A Queue defined in the `QInstance`, which requires polling (e.g., SQS), can be scheduled.
+* `TABLE_AUTOMATIONS` - A Table in the `QInstance`, with `AutomationDetails` referring to an
+AutomationProvider which requires polling, can be scheduled.
+
+If an application only wants to use a subset of these `SchedulableType` options,
+or to add custom `SchedulableType` options,
+the `QInstance` will need to have 1 or more `SchedulableType` objects in it before the `QInstanceEnricher` runs.
+
+=== User-defined Scheduled Jobs
+To allow users to schedule jobs (rather than using programmer-defined schedules (in meta-data)),
+you can add a set of tables to your `QInstance`, using the `ScheduledJobsMetaDataProvider` class:
+
+[source,java]
+.Adding the ScheduledJob tables and related meta-data to a QInstance
+----
+new ScheduledJobsMetaDataProvider().defineAll(
+   qInstance, backendName, table -> tableEnricher(table));
+----
+
+This meta-data provider adds a "scheduledJob" and "scheduledJobParameter" table, along with
+some PossibleValueSources.
+These tables include post-action customizers, which manage (re-, un-) scheduling jobs based on
+changes made to records in this these tables.
+
+Also, when `QScheduleManager` is started, it will query these tables,and will schedule jobs as defined therein.
+
+_You can use a mix of user-defined and meta-data defined scheduled jobs in your instance.
+However, if a ScheduledJob record references a process, queue, or table automation with a
+meta-data defined schedule, the ScheduledJob will NOT be started by ScheduleManager --
+rather, the meta-data definition will "win"._
+
+[source,sql]
+.Example of inserting scheduled jobs records directly into an SQL backend
+----
+-- A process:
+INSERT INTO scheduled_job (label, scheduler_name, cron_expression, cron_time_zone_id, repeat_seconds, type, is_active) VALUES
+   ('myProcess', 'QuartzScheduler', null, null, 300, 'PROCESS', 1);
+INSERT INTO scheduled_job_parameter (scheduled_job_id, `key`, value) VALUES
+   ((SELECT id FROM scheduled_job WHERE label = 'myProcess'), 'processName', 'myProcess');
+
+-- A table's insert & update automations:
+INSERT INTO scheduled_job (label, scheduler_name, cron_expression, cron_time_zone_id, repeat_seconds, type, is_active) VALUES
+   ('myTable.PENDING_INSERT_AUTOMATIONS', 'QuartzScheduler', null, null, 15, 'TABLE_AUTOMATIONS', 1),
+   ('myTable.PENDING_UPDATE_AUTOMATIONS', 'QuartzScheduler', null, null, 15, 'TABLE_AUTOMATIONS', 1);
+INSERT INTO scheduled_job_parameter (scheduled_job_id, `key`, value) VALUES
+   ((SELECT id FROM scheduled_job WHERE label = 'myTable.PENDING_INSERT_AUTOMATIONS'), 'tableName', 'myTable'),
+   ((SELECT id FROM scheduled_job WHERE label = 'myTable.PENDING_INSERT_AUTOMATIONS'), 'automationStatus', 'PENDING_INSERT_AUTOMATIONS'),
+   ((SELECT id FROM scheduled_job WHERE label = 'myTable.PENDING_UPDATE_AUTOMATIONS'), 'tableName', 'myTable'),
+   ((SELECT id FROM scheduled_job WHERE label = 'myTable.PENDING_UPDATE_AUTOMATIONS'), 'automationStatus', 'PENDING_UPDATE_AUTOMATIONS');
+
+-- A queue processor:
+INSERT INTO scheduled_job (label, scheduler_name, cron_expression, cron_time_zone_id, repeat_seconds, type, is_active) VALUES
+   ('mySqsQueue', 'QuartzScheduler', null, null, 60, 'QUEUE_PROCESSOR', 1);
+INSERT INTO scheduled_job_parameter (scheduled_job_id, `key`, value) VALUES
+   ((SELECT id FROM scheduled_job WHERE label = 'mySqsQueue'), 'queueName', 'mySqsQueue');
+----
+
+=== Running Scheduled Jobs
+In a server running QQQ, if you wish to start running scheduled jobs, you need to initialize
+the `QScheduleManger` singleton class, then call its `start()` method.
+
+Note that internally, this class will check for a system property of `qqq.scheduleManager.enabled`
+or an environment variable of `QQQ_SCHEDULE_MANAGER_ENABLED`, and if the first value found is
+`"false"`, then the scheduler will not actually run its jobs (but, in the case of the `QuartzSchdeuler`,
+it will be available for managing scheduled jobs).
+
+The method `QScheduleManager.initInstance` requires 2 parameters:  Your `QInstance`, and a
+`Supplier<QSession>` lambda, which returns the session that will be used for scheduled jobs when they
+are executed.
+
+[source,java]
+.Starting the Schedule Manager service
+----
+QScheduleManager.initInstance(qInstance, () -> systemUserSession).start();
+----
+
+=== Examples
+[source,java]
+.Attach a schedule in meta-data to a Process
+----
+QProcessMetaData myProcess = new QProcessMetaData()
+   // ...
+   .withSchedule(new QScheduleMetaData()
+      .withSchedulerName("myScheduler")
+      .withDescription("Run myProcess every five minutes")
+      .withRepeatSeconds(300))
+----
+

docs/utilities/RecordLookupHelper.adoc

@@ -0,0 +1,272 @@
+== RecordLookupHelper
+include::../variables.adoc[]
+
+`RecordLookupHelper` is a utility class that exists to help you... lookup records :)
+
+OK, I'll try to give a little more context:
+
+=== Motivation 1:  Performance
+One of the most significant performance optimizations that the team behind QQQ has found time and time again,
+is to minimize the number of times you have to perform an I/O operation.
+To just say it more plainly:
+Make fewer calls to your database (or other backend).
+
+This is part of why the DML actions in QQQ (InsertAction, UpdateAction, DeleteAction) are all written to work on multiple records:
+If you've got to insert 1,000 records, the performance difference between doing that as 1,000 SQL INSERT statements vs. just 1 statement cannot be overstated.
+
+Similarly then, for looking up records:
+If we can do 1 round-trip to the database backend - that is - 1 query to fetch _n_ records,
+then in almost all cases it will be significantly faster than doing _n_ queries, one-by-one, for those _n_ records.
+
+The primary reason why `RecordLookupHelper` exists is to help you cut down on the number of times you have to make a round-trip to a backend data store to fetch records within a process.
+
+[sidebar]
+This basically is version of caching, isn't it?
+Take a set of data from "far away" (e.g., database), and bring it "closer" (local or instance variables), for faster access.
+So we may describe this as a "cache" through the rest of this document.
+
+=== Motivation 2:  Convenience
+
+So, given that one wants to try to minimize the number of queries being executed to look up data in a QQQ processes,
+one can certainly do this "by-hand" in each process that they write.
+
+Doing this kind of record caching in a QQQ Process `BackendStep` may be done as:
+
+* Adding a `Map<Integer, QRecord>` as a field in your class.
+* Setting up and running a `QueryAction`, with a filter based on the collection of the keys you need to look up, then iterating over (or streaming) the results into the map field.
+* Getting values out of the map when you need to use them (dealing with missing values as needed).
+
+That's not so bad, but, it does get a little verbose, especially if you're going to have several such caches in your class.
+
+As such, the second reason that `RecordLookupHelper` exists, is to be a reusable and convenient way to do this kind of optimization,
+by providing methods to perform the bulk query & map building operation described above,
+while also providing some convenient methods for accessing such data after it's been fetched.
+In addition, a single instance of `RecordLookupHelper` can provide this service for multiple tables at once
+(e.g., so you don't need to add a field to your class for each type of data that you're trying to cache).
+
+=== Use Cases
+==== Preload records, then access them
+Scenario:
+
+* We're writing a process `BackendStep` that uses `shipment` records as input.
+* We need to know the `order` record associated with each `shipment` (via an `orderId` foreign key), for some business logic that isn't germaine to the explanation of `RecordLookupHelper`.
+* We also to access some field on the `shippingPartner` record assigned to each `shipment`.
+** Note that here, the `shipment` table has a `partnerCode` field, which relates to the `code` unique-key in the `shippingPartner` table.
+** It's also worth mentioning, we only have a handful of `shippingPartner` records in our database, and we never expect to have very many more than that.
+
+[source,java]
+.Example of a process step using a RecordLookupHelper to preload records
+----
+public class MyShipmentProcessStep implements BackendStep
+{
+   // Add a new RecordLookupHelper field, which will "cache" both orders and shippingPartners
+   private RecordLookupHelper recordLookupHelper = new RecordLookupHelper();
+
+   @Override
+   public void run(RunBackendStepInput input, RunBackendStepOutput output) throws QException;
+   {
+      // lookup the full shippingPartner table (because it's cheap & easy to do so)
+      // use the partner's "code" as the key field (e.g,. they key in the helper's internal map).
+      recordLookupHelper.preloadRecords("shippingPartner", "code");
+
+      // get all of the orderIds from the input shipments
+      List<Serializable> orderIds = input.getRecords().stream()
+         .map(r -> r.getValue("id")).toList();
+
+      // fetch the orders related to by these shipments
+      recordLookupHelper.preloadRecords("order", "id", orderIds);
+
+      for(QRecord shipment : input.getRecords())
+      {
+         // get someConfigField from the shippingPartner assigned to the shipment
+         Boolean someConfig = recordLookupHelper.getRecordValue("shippingPartner", "someConfigField", "code", shipment.getValue("partnerCode"));
+
+         // get the order record assigned to the shipment
+         QRecord order = recordLookupHelper.getRecordByKey("order", "id", shipment.getValue("orderId"));
+      }
+   }
+}
+----
+
+==== Lazy fetching records
+Scenario:
+
+* We have a `BackendStep` that is taking in `purchaseOrderHeader` records, from an API partner.
+* For each record, we need to make an API call to the partner to fetch the `purchaseOrderLine` records under that header.
+** In this contrived example, the partner's API forces us to do these lookups order-by-order...
+* Each `purchaseOrderLine` that we fetch will have a `sku` on it - a reference to our `item` table.
+** We need to look up each `item` to apply some business logic.
+** We assume there are very many item records in the backend database, so we don't want to pre-load the full table.
+Also, we don't know what `sku` values we will need until we fetch the `purchaseOrderLine`.
+
+This is a situation where we can use `RecordLookupHelper` to lazily fetch the `item` records as we discover them,
+and it will take care of not re-fetching ones that it has already loaded.
+
+[source,java]
+.Example of a process step using a RecordLookupHelper to lazy fetch records
+----
+public class MyPurchaseOrderProcessStep implements BackendStep
+{
+   // Add a new RecordLookupHelper field, which will "cache" lazy-loaded item records
+   private RecordLookupHelper recordLookupHelper = new RecordLookupHelper();
+
+   @Override
+   public void run(RunBackendStepInput input, RunBackendStepOutput output) throws QException;
+   {
+      for(QRecord poHeader : input.getRecords())
+      {
+         // fetch the lines under the header
+         Serializable poNo = poHeader.getValue("poNo");
+         List<QRecord> poLines = new QueryAction().execute(new QueryInput("purchaseOrderLine")
+            .withFilter(new QQueryFilter(new QFilterCriteria("poNo", EQUALS, poNo))));
+
+         for(QRecord poLine : poLines)
+         {
+            // use recordLookupHelper to lazy-load item records by SKU.
+            QRecord item = recordLookupHelper.getRecordByKey("item", "sku", poLine.getValue("sku"));
+
+            // business logic related to item performed here.
+         }
+      }
+   }
+}
+----
+
+In this example, we will be doing exactly 1 query on the `item` table for each unique `sku` that is found across all of the `poLine` records we process.
+That is to say, if the same `sku` appears on only 1 `poLine`, or if it appears on 100 `poLines`, we will still only query once for that `sku`.
+
+A slight tweak could be made to the example above, to make 1 `item` table lookup for each `poHeader` record:
+
+[source,java]
+.Tweaked example doing 1 item lookup per poLine
+----
+// continuing from above, after the List<QRecord> poLines has been built
+
+// get all of the skus from the lines
+List<Serializable> skus = poLines.stream().map(r -> r.getValue("sku")).toList();
+
+// preload the items for the skus
+recordLookupHelper.preloadRecords("item", "sku", new QQueryFilter(new QFilterCriteria("sku", IN, skus)));
+
+for(QRecord poLine : poLines)
+{
+   // get the items from the helper
+   QRecord item = recordLookupHelper.getRecordByKey("item", "sku", poLine.getValue("sku"));
+
+----
+
+In this example, we've made a trade-off:  We will query the `item` table exactly 1 time for each `poHeader` that we process.
+However, if the same `sku` is on every PO that we process, we will end up fetching it multiple times.
+
+This could end up being better or worse than the previous example, depending on the distribution of the data we are dealing with.
+
+A further tweak, a hybrid approach, could potentially reap the benefits of both of these examples (at the tradeoff of, more code, more complexity):
+
+[source,java]
+.Tweaked example doing 1 item lookup per poLine, but only for not-previously-encountered skus
+----
+// Critically - we must tell our recordLookupHelper to NOT do any one-off lookups in this table
+recordLookupHelper.setMayNotDoOneOffLookups("item", "sku");
+
+// continuing from above, after the List<QRecord> poLines has been built
+
+// get all of the skus from the lines
+List<Serializable> skus = poLines.stream().map(r -> r.getValue("sku")).toList();
+
+// determine which skus have not yet been loaded - e.g., they are not in the recordLookupHelper.
+// this is why we needed to tell it above not to do one-off lookups; else it would lazy-load each sku here.
+List<Serializable> skusToLoad = new ArrayList<>();
+for(Serializable sku : skus)
+{
+   if(recordLookupHelper.getRecordByKey("item", "sku", sku) == null)
+   {
+      skusToLoad.add(sku);
+   }
+}
+
+// preload the item records for any skus that are still needed
+if(!skusToLoad.isEmpty())
+{
+   recordLookupHelper.preloadRecords("item", "sku",
+      new QQueryFilter(new QFilterCriteria("sku", IN, skusToLoad)));
+}
+
+// continue as above
+
+----
+
+In this example, we will start by querying the `item` table once for each `poHeader`, but,
+if we eventually encounter a PO where all of its `skus` have already been loaded, then we may be able to avoid any `item` queries for such a PO.
+
+=== Implementation Details
+
+* Internally, an instance of `RecordLookupHelper` maintains a number of `Maps`,
+with QQQ table names and field names as keys.
+* The accessing/lazy-fetching methods (e.g., any method whose name starts with `getRecord`)
+all begin by looking in these internal maps for the `tableName` and `keyFieldName` that they take as parameters.
+** If they find an entry in the maps, then it is used for producing a return value.
+** If they do not find an entry, then they will perform the a `QueryAction`,
+to try to fetch the requested record from the table's backend.
+*** Unless the `setMayNotDoOneOffLookups` method has been called for the `(tableName, keyFieldName)` pair.
+
+=== Full API
+
+==== Methods for accessing and lazy-fetching
+* `getRecordByKey(String tableName, String keyFieldName, Serializable key)`
+
+Get a `QRecord` from `tableName`, where `keyFieldName` = `key`.
+
+* `getRecordValue(String tableName, String requestedField, String keyFieldName, Serializable key)`
+
+Get the field `requestedField` from the record in `tableName`, where `keyFieldName` = `key`, as a `Serializable`.
+If the record is not found, `null` is returned.
+
+* `getRecordValue(String tableName, String requestedField, String keyFieldName, Serializable key, Class<T> type)`
+
+Get the field `requestedField` from the record in `tableName`, where `keyFieldName` = `key`, as an instance of `type`.
+If the record is not found, `null` is returned.
+
+* `getRecordId(String tableName, String keyFieldName, Serializable key)`
+
+Get the primary key of the record in `tableName`, where `keyFieldName` = `key`, as a `Serializable`.
+If the record is not found, `null` is returned.
+
+* `getRecordId(String tableName, String keyFieldName, Serializable key, Class<T> type)`
+
+Get the primary key of the record in `tableName`, where `keyFieldName` = `key`, as an instance of `type`.
+If the record is not found, `null` is returned.
+
+* `getRecordByUniqueKey(String tableName, Map<String, Serializable> uniqueKey)`
+
+Get a `QRecord` from `tableName`, where the record matches the field/value pairs in `uniqueKey`.
+
+_Note: this method does not use the same internal map as the rest of the class.
+As such, it does not take advantage of any data fetched via the preload methods.
+It is only used for caching lazy-fetches._
+
+==== Methods for preloading
+* `preloadRecords(String tableName, String keyFieldName)`
+
+Query for all records from `tableName`, storing them in an internal map keyed by the field `keyFieldName`.
+
+* `preloadRecords(String tableName, String keyFieldName, QQueryFilter filter)`
+
+Query for records matching `filter` from `tableName`,
+storing them in an internal map keyed by the field `keyFieldName`.
+
+* `preloadRecords(String tableName, String keyFieldName, List<Serializable> inList)`
+
+Query for records with the field `keyFieldName` having a value in `inList` from `tableName`,
+storing them in an internal map keyed by the field `keyFieldName`.
+
+==== Config Methods
+* `setMayNotDoOneOffLookups(String tableName, String fieldName)`
+
+For cases where you know that you have preloaded records for `tableName`, keyed by `fieldName`,
+and you know that some of the keys may not have been found,
+so you want to avoid doing a query when a missed key is found in one of the `getRecord...` methods,
+then if you call this method, an internal flag is set, which will prevent any such one-off lookups.
+
+In other words, if this method has been called for a `(tableName, fieldName)` pair,
+then the `getRecord...` methods will only look in the internal map for records,
+and no queries will be performed to look for records.

docs/variables.adoc

@@ -0,0 +1,25 @@
+:link-app: <<Apps,QQQ App>>
+:link-apps: <<Apps,QQQ Apps>>
+:link-backend: <<Backends,QQQ Backend>>
+:link-backends: <<Backends,QQQ Backends>>
+:link-field: <<Fields,QQQ Field>>
+:link-fields: <<Fields,QQQ Fields>>
+:link-icon: <<Icons,Icon>>
+:link-icons: <<Icons,Icons>>
+:link-instance: <<QInstance,QInstance>>
+:link-join: <<Joins,QQQ Join>>
+:link-joins: <<Joins,QQQ Joins>>
+:link-permissionRule: <<PermissionRules,Permission Rule>>
+:link-permissionRules: <<PermissionRules,Permission Rules>>
+:link-possibleValueSource: <<PossibleValueSources,QQQ Possible Value Source>>
+:link-possibleValueSources: <<PossibleValueSources,QQQ Possible Value Sources>>
+:link-process: <<Processes,QQQ Process>>
+:link-processes: <<Processes,QQQ Processes>>
+:link-report: <<Reports,QQQ Report>>
+:link-reports: <<Reports,QQQ Reports>>
+:link-securityKeyType: <<SecurityKeyTypes,Security Key Type>>
+:link-securityKeyTypes: <<SecurityKeyTypes,Security Key Types>>
+:link-table: <<Tables,QQQ Table>>
+:link-tables: <<Tables,QQQ Tables>>
+:link-widget: <<Widgets,QQQ Widget>>
+:link-widgets: <<Widgets,QQQ Widgets>>

pom.xml

@@ -33,6 +33,7 @@
       <module>qqq-backend-module-api</module>
       <module>qqq-backend-module-filesystem</module>
       <module>qqq-backend-module-rdbms</module>
+      <module>qqq-backend-module-mongodb</module>
       <module>qqq-language-support-javascript</module>
       <module>qqq-middleware-picocli</module>
       <module>qqq-middleware-javalin</module>
@@ -44,7 +45,7 @@
    </modules>
 
    <properties>
-      <revision>0.13.0</revision>
+      <revision>0.20.0-SNAPSHOT</revision>
 
       <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
       <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
@@ -79,12 +80,12 @@
          <dependency>
             <groupId>org.apache.logging.log4j</groupId>
             <artifactId>log4j-api</artifactId>
-            <version>2.17.1</version>
+            <version>2.23.0</version>
          </dependency>
          <dependency>
             <groupId>org.apache.logging.log4j</groupId>
             <artifactId>log4j-core</artifactId>
-            <version>2.17.1</version>
+            <version>2.23.0</version>
          </dependency>
          <dependency>
             <groupId>org.junit.jupiter</groupId>
@@ -108,6 +109,16 @@
    </dependencyManagement>
 
    <build>
+      <resources>
+         <resource>
+            <directory>src/main/java</directory>
+            <filtering>false</filtering>
+         </resource>
+         <resource>
+            <directory>src/main/resources</directory>
+            <filtering>false</filtering>
+         </resource>
+      </resources>
       <plugins>
          <!-- plugins specifically for this module -->
          <!-- none at this time -->
@@ -206,6 +217,63 @@
                <skipUpdateVersion>true</skipUpdateVersion>
             </configuration>
          </plugin>
+         <plugin>
+            <artifactId>exec-maven-plugin</artifactId>
+            <groupId>org.codehaus.mojo</groupId>
+            <version>3.0.0</version>
+            <executions>
+               <execution>
+                  <id>test-coverage-summary</id>
+                  <phase>verify</phase>
+                  <goals>
+                     <goal>exec</goal>
+                  </goals>
+                  <configuration>
+                     <executable>sh</executable>
+                     <arguments>
+                        <argument>-c</argument>
+                        <argument>
+                           <![CDATA[
+if [ ! -e target/site/jacoco/index.html ]; then
+   echo "No jacoco coverage report here.";
+   exit;
+fi
+
+echo
+echo "Jacoco coverage summary report:"
+echo "   See also target/site/jacoco/index.html"
+echo "   and https://www.jacoco.org/jacoco/trunk/doc/counters.html"
+echo "------------------------------------------------------------"
+
+which xpath > /dev/null 2>&1
+if [ "$?" == "0" ]; then
+   echo "Element\nInstructions Missed\nInstruction Coverage\nBranches Missed\nBranch Coverage\nComplexity Missed\nComplexity Hit\nLines Missed\nLines Hit\nMethods Missed\nMethods Hit\nClasses Missed\nClasses Hit\n" > /tmp/$$.headers
+   xpath -n -q -e '/html/body/table/tfoot/tr[1]/td/text()' target/site/jacoco/index.html > /tmp/$$.values
+   paste /tmp/$$.headers /tmp/$$.values | tail +2 | awk -v FS='\t' '{printf("%-20s %s\n",$1,$2)}'
+   rm /tmp/$$.headers /tmp/$$.values
+else
+   echo "xpath is not installed.  Jacoco coverage summary will not be produced here...";
+fi
+
+which xpath > /dev/null 2>&1
+if [ "$?" == "0" ]; then
+   echo "Untested classes, per Jacoco:"
+   echo "-----------------------------"
+   for i in target/site/jacoco/*/index.html; do
+      html2text -width 500 -nobs $i | sed '1,/^Total/d;' | grep -v Created | sed 's/ \+/ /g' | sed 's/ [[:digit:]]$//' | grep -v 0$ | cut -d' ' -f1;
+   done;
+   echo
+else
+   echo "html2text is not installed.  Untested classes from Jacoco will not be printed here...";
+fi
+
+                           ]]>
+                        </argument>
+                     </arguments>
+                  </configuration>
+               </execution>
+            </executions>
+         </plugin>
          <plugin>
             <groupId>org.jacoco</groupId>
             <artifactId>jacoco-maven-plugin</artifactId>
@@ -249,59 +317,43 @@
                </execution>
                <execution>
                   <id>post-unit-test</id>
-                  <phase>verify</phase>
+                  <!-- <phase>verify</phase> -->
+                  <phase>post-integration-test</phase>
                   <goals>
                      <goal>report</goal>
                   </goals>
                </execution>
             </executions>
          </plugin>
-         <plugin>
-            <artifactId>exec-maven-plugin</artifactId>
-            <groupId>org.codehaus.mojo</groupId>
-            <version>3.0.0</version>
-            <executions>
-               <execution>
-                  <id>test-coverage-summary</id>
-                  <phase>verify</phase>
-                  <goals>
-                     <goal>exec</goal>
-                  </goals>
-                  <configuration>
-                     <executable>sh</executable>
-                     <arguments>
-                        <argument>-c</argument>
-                        <argument>
-                           <![CDATA[
-if [ ! -e target/site/jacoco/index.html ]; then
-   echo "No jacoco coverage report here.";
-   exit;
-fi
-
-echo
-echo "Jacoco coverage summary report:"
-echo "   See also target/site/jacoco/index.html"
-echo "   and https://www.jacoco.org/jacoco/trunk/doc/counters.html"
-echo "------------------------------------------------------------"
-which xpath > /dev/null 2>&1
-if [ "$?" == "0" ]; then
-   echo "Element\nInstructions Missed\nInstruction Coverage\nBranches Missed\nBranch Coverage\nComplexity Missed\nComplexity Hit\nLines Missed\nLines Hit\nMethods Missed\nMethods Hit\nClasses Missed\nClasses Hit\n" > /tmp/$$.headers
-   xpath -n -q -e '/html/body/table/tfoot/tr[1]/td/text()' target/site/jacoco/index.html > /tmp/$$.values
-   paste /tmp/$$.headers /tmp/$$.values | tail +2 | awk -v FS='\t' '{printf("%-20s %s\n",$1,$2)}'
-   rm /tmp/$$.headers /tmp/$$.values
-else
-   echo "xpath is not installed.  Jacoco coverage summary will not be produced here..";
-fi
-                           ]]>
-                        </argument>
-                     </arguments>
-                  </configuration>
-               </execution>
-            </executions>
-         </plugin>
       </plugins>
    </build>
 
+   <!-- mvn javadoc:aggregate -->
+   <reporting>
+      <plugins>
+         <plugin>
+            <groupId>org.apache.maven.plugins</groupId>
+            <artifactId>maven-javadoc-plugin</artifactId>
+            <version>3.6.2</version>
+            <reportSets>
+               <reportSet>
+                  <id>aggregate</id>
+                  <inherited>false</inherited>
+                  <reports>
+                     <report>aggregate</report>
+                  </reports>
+               </reportSet>
+               <reportSet>
+                  <id>default</id>
+                  <reports>
+                     <report>javadoc</report>
+                  </reports>
+               </reportSet>
+            </reportSets>
+         </plugin>
+      </plugins>
+   </reporting>
+
    <repositories>
       <repository>
          <id>github-qqq-maven-registry</id>

qqq-backend-core/pom.xml

@@ -56,6 +56,10 @@
          <groupId>software.amazon.awssdk</groupId>
          <artifactId>quicksight</artifactId>
       </dependency>
+      <dependency>
+         <groupId>software.amazon.awssdk</groupId>
+         <artifactId>apigateway</artifactId>
+      </dependency>
       <dependency>
          <groupId>com.amazonaws</groupId>
          <artifactId>aws-java-sdk-secretsmanager</artifactId>
@@ -80,7 +84,7 @@
       <dependency>
          <groupId>org.json</groupId>
          <artifactId>json</artifactId>
-         <version>20230227</version>
+         <version>20231013</version>
       </dependency>
       <dependency>
          <groupId>org.apache.commons</groupId>
@@ -100,8 +104,18 @@
       </dependency>
       <dependency>
          <groupId>com.auth0</groupId>
-         <artifactId>mvc-auth-commons</artifactId>
-         <version>1.9.2</version>
+         <artifactId>auth0</artifactId>
+         <version>2.1.0</version>
+      </dependency>
+      <dependency>
+         <groupId>com.auth0</groupId>
+         <artifactId>java-jwt</artifactId>
+         <version>4.4.0</version>
+      </dependency>
+      <dependency>
+         <groupId>com.auth0</groupId>
+         <artifactId>jwks-rsa</artifactId>
+         <version>0.22.0</version>
       </dependency>
       <dependency>
          <groupId>io.github.cdimascio</groupId>
@@ -149,6 +163,19 @@
          <version>1.12.321</version>
       </dependency>
 
+      <dependency>
+         <groupId>org.quartz-scheduler</groupId>
+         <artifactId>quartz</artifactId>
+         <version>2.3.2</version>
+      </dependency>
+
+      <!-- Many of the deps we bring in use slf4j.  This dep maps slf4j to our logger, log4j -->
+      <dependency>
+         <groupId>org.apache.logging.log4j</groupId>
+         <artifactId>log4j-slf4j-impl</artifactId>
+         <version>2.23.0</version>
+      </dependency>
+
       <!-- Common deps for all qqq modules -->
       <dependency>
          <groupId>org.apache.maven.plugins</groupId>

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/AbstractQActionBiConsumer.java

@@ -23,7 +23,6 @@ package com.kingsrook.qqq.backend.core.actions;
 
 
 import java.util.concurrent.CompletableFuture;
-import java.util.concurrent.Executors;
 import java.util.concurrent.Future;
 import com.kingsrook.qqq.backend.core.context.CapturedContext;
 import com.kingsrook.qqq.backend.core.context.QContext;
@@ -54,7 +53,7 @@ public abstract class AbstractQActionBiConsumer<I extends AbstractActionInput, O
    {
       CapturedContext         capturedContext   = QContext.capture();
       CompletableFuture<Void> completableFuture = new CompletableFuture<>();
-      Executors.newCachedThreadPool().submit(() ->
+      ActionHelper.getExecutorService().submit(() ->
       {
          try
          {

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/AbstractQActionFunction.java

@@ -23,7 +23,6 @@ package com.kingsrook.qqq.backend.core.actions;
 
 
 import java.util.concurrent.CompletableFuture;
-import java.util.concurrent.Executors;
 import java.util.concurrent.Future;
 import com.kingsrook.qqq.backend.core.context.CapturedContext;
 import com.kingsrook.qqq.backend.core.context.QContext;
@@ -54,7 +53,7 @@ public abstract class AbstractQActionFunction<I extends AbstractActionInput, O e
    {
       CapturedContext      capturedContext   = QContext.capture();
       CompletableFuture<O> completableFuture = new CompletableFuture<>();
-      Executors.newCachedThreadPool().submit(() ->
+      ActionHelper.getExecutorService().submit(() ->
       {
          try
          {

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/ActionHelper.java

@@ -24,6 +24,10 @@ package com.kingsrook.qqq.backend.core.actions;
 
 import java.io.Serializable;
 import java.util.List;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.SynchronousQueue;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
 import java.util.function.Function;
 import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.exceptions.QAuthenticationException;
@@ -40,6 +44,20 @@ import com.kingsrook.qqq.backend.core.modules.authentication.QAuthenticationModu
  *******************************************************************************/
 public class ActionHelper
 {
+   /////////////////////////////////////////////////////////////////////////////
+   // we would probably use Executors.newCachedThreadPool() - but - it has no //
+   // maxPoolSize...  we think some limit is good, so that at a large number  //
+   // of attempted concurrent jobs we'll have new jobs block, rather than     //
+   // exhausting all server resources and locking up "everything"             //
+   // also, it seems like keeping a handful of core-threads around is very    //
+   // little actual waste, and better than ever wasting time starting a new   //
+   // one, which we know we'll often be doing.                                //
+   /////////////////////////////////////////////////////////////////////////////
+   private static Integer         CORE_THREADS    = 8;
+   private static Integer         MAX_THREADS     = 500;
+   private static ExecutorService executorService = new ThreadPoolExecutor(CORE_THREADS, MAX_THREADS, 60L, TimeUnit.SECONDS, new SynchronousQueue<>());
+
+
 
    /*******************************************************************************
     **
@@ -69,6 +87,17 @@ public class ActionHelper
 
 
 
+   /*******************************************************************************
+    ** access an executor service for sharing among the executeAsync methods of all
+    ** actions.
+    *******************************************************************************/
+   static ExecutorService getExecutorService()
+   {
+      return (executorService);
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/QBackendTransaction.java

@@ -23,6 +23,9 @@ package com.kingsrook.qqq.backend.core.actions;
 
 
 import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.actions.AbstractTableActionInput;
+import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleDispatcher;
+import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleInterface;
 
 
 /*******************************************************************************
@@ -30,12 +33,26 @@ import com.kingsrook.qqq.backend.core.exceptions.QException;
  ** part of a transaction.
  **
  ** Most obvious use-case would be a JDBC Connection.  See subclass in rdbms module.
+ ** Ditto MongoDB.
  **
  ** Note:  One would imagine that this class shouldn't ever implement Serializable...
  *******************************************************************************/
 public class QBackendTransaction
 {
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public static QBackendTransaction openFor(AbstractTableActionInput input) throws QException
+   {
+      QBackendModuleDispatcher qBackendModuleDispatcher = new QBackendModuleDispatcher();
+      QBackendModuleInterface  qModule                  = qBackendModuleDispatcher.getQBackendModule(input.getBackend());
+      QBackendTransaction      transaction              = qModule.openTransaction(input);
+      return (transaction);
+   }
+
+
+
    /*******************************************************************************
     ** Commit the transaction.
     *******************************************************************************/

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/async/AsyncJobManager.java

@@ -28,16 +28,22 @@ import java.util.UUID;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.CompletionException;
 import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.SynchronousQueue;
+import java.util.concurrent.ThreadPoolExecutor;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.TimeoutException;
 import com.kingsrook.qqq.backend.core.context.CapturedContext;
 import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.exceptions.QUserFacingException;
 import com.kingsrook.qqq.backend.core.logging.QLogger;
 import com.kingsrook.qqq.backend.core.state.InMemoryStateProvider;
 import com.kingsrook.qqq.backend.core.state.StateProviderInterface;
 import com.kingsrook.qqq.backend.core.state.StateType;
 import com.kingsrook.qqq.backend.core.state.UUIDAndTypeStateKey;
+import com.kingsrook.qqq.backend.core.utils.StringUtils;
+import org.apache.logging.log4j.Level;
 import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
 
 
@@ -48,6 +54,22 @@ public class AsyncJobManager
 {
    private static final QLogger LOG = QLogger.getLogger(AsyncJobManager.class);
 
+   /////////////////////////////////////////////////////////////////////////////
+   // we would probably use Executors.newCachedThreadPool() - but - it has no //
+   // maxPoolSize...  we think some limit is good, so that at a large number  //
+   // of attempted concurrent jobs we'll have new jobs block, rather than     //
+   // exhausting all server resources and locking up "everything"             //
+   // also, it seems like keeping a handful of core-threads around is very    //
+   // little actual waste, and better than ever wasting time starting a new   //
+   // one, which we know we'll often be doing.                                //
+   /////////////////////////////////////////////////////////////////////////////
+   private static Integer         CORE_THREADS    = 8;
+   private static Integer         MAX_THREADS     = 500;
+   private static ExecutorService executorService = new ThreadPoolExecutor(CORE_THREADS, MAX_THREADS, 60L, TimeUnit.SECONDS, new SynchronousQueue<>());
+
+
+   private String forcedJobUUID = null;
+
 
 
    /*******************************************************************************
@@ -67,7 +89,8 @@ public class AsyncJobManager
     *******************************************************************************/
    public <T extends Serializable> T startJob(String jobName, long timeout, TimeUnit timeUnit, AsyncJob<T> asyncJob) throws JobGoingAsyncException, QException
    {
-      UUIDAndTypeStateKey uuidAndTypeStateKey = new UUIDAndTypeStateKey(UUID.randomUUID(), StateType.ASYNC_JOB_STATUS);
+      UUID                jobUUID             = StringUtils.hasContent(forcedJobUUID) ? UUID.fromString(forcedJobUUID) : UUID.randomUUID();
+      UUIDAndTypeStateKey uuidAndTypeStateKey = new UUIDAndTypeStateKey(jobUUID, StateType.ASYNC_JOB_STATUS);
       AsyncJobStatus      asyncJobStatus      = new AsyncJobStatus();
       asyncJobStatus.setState(AsyncJobState.RUNNING);
       getStateProvider().put(uuidAndTypeStateKey, asyncJobStatus);
@@ -79,7 +102,7 @@ public class AsyncJobManager
          {
             QContext.init(capturedContext);
             return (runAsyncJob(jobName, asyncJob, uuidAndTypeStateKey, asyncJobStatus));
-         });
+         }, executorService);
 
          if(timeout == 0)
          {
@@ -136,7 +159,7 @@ public class AsyncJobManager
    private <T extends Serializable> T runAsyncJob(String jobName, AsyncJob<T> asyncJob, UUIDAndTypeStateKey uuidAndTypeStateKey, AsyncJobStatus asyncJobStatus)
    {
       String originalThreadName = Thread.currentThread().getName();
-      Thread.currentThread().setName("Job:" + jobName + ":" + uuidAndTypeStateKey.getUuid().toString().substring(0, 8));
+      Thread.currentThread().setName("Job:" + jobName);
       try
       {
          LOG.debug("Starting job " + uuidAndTypeStateKey.getUuid());
@@ -151,7 +174,11 @@ public class AsyncJobManager
          asyncJobStatus.setState(AsyncJobState.ERROR);
          asyncJobStatus.setCaughtException(e);
          getStateProvider().put(uuidAndTypeStateKey, asyncJobStatus);
-         LOG.warn("Job ended with an exception", e, logPair("jobId", uuidAndTypeStateKey.getUuid()));
+
+         //////////////////////////////////////////////////////
+         // if user facing, just log an info, warn otherwise //
+         //////////////////////////////////////////////////////
+         LOG.log((e instanceof QUserFacingException) ? Level.INFO : Level.WARN, "Job ended with an exception", e, logPair("jobId", uuidAndTypeStateKey.getUuid()));
          throw (new CompletionException(e));
       }
       finally
@@ -199,4 +226,35 @@ public class AsyncJobManager
       jobStatus.ifPresent(asyncJobStatus -> asyncJobStatus.setCancelRequested(true));
    }
 
+
+
+   /*******************************************************************************
+    ** Getter for forcedJobUUID
+    *******************************************************************************/
+   public String getForcedJobUUID()
+   {
+      return (this.forcedJobUUID);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for forcedJobUUID
+    *******************************************************************************/
+   public void setForcedJobUUID(String forcedJobUUID)
+   {
+      this.forcedJobUUID = forcedJobUUID;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for forcedJobUUID
+    *******************************************************************************/
+   public AsyncJobManager withForcedJobUUID(String forcedJobUUID)
+   {
+      this.forcedJobUUID = forcedJobUUID;
+      return (this);
+   }
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/async/AsyncRecordPipeLoop.java

@@ -25,6 +25,7 @@ package com.kingsrook.qqq.backend.core.actions.async;
 import java.io.Serializable;
 import java.util.Optional;
 import java.util.concurrent.TimeUnit;
+import com.kingsrook.qqq.backend.core.actions.reporting.BufferedRecordPipe;
 import com.kingsrook.qqq.backend.core.actions.reporting.RecordPipe;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.logging.QLogger;
@@ -82,6 +83,15 @@ public class AsyncRecordPipeLoop
       long    jobStartTime          = System.currentTimeMillis();
       boolean everCalledConsumer    = false;
 
+      ////////////////////////////////////////////////////////////////////////////
+      // in case the pipe capacity has been made very small (useful in tests!), //
+      // then make the minRecordsToConsume match it.                            //
+      ////////////////////////////////////////////////////////////////////////////
+      if(recordPipe.getCapacity() < minRecordsToConsume)
+      {
+         minRecordsToConsume = recordPipe.getCapacity();
+      }
+
       while(jobState.equals(AsyncJobState.RUNNING))
       {
          if(recordPipe.countAvailableRecords() < minRecordsToConsume)
@@ -142,6 +152,11 @@ public class AsyncRecordPipeLoop
          jobState = asyncJobStatus.getState();
       }
 
+      if(recordPipe instanceof BufferedRecordPipe bufferedRecordPipe)
+      {
+         bufferedRecordPipe.finalFlush();
+      }
+
       LOG.debug("Job [" + jobUUID + "][" + jobName + "] completed with status: " + asyncJobStatus);
 
       ///////////////////////////////////

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/audits/AuditAction.java

@@ -29,6 +29,7 @@ import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
+import java.util.Optional;
 import com.kingsrook.qqq.backend.core.actions.AbstractQActionFunction;
 import com.kingsrook.qqq.backend.core.actions.tables.GetAction;
 import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
@@ -44,10 +45,12 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertOutput;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
 import com.kingsrook.qqq.backend.core.model.metadata.security.RecordSecurityLock;
+import com.kingsrook.qqq.backend.core.model.metadata.security.RecordSecurityLockFilters;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 import com.kingsrook.qqq.backend.core.model.session.QUser;
 import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
 import com.kingsrook.qqq.backend.core.utils.Pair;
+import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
 
 
 /*******************************************************************************
@@ -76,6 +79,21 @@ public class AuditAction extends AbstractQActionFunction<AuditInput, AuditOutput
 
 
 
+   /*******************************************************************************
+    ** Execute to insert 1 audit, with a list of detail child records provided as just string messages
+    *******************************************************************************/
+   public static void executeWithStringDetails(String tableName, Integer recordId, Map<String, Serializable> securityKeyValues, String message, List<String> detailMessages)
+   {
+      List<QRecord> detailRecords = null;
+      if(CollectionUtils.nullSafeHasContents(detailMessages))
+      {
+         detailRecords = detailMessages.stream().map(m -> new QRecord().withValue("message", m)).toList();
+      }
+      execute(tableName, recordId, securityKeyValues, message, detailRecords);
+   }
+
+
+
    /*******************************************************************************
     ** Execute to insert 1 audit, with a list of detail child records
     *******************************************************************************/
@@ -92,12 +110,26 @@ public class AuditAction extends AbstractQActionFunction<AuditInput, AuditOutput
 
 
 
+   /*******************************************************************************
+    ** Simple overload that internally figures out primary key and security key values
+    **
+    ** Be aware - if the record doesn't have its security key values set (say it's a
+    ** partial record as part of an update), then those values won't be in the
+    ** security key map...  This should probably be considered a bug.
+    *******************************************************************************/
+   public static void appendToInput(AuditInput auditInput, QTableMetaData table, QRecord record, String auditMessage)
+   {
+      appendToInput(auditInput, table.getName(), record.getValueInteger(table.getPrimaryKeyField()), getRecordSecurityKeyValues(table, record, Optional.empty()), auditMessage);
+   }
+
+
+
    /*******************************************************************************
     ** Add 1 auditSingleInput to an AuditInput object - with no details (child records).
     *******************************************************************************/
-   public static AuditInput appendToInput(AuditInput auditInput, String tableName, Integer recordId, Map<String, Serializable> securityKeyValues, String message)
+   public static void appendToInput(AuditInput auditInput, String tableName, Integer recordId, Map<String, Serializable> securityKeyValues, String message)
    {
-      return (appendToInput(auditInput, tableName, recordId, securityKeyValues, message, null));
+      appendToInput(auditInput, tableName, recordId, securityKeyValues, message, null);
    }
 
 
@@ -123,6 +155,44 @@ public class AuditAction extends AbstractQActionFunction<AuditInput, AuditOutput
 
 
 
+   /*******************************************************************************
+    ** For a given record, from a given table, build a map of the record's security
+    ** key values.
+    **
+    ** If, in case, the record has null value(s), and the oldRecord is given (e.g.,
+    ** for the case of an update, where the record may not have all fields set, and
+    ** oldRecord should be known for doing field-diffs), then try to get the value(s)
+    ** from oldRecord.
+    **
+    ** Currently, will leave values null if they aren't found after that.
+    **
+    ** An alternative could be to re-fetch the record from its source if needed...
+    *******************************************************************************/
+   public static Map<String, Serializable> getRecordSecurityKeyValues(QTableMetaData table, QRecord record, Optional<QRecord> oldRecord)
+   {
+      Map<String, Serializable> securityKeyValues = new HashMap<>();
+      for(RecordSecurityLock recordSecurityLock : RecordSecurityLockFilters.filterForReadLocks(CollectionUtils.nonNullList(table.getRecordSecurityLocks())))
+      {
+         Serializable keyValue = record == null ? null : record.getValue(recordSecurityLock.getFieldName());
+
+         if(keyValue == null && oldRecord.isPresent())
+         {
+            LOG.debug("Table with a securityLock, but value not found in field", logPair("table", table.getName()), logPair("field", recordSecurityLock.getFieldName()));
+            keyValue = oldRecord.get().getValue(recordSecurityLock.getFieldName());
+         }
+
+         if(keyValue == null)
+         {
+            LOG.debug("Table with a securityLock, but value not found in field", logPair("table", table.getName()), logPair("field", recordSecurityLock.getFieldName()), logPair("oldRecordIsPresent", oldRecord.isPresent()));
+         }
+
+         securityKeyValues.put(recordSecurityLock.getSecurityKeyType(), keyValue);
+      }
+      return securityKeyValues;
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/
@@ -151,7 +221,7 @@ public class AuditAction extends AbstractQActionFunction<AuditInput, AuditOutput
                ///////////////////////////////////////////////////
                // validate security keys on the table are given //
                ///////////////////////////////////////////////////
-               for(RecordSecurityLock recordSecurityLock : CollectionUtils.nonNullList(table.getRecordSecurityLocks()))
+               for(RecordSecurityLock recordSecurityLock : RecordSecurityLockFilters.filterForReadLocks(CollectionUtils.nonNullList(table.getRecordSecurityLocks())))
                {
                   if(auditSingleInput.getSecurityKeyValues() == null || !auditSingleInput.getSecurityKeyValues().containsKey(recordSecurityLock.getSecurityKeyType()))
                   {
@@ -215,14 +285,17 @@ public class AuditAction extends AbstractQActionFunction<AuditInput, AuditOutput
                }
             }
 
-            insertInput = new InsertInput();
-            insertInput.setTableName("auditDetail");
-            insertInput.setRecords(auditDetailRecords);
-            new InsertAction().execute(insertInput);
+            if(!auditDetailRecords.isEmpty())
+            {
+               insertInput = new InsertInput();
+               insertInput.setTableName("auditDetail");
+               insertInput.setRecords(auditDetailRecords);
+               new InsertAction().execute(insertInput);
+            }
          }
          catch(Exception e)
          {
-            LOG.error("Error performing an audit", e);
+            LOG.warn("Error performing an audit", e);
          }
       }
 

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/audits/DMLAuditAction.java

@@ -23,15 +23,19 @@ package com.kingsrook.qqq.backend.core.actions.audits;
 
 
 import java.io.Serializable;
+import java.math.BigDecimal;
 import java.time.Instant;
 import java.time.ZoneId;
+import java.time.temporal.ChronoUnit;
 import java.util.ArrayList;
 import java.util.Comparator;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
 import java.util.Optional;
+import java.util.Set;
 import com.kingsrook.qqq.backend.core.actions.AbstractQActionFunction;
 import com.kingsrook.qqq.backend.core.actions.values.QPossibleValueTranslator;
 import com.kingsrook.qqq.backend.core.actions.values.QValueFormatter;
@@ -52,12 +56,12 @@ import com.kingsrook.qqq.backend.core.model.metadata.audits.AuditLevel;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
 import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
-import com.kingsrook.qqq.backend.core.model.metadata.security.RecordSecurityLock;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 import com.kingsrook.qqq.backend.core.model.session.QSession;
 import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
 import com.kingsrook.qqq.backend.core.utils.StringUtils;
 import com.kingsrook.qqq.backend.core.utils.ValueUtils;
+import static com.kingsrook.qqq.backend.core.actions.audits.AuditAction.getRecordSecurityKeyValues;
 import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
 
 
@@ -69,6 +73,10 @@ public class DMLAuditAction extends AbstractQActionFunction<DMLAuditInput, DMLAu
 {
    private static final QLogger LOG = QLogger.getLogger(DMLAuditAction.class);
 
+   public static final String AUDIT_CONTEXT_FIELD_NAME = "auditContext";
+
+   private static Set<String> loggedUnauditableTableNames = new HashSet<>();
+
 
 
    /*******************************************************************************
@@ -79,7 +87,6 @@ public class DMLAuditAction extends AbstractQActionFunction<DMLAuditInput, DMLAu
    {
       DMLAuditOutput           output           = new DMLAuditOutput();
       AbstractTableActionInput tableActionInput = input.getTableActionInput();
-      List<QRecord>            recordList       = input.getRecordList();
       List<QRecord>            oldRecordList    = input.getOldRecordList();
       QTableMetaData           table            = tableActionInput.getTable();
       long                     start            = System.currentTimeMillis();
@@ -87,6 +94,9 @@ public class DMLAuditAction extends AbstractQActionFunction<DMLAuditInput, DMLAu
 
       try
       {
+         List<QRecord> recordList = CollectionUtils.nonNullList(input.getRecordList()).stream()
+            .filter(r -> CollectionUtils.nullSafeIsEmpty(r.getErrors())).toList();
+
          AuditLevel auditLevel = getAuditLevel(tableActionInput);
          if(auditLevel == null || auditLevel.equals(AuditLevel.NONE) || CollectionUtils.nullSafeIsEmpty(recordList))
          {
@@ -96,29 +106,22 @@ public class DMLAuditAction extends AbstractQActionFunction<DMLAuditInput, DMLAu
             return (output);
          }
 
-         String                        contextSuffix = "";
-         Optional<AbstractActionInput> actionInput   = QContext.getFirstActionInStack();
-         if(actionInput.isPresent() && actionInput.get() instanceof RunProcessInput runProcessInput)
+         ///////////////////////////////////////////////////////////////////////////////////////////////////////
+         // currently, the table's primary key must be integer... so, log (once) and return early if not that //
+         // (or, if no primary key!)                                                                          //
+         ///////////////////////////////////////////////////////////////////////////////////////////////////////
+         QFieldMetaData field = table.getFields().get(table.getPrimaryKeyField());
+         if(field == null || !QFieldType.INTEGER.equals(field.getType()))
          {
-            String           processName = runProcessInput.getProcessName();
-            QProcessMetaData process     = QContext.getQInstance().getProcess(processName);
-            if(process != null)
+            if(!loggedUnauditableTableNames.contains(table.getName()))
             {
-               contextSuffix = " during process: " + process.getLabel();
+               LOG.info("Cannot audit table without integer as its primary key", logPair("tableName", table.getName()));
+               loggedUnauditableTableNames.add(table.getName());
             }
+            return (output);
          }
 
-         QSession qSession   = QContext.getQSession();
-         String   apiVersion = qSession.getValue("apiVersion");
-         if(apiVersion != null)
-         {
-            String apiLabel = qSession.getValue("apiLabel");
-            if(!StringUtils.hasContent(apiLabel))
-            {
-               apiLabel = "API";
-            }
-            contextSuffix += (" via " + apiLabel + " Version: " + apiVersion);
-         }
+         String contextSuffix = getContentSuffix(input);
 
          AuditInput auditInput = new AuditInput();
          if(auditLevel.equals(AuditLevel.RECORD) || (auditLevel.equals(AuditLevel.FIELD) && !dmlType.supportsFields))
@@ -129,7 +132,7 @@ public class DMLAuditAction extends AbstractQActionFunction<DMLAuditInput, DMLAu
             ////////////////////////////////////////////////////////////////////////////////////////////////////////////
             for(QRecord record : recordList)
             {
-               AuditAction.appendToInput(auditInput, table.getName(), record.getValueInteger(table.getPrimaryKeyField()), getRecordSecurityKeyValues(table, record), "Record was " + dmlType.pastTenseVerb + contextSuffix);
+               AuditAction.appendToInput(auditInput, table.getName(), record.getValueInteger(table.getPrimaryKeyField()), getRecordSecurityKeyValues(table, record, Optional.empty()), "Record was " + dmlType.pastTenseVerb + contextSuffix);
             }
          }
          else if(auditLevel.equals(AuditLevel.FIELD))
@@ -139,7 +142,7 @@ public class DMLAuditAction extends AbstractQActionFunction<DMLAuditInput, DMLAu
             ///////////////////////////////////////////////////////////////////
             // do many audits, all with field level details, for FIELD level //
             ///////////////////////////////////////////////////////////////////
-            QPossibleValueTranslator qPossibleValueTranslator = new QPossibleValueTranslator(QContext.getQInstance(), qSession);
+            QPossibleValueTranslator qPossibleValueTranslator = new QPossibleValueTranslator(QContext.getQInstance(), QContext.getQSession());
             qPossibleValueTranslator.translatePossibleValuesInRecords(table, CollectionUtils.mergeLists(recordList, oldRecordList));
 
             //////////////////////////////////////////
@@ -161,67 +164,8 @@ public class DMLAuditAction extends AbstractQActionFunction<DMLAuditInput, DMLAu
                List<QRecord> details = new ArrayList<>();
                for(String fieldName : sortedFieldNames)
                {
-                  if(!record.getValues().containsKey(fieldName))
-                  {
-                     ////////////////////////////////////////////////////////////////////////////////////////////////
-                     // if the stored record doesn't have this field name, then don't audit anything about it      //
-                     // this is to deal with our Patch style updates not looking like every field was cleared out. //
-                     ////////////////////////////////////////////////////////////////////////////////////////////////
-                     continue;
-                  }
-
-                  if(fieldName.equals("modifyDate") || fieldName.equals("createDate") || fieldName.equals("automationStatus"))
-                  {
-                     continue;
-                  }
-
-                  QFieldMetaData field        = table.getField(fieldName);
-                  Serializable   value        = ValueUtils.getValueAsFieldType(field.getType(), record.getValue(fieldName));
-                  Serializable   oldValue     = oldRecord == null ? null : ValueUtils.getValueAsFieldType(field.getType(), oldRecord.getValue(fieldName));
-                  QRecord        detailRecord = null;
-
-                  if(oldRecord == null)
-                  {
-                     if(DMLType.INSERT.equals(dmlType) && value == null)
-                     {
-                        continue;
-                     }
-
-                     String formattedValue = getFormattedValueForAuditDetail(record, fieldName, field, value);
-                     detailRecord = new QRecord().withValue("message", "Set " + field.getLabel() + " to " + formattedValue);
-                     detailRecord.withValue("newValue", formattedValue);
-                  }
-                  else
-                  {
-                     if(!Objects.equals(oldValue, value))
-                     {
-                        String formattedValue    = getFormattedValueForAuditDetail(record, fieldName, field, value);
-                        String formattedOldValue = getFormattedValueForAuditDetail(oldRecord, fieldName, field, oldValue);
-
-                        if(oldValue == null)
-                        {
-                           detailRecord = new QRecord().withValue("message", "Set " + field.getLabel() + " to " + formatFormattedValueForDetailMessage(field, formattedValue));
-                           detailRecord.withValue("newValue", formattedValue);
-                        }
-                        else if(value == null)
-                        {
-                           detailRecord = new QRecord().withValue("message", "Removed " + formatFormattedValueForDetailMessage(field, formattedOldValue) + " from " + field.getLabel());
-                           detailRecord.withValue("oldValue", formattedOldValue);
-                        }
-                        else
-                        {
-                           detailRecord = new QRecord().withValue("message", "Changed " + field.getLabel() + " from " + formatFormattedValueForDetailMessage(field, formattedOldValue) + " to " + formatFormattedValueForDetailMessage(field, formattedValue));
-                           detailRecord.withValue("oldValue", formattedOldValue);
-                           detailRecord.withValue("newValue", formattedValue);
-                        }
-                     }
-                  }
-
-                  if(detailRecord != null)
-                  {
-                     detailRecord.withValue("fieldName", fieldName);
-                     details.add(detailRecord);
-                  }
+                  makeAuditDetailRecordForField(fieldName, table, dmlType, record, oldRecord)
+                     .ifPresent(details::add);
                }
 
                if(details.isEmpty() && DMLType.UPDATE.equals(dmlType))
@@ -231,7 +175,7 @@ public class DMLAuditAction extends AbstractQActionFunction<DMLAuditInput, DMLAu
                }
                else
                {
-                  AuditAction.appendToInput(auditInput, table.getName(), record.getValueInteger(table.getPrimaryKeyField()), getRecordSecurityKeyValues(table, record), "Record was " + dmlType.pastTenseVerb + contextSuffix, details);
+                  AuditAction.appendToInput(auditInput, table.getName(), record.getValueInteger(table.getPrimaryKeyField()), getRecordSecurityKeyValues(table, record, Optional.ofNullable(oldRecord)), "Record was " + dmlType.pastTenseVerb + contextSuffix, details);
                }
             }
          }
@@ -239,11 +183,11 @@ public class DMLAuditAction extends AbstractQActionFunction<DMLAuditInput, DMLAu
          // new AuditAction().executeAsync(auditInput); // todo async??? maybe get that from rules???
          new AuditAction().execute(auditInput);
          long end = System.currentTimeMillis();
-         LOG.debug("Audit performance", logPair("auditLevel", String.valueOf(auditLevel)), logPair("recordCount", recordList.size()), logPair("millis", (end - start)));
+         LOG.trace("Audit performance", logPair("auditLevel", String.valueOf(auditLevel)), logPair("recordCount", recordList.size()), logPair("millis", (end - start)));
       }
       catch(Exception e)
       {
-         LOG.error("Error performing DML audit", e, logPair("type", String.valueOf(dmlType)), logPair("table", table.getName()));
+         LOG.warn("Error performing DML audit", e, logPair("type", String.valueOf(dmlType)), logPair("table", table.getName()));
       }
 
       return (output);
@@ -251,6 +195,272 @@ public class DMLAuditAction extends AbstractQActionFunction<DMLAuditInput, DMLAu
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   static String getContentSuffix(DMLAuditInput input)
+   {
+      StringBuilder contextSuffix = new StringBuilder();
+
+      /////////////////////////////////////////////////////////////////////////////
+      // start with context from the input wrapper                               //
+      // note, these contexts get propagated down from Input/Update/Delete Input //
+      /////////////////////////////////////////////////////////////////////////////
+      if(StringUtils.hasContent(input.getAuditContext()))
+      {
+         contextSuffix.append(" ").append(input.getAuditContext());
+      }
+
+      //////////////////////////////////////////////////////////////
+      // look for a context value place directly into the session //
+      //////////////////////////////////////////////////////////////
+      QSession qSession = QContext.getQSession();
+      if(qSession != null)
+      {
+         String sessionContext = qSession.getValue(AUDIT_CONTEXT_FIELD_NAME);
+         if(StringUtils.hasContent(sessionContext))
+         {
+            contextSuffix.append(" ").append(sessionContext);
+         }
+      }
+
+      /////////////////////////////////////////////////////////////////////////////////////
+      // note process label (and a possible context from the process's state) if present //
+      /////////////////////////////////////////////////////////////////////////////////////
+      Optional<AbstractActionInput> actionInput = QContext.getFirstActionInStack();
+      if(actionInput.isPresent() && actionInput.get() instanceof RunProcessInput runProcessInput)
+      {
+         String processAuditContext = ValueUtils.getValueAsString(runProcessInput.getValue(AUDIT_CONTEXT_FIELD_NAME));
+         if(StringUtils.hasContent(processAuditContext))
+         {
+            contextSuffix.append(" ").append(processAuditContext);
+         }
+
+         String           processName = runProcessInput.getProcessName();
+         QProcessMetaData process     = QContext.getQInstance().getProcess(processName);
+         if(process != null)
+         {
+            contextSuffix.append(" during process: ").append(process.getLabel());
+         }
+      }
+
+      ///////////////////////////////////////////////////
+      // use api label & version if present in session //
+      ///////////////////////////////////////////////////
+      if(qSession != null)
+      {
+         String apiVersion = qSession.getValue("apiVersion");
+         if(apiVersion != null)
+         {
+            String apiLabel = qSession.getValue("apiLabel");
+            if(!StringUtils.hasContent(apiLabel))
+            {
+               apiLabel = "API";
+            }
+            contextSuffix.append(" via ").append(apiLabel).append(" Version: ").append(apiVersion);
+         }
+      }
+
+      return (contextSuffix.toString());
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   static Optional<QRecord> makeAuditDetailRecordForField(String fieldName, QTableMetaData table, DMLType dmlType, QRecord record, QRecord oldRecord)
+   {
+      if(!record.getValues().containsKey(fieldName))
+      {
+         ////////////////////////////////////////////////////////////////////////////////////////////////
+         // if the stored record doesn't have this field name, then don't audit anything about it      //
+         // this is to deal with our Patch style updates not looking like every field was cleared out. //
+         ////////////////////////////////////////////////////////////////////////////////////////////////
+         return (Optional.empty());
+      }
+
+      if(fieldName.equals("modifyDate") || fieldName.equals("createDate") || fieldName.equals("automationStatus"))
+      {
+         return (Optional.empty());
+      }
+
+      QFieldMetaData field        = table.getField(fieldName);
+      Serializable   value        = ValueUtils.getValueAsFieldType(field.getType(), record.getValue(fieldName));
+      Serializable   oldValue     = oldRecord == null ? null : ValueUtils.getValueAsFieldType(field.getType(), oldRecord.getValue(fieldName));
+      QRecord        detailRecord = null;
+
+      if(oldRecord == null)
+      {
+         if(DMLType.INSERT.equals(dmlType) && value == null)
+         {
+            return (Optional.empty());
+         }
+
+         if(field.getType().equals(QFieldType.BLOB) || field.getType().needsMasked())
+         {
+            detailRecord = new QRecord().withValue("message", "Set " + field.getLabel());
+         }
+         else
+         {
+            String formattedValue = getFormattedValueForAuditDetail(record, fieldName, field, value);
+            detailRecord = new QRecord().withValue("message", "Set " + field.getLabel() + " to " + formattedValue);
+            detailRecord.withValue("newValue", formattedValue);
+         }
+      }
+      else
+      {
+         if(areValuesDifferentForAudit(field, value, oldValue))
+         {
+            if(field.getType().equals(QFieldType.BLOB) || field.getType().needsMasked())
+            {
+               if(oldValue == null)
+               {
+                  detailRecord = new QRecord().withValue("message", "Set " + field.getLabel());
+               }
+               else if(value == null)
+               {
+                  detailRecord = new QRecord().withValue("message", "Removed " + field.getLabel());
+               }
+               else
+               {
+                  detailRecord = new QRecord().withValue("message", "Changed " + field.getLabel());
+               }
+            }
+            else
+            {
+               String formattedValue    = getFormattedValueForAuditDetail(record, fieldName, field, value);
+               String formattedOldValue = getFormattedValueForAuditDetail(oldRecord, fieldName, field, oldValue);
+
+               if(oldValue == null)
+               {
+                  detailRecord = new QRecord().withValue("message", "Set " + field.getLabel() + " to " + formatFormattedValueForDetailMessage(field, formattedValue));
+                  detailRecord.withValue("newValue", formattedValue);
+               }
+               else if(value == null)
+               {
+                  detailRecord = new QRecord().withValue("message", "Removed " + formatFormattedValueForDetailMessage(field, formattedOldValue) + " from " + field.getLabel());
+                  detailRecord.withValue("oldValue", formattedOldValue);
+               }
+               else
+               {
+                  detailRecord = new QRecord().withValue("message", "Changed " + field.getLabel() + " from " + formatFormattedValueForDetailMessage(field, formattedOldValue) + " to " + formatFormattedValueForDetailMessage(field, formattedValue));
+                  detailRecord.withValue("oldValue", formattedOldValue);
+                  detailRecord.withValue("newValue", formattedValue);
+               }
+            }
+         }
+      }
+
+      if(detailRecord != null)
+      {
+         ////////////////////////////////////////////////////////////////////
+         // useful if doing dev in here - but overkill for any other time. //
+         ////////////////////////////////////////////////////////////////////
+         // LOG.debug("Returning with message: " + detailRecord.getValueString("message"));
+         detailRecord.withValue("fieldName", fieldName);
+         return (Optional.of(detailRecord));
+      }
+
+      return (Optional.empty());
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   static boolean areValuesDifferentForAudit(QFieldMetaData field, Serializable value, Serializable oldValue)
+   {
+      try
+      {
+         ///////////////////
+         // decimal rules //
+         ///////////////////
+         if(field.getType().equals(QFieldType.DECIMAL))
+         {
+            BigDecimal newBD = ValueUtils.getValueAsBigDecimal(value);
+            BigDecimal oldBD = ValueUtils.getValueAsBigDecimal(oldValue);
+
+            if(newBD == null && oldBD == null)
+            {
+               return (false);
+            }
+
+            if(newBD == null || oldBD == null)
+            {
+               return (true);
+            }
+
+            return (newBD.compareTo(oldBD) != 0);
+         }
+
+         ////////////////////
+         // dateTime rules //
+         ////////////////////
+         if(field.getType().equals(QFieldType.DATE_TIME))
+         {
+            Instant newI = ValueUtils.getValueAsInstant(value);
+            Instant oldI = ValueUtils.getValueAsInstant(oldValue);
+
+            if(newI == null && oldI == null)
+            {
+               return (false);
+            }
+
+            if(newI == null || oldI == null)
+            {
+               return (true);
+            }
+
+            ////////////////////////////////
+            // just compare to the second //
+            ////////////////////////////////
+            return (newI.truncatedTo(ChronoUnit.SECONDS).compareTo(oldI.truncatedTo(ChronoUnit.SECONDS)) != 0);
+         }
+
+         //////////////////
+         // string rules //
+         //////////////////
+         if(field.getType().isStringLike())
+         {
+            String newString = ValueUtils.getValueAsString(value);
+            String oldString = ValueUtils.getValueAsString(oldValue);
+
+            boolean newIsNullOrEmpty = !StringUtils.hasContent(newString);
+            boolean oldIsNullOrEmpty = !StringUtils.hasContent(oldString);
+
+            if(newIsNullOrEmpty && oldIsNullOrEmpty)
+            {
+               return (false);
+            }
+
+            if(newIsNullOrEmpty || oldIsNullOrEmpty)
+            {
+               return (true);
+            }
+
+            return (newString.compareTo(oldString) != 0);
+         }
+
+         /////////////////////////////////////
+         // default just use Objects.equals //
+         /////////////////////////////////////
+         return !Objects.equals(oldValue, value);
+      }
+      catch(Exception e)
+      {
+         LOG.debug("Error checking areValuesDifferentForAudit", e, logPair("fieldName", field.getName()), logPair("value", value), logPair("oldValue", oldValue));
+      }
+
+      ////////////////////////////////////
+      // default to something simple... //
+      ////////////////////////////////////
+      return !Objects.equals(oldValue, value);
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/
@@ -340,21 +550,6 @@ public class DMLAuditAction extends AbstractQActionFunction<DMLAuditInput, DMLAu
 
 
 
-   /*******************************************************************************
-    **
-    *******************************************************************************/
-   private static Map<String, Serializable> getRecordSecurityKeyValues(QTableMetaData table, QRecord record)
-   {
-      Map<String, Serializable> securityKeyValues = new HashMap<>();
-      for(RecordSecurityLock recordSecurityLock : CollectionUtils.nonNullList(table.getRecordSecurityLocks()))
-      {
-         securityKeyValues.put(recordSecurityLock.getSecurityKeyType(), record == null ? null : record.getValue(recordSecurityLock.getFieldName()));
-      }
-      return securityKeyValues;
-   }
-
-
-
    /*******************************************************************************
     **
     *******************************************************************************/
@@ -374,7 +569,7 @@ public class DMLAuditAction extends AbstractQActionFunction<DMLAuditInput, DMLAu
    /*******************************************************************************
     **
     *******************************************************************************/
-   private enum DMLType
+   enum DMLType
    {
       INSERT("Inserted", true),
       UPDATE("Edited", true),

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/automation/AutomationStatus.java

@@ -22,6 +22,7 @@
 package com.kingsrook.qqq.backend.core.actions.automation;
 
 
+import java.util.Objects;
 import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.PossibleValueEnum;
 
 
@@ -55,6 +56,30 @@ public enum AutomationStatus implements PossibleValueEnum<Integer>
 
 
 
+   /*******************************************************************************
+    ** Get instance by id
+    **
+    *******************************************************************************/
+   public static AutomationStatus getById(Integer id)
+   {
+      if(id == null)
+      {
+         return (null);
+      }
+
+      for(AutomationStatus value : AutomationStatus.values())
+      {
+         if(Objects.equals(value.id, id))
+         {
+            return (value);
+         }
+      }
+
+      return (null);
+   }
+
+
+
    /*******************************************************************************
     ** Getter for id
     **
@@ -106,10 +131,10 @@ public enum AutomationStatus implements PossibleValueEnum<Integer>
    public String getInsertOrUpdate()
    {
       return switch(this)
-         {
-            case PENDING_INSERT_AUTOMATIONS, RUNNING_INSERT_AUTOMATIONS, FAILED_INSERT_AUTOMATIONS -> "Insert";
-            case PENDING_UPDATE_AUTOMATIONS, RUNNING_UPDATE_AUTOMATIONS, FAILED_UPDATE_AUTOMATIONS -> "Update";
-            case OK -> "";
-         };
+      {
+         case PENDING_INSERT_AUTOMATIONS, RUNNING_INSERT_AUTOMATIONS, FAILED_INSERT_AUTOMATIONS -> "Insert";
+         case PENDING_UPDATE_AUTOMATIONS, RUNNING_UPDATE_AUTOMATIONS, FAILED_UPDATE_AUTOMATIONS -> "Update";
+         case OK -> "";
+      };
    }
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/automation/RecordAutomationStatusUpdater.java

@@ -22,31 +22,40 @@
 package com.kingsrook.qqq.backend.core.actions.automation;
 
 
-import java.util.ArrayList;
+import java.io.Serializable;
+import java.time.Duration;
+import java.time.temporal.ChronoUnit;
+import java.util.Collections;
+import java.util.HashSet;
 import java.util.List;
-import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+import com.kingsrook.qqq.backend.core.actions.QBackendTransaction;
 import com.kingsrook.qqq.backend.core.actions.tables.CountAction;
+import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
 import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction;
 import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.instances.QMetaDataVariableInterpreter;
 import com.kingsrook.qqq.backend.core.logging.QLogger;
 import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountOutput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
 import com.kingsrook.qqq.backend.core.model.automation.TableTrigger;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
-import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.AutomationStatusTrackingType;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.QTableAutomationDetails;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.TableAutomationAction;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.TriggerEvent;
-import com.kingsrook.qqq.backend.core.model.session.QSession;
 import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
+import com.kingsrook.qqq.backend.core.utils.memoization.Memoization;
 import org.apache.commons.lang.NotImplementedException;
+import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
 
 
 /*******************************************************************************
@@ -56,19 +65,37 @@ public class RecordAutomationStatusUpdater
 {
    private static final QLogger LOG = QLogger.getLogger(RecordAutomationStatusUpdater.class);
 
+   ///////////////////////////////////////////////////////////////////////////////////////////////////////
+   // feature flag - by default, will be true - before setting records to PENDING_UPDATE_AUTOMATIONS,   //
+   // we will fetch them (if we didn't take them in from the caller, which, UpdateAction does if its    //
+   // backend supports it), to check their current automationStatus - and if they are currently PENDING //
+   // or RUNNING inserts or updates, we won't update them.                                              //
+   ///////////////////////////////////////////////////////////////////////////////////////////////////////
+   private static boolean allowPreUpdateFetch = new QMetaDataVariableInterpreter().getBooleanFromPropertyOrEnvironment("qqq.recordAutomationStatusUpdater.allowPreUpdateFetch", "QQQ_RECORD_AUTOMATION_STATUS_UPDATER_ALLOW_PRE_UPDATE_FETCH", true);
+
+   ///////////////////////////////////////////////////////////////////////////////////////////////
+   // feature flag - by default, we'll memoize the check for triggers - but we can turn it off. //
+   ///////////////////////////////////////////////////////////////////////////////////////////////
+   private static boolean memoizeCheckForTriggers = new QMetaDataVariableInterpreter().getBooleanFromPropertyOrEnvironment("qqq.recordAutomationStatusUpdater.memoizeCheckForTriggers", "QQQ_RECORD_AUTOMATION_STATUS_UPDATER_MEMOIZE_CHECK_FOR_TRIGGERS", true);
+
+   private static Memoization<Key, Boolean> areThereTableTriggersForTableMemoization = new Memoization<Key, Boolean>().withTimeout(Duration.of(60, ChronoUnit.SECONDS));
+
 
 
    /*******************************************************************************
     ** for a list of records from a table, set their automation status - based on
     ** how the table is configured.
     *******************************************************************************/
-   public static boolean setAutomationStatusInRecords(QSession session, QTableMetaData table, List<QRecord> records, AutomationStatus automationStatus)
+   public static boolean setAutomationStatusInRecords(QTableMetaData table, List<QRecord> records, AutomationStatus automationStatus, QBackendTransaction transaction, List<QRecord> oldRecordList)
    {
       if(table == null || table.getAutomationDetails() == null || CollectionUtils.nullSafeIsEmpty(records))
       {
          return (false);
       }
 
+      QTableAutomationDetails automationDetails   = table.getAutomationDetails();
+      Set<Serializable>       pkeysWeMayNotUpdate = new HashSet<>();
+
       ///////////////////////////////////////////////////////////////////////////////////////////////////
       // In case an automation is running, and it updates records - don't let those records be marked  //
       // as PENDING_UPDATE_AUTOMATIONS... this is meant to avoid having a record's automation update   //
@@ -82,32 +109,80 @@ public class RecordAutomationStatusUpdater
          for(StackTraceElement stackTraceElement : e.getStackTrace())
          {
             String className = stackTraceElement.getClassName();
-            if(className.contains("com.kingsrook.qqq.backend.core.actions.automation") && !className.equals(RecordAutomationStatusUpdater.class.getName()) && !className.endsWith("Test"))
+            if(className.contains(RecordAutomationStatusUpdater.class.getPackageName()) && !className.equals(RecordAutomationStatusUpdater.class.getName()) && !className.endsWith("Test") && !className.contains("Test$"))
             {
                LOG.debug("Avoiding re-setting automation status to PENDING_UPDATE while running an automation");
                return (false);
             }
          }
+
+         ///////////////////////////////////////////////////////////////////////////////
+         // if table uses field-in-table status tracking, then check the old records, //
+         // before we set them to pending-updates, to avoid losing other pending or   //
+         // running status information.  We will allow moving  from OK or the 2       //
+         // failed statuses into pending-updates - which seems right.                 //
+         // This is added to fix cases where an update that comes in before insert    //
+         // -automations have run, will cause the pending-insert status to be missed. //
+         ///////////////////////////////////////////////////////////////////////////////
+         if(automationDetails.getStatusTracking() != null && AutomationStatusTrackingType.FIELD_IN_TABLE.equals(automationDetails.getStatusTracking().getType()))
+         {
+            try
+            {
+               if(CollectionUtils.nullSafeIsEmpty(oldRecordList))
+               {
+                  ///////////////////////////////////////////////////////////////////////////////////////////////
+                  // if we didn't get the oldRecordList as input (though UpdateAction should usually pass it?) //
+                  // then check feature-flag if we're allowed to do a lookup here & now.  If so, then do.      //
+                  ///////////////////////////////////////////////////////////////////////////////////////////////
+                  if(allowPreUpdateFetch)
+                  {
+                     List<Serializable> pkeysToLookup = records.stream().map(r -> r.getValue(table.getPrimaryKeyField())).toList();
+                     oldRecordList = new QueryAction().execute(new QueryInput(table.getName())
+                        .withFilter(new QQueryFilter(new QFilterCriteria(table.getPrimaryKeyField(), QCriteriaOperator.IN, pkeysToLookup)))
+                        .withTransaction(transaction)
+                     ).getRecords();
+                  }
+               }
+
+               for(QRecord freshRecord : CollectionUtils.nonNullList(oldRecordList))
+               {
+                  Serializable recordStatus = freshRecord.getValue(automationDetails.getStatusTracking().getFieldName());
+                  if(AutomationStatus.PENDING_INSERT_AUTOMATIONS.getId().equals(recordStatus)
+                     || AutomationStatus.PENDING_UPDATE_AUTOMATIONS.getId().equals(recordStatus)
+                     || AutomationStatus.RUNNING_INSERT_AUTOMATIONS.getId().equals(recordStatus)
+                     || AutomationStatus.RUNNING_UPDATE_AUTOMATIONS.getId().equals(recordStatus))
+                  {
+                     Serializable primaryKey = freshRecord.getValue(table.getPrimaryKeyField());
+                     LOG.debug("May not update automation status", logPair("table", table.getName()), logPair("id", primaryKey), logPair("currentStatus", recordStatus), logPair("requestedStatus", automationStatus.getId()));
+                     pkeysWeMayNotUpdate.add(primaryKey);
+                  }
+               }
+            }
+            catch(QException qe)
+            {
+               LOG.error("Error checking existing automation status before setting new automation status - more records will be updated than maybe should be...", qe);
+            }
+         }
       }
 
+      ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      // Avoid setting records to PENDING_INSERT or PENDING_UPDATE even if they don't have any insert or update automations or triggers //
+      // such records should go straight to OK status.                                                                                  //
+      ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
       if(canWeSkipPendingAndGoToOkay(table, automationStatus))
       {
          automationStatus = AutomationStatus.OK;
       }
 
-      QTableAutomationDetails automationDetails = table.getAutomationDetails();
       if(automationDetails.getStatusTracking() != null && AutomationStatusTrackingType.FIELD_IN_TABLE.equals(automationDetails.getStatusTracking().getType()))
       {
          for(QRecord record : records)
          {
-            ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-            // todo - seems like there's some case here, where if an order was in PENDING_INSERT, but then some other job updated the record, that we'd  //
-            //  lose that pending status, which would be a Bad Thing™...                                                                                 //
-            //  problem is - we may not have the full record in here, so we can't necessarily check the record to see what status it's currently in...   //
-            ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-
-            record.setValue(automationDetails.getStatusTracking().getFieldName(), automationStatus.getId());
-            // todo - another field - for the automation timestamp??
+            if(!pkeysWeMayNotUpdate.contains(record.getValue(table.getPrimaryKeyField())))
+            {
+               record.setValue(automationDetails.getStatusTracking().getFieldName(), automationStatus.getId());
+               // todo - another field - for the automation timestamp??
+            }
          }
       }
 
@@ -121,9 +196,13 @@ public class RecordAutomationStatusUpdater
     ** being asked to set status to PENDING_INSERT (or PENDING_UPDATE), then just
     ** move the status straight to OK.
     *******************************************************************************/
-   private static boolean canWeSkipPendingAndGoToOkay(QTableMetaData table, AutomationStatus automationStatus)
+   static boolean canWeSkipPendingAndGoToOkay(QTableMetaData table, AutomationStatus automationStatus)
    {
-      List<TableAutomationAction> tableActions = Objects.requireNonNullElse(table.getAutomationDetails().getActions(), new ArrayList<>());
+      List<TableAutomationAction> tableActions = Collections.emptyList();
+      if(table.getAutomationDetails() != null && table.getAutomationDetails().getActions() != null)
+      {
+         tableActions = table.getAutomationDetails().getActions();
+      }
 
       if(automationStatus.equals(AutomationStatus.PENDING_INSERT_AUTOMATIONS))
       {
@@ -135,6 +214,12 @@ public class RecordAutomationStatusUpdater
          {
             return (false);
          }
+
+         ////////////////////////////////////////////////////////////////////////////////////////
+         // if we're going to pending-insert, and there are no insert automations or triggers, //
+         // then we may skip pending and go to okay.                                           //
+         ////////////////////////////////////////////////////////////////////////////////////////
+         return (true);
       }
       else if(automationStatus.equals(AutomationStatus.PENDING_UPDATE_AUTOMATIONS))
       {
@@ -146,9 +231,21 @@ public class RecordAutomationStatusUpdater
          {
             return (false);
          }
-      }
 
-      return (true);
+         ////////////////////////////////////////////////////////////////////////////////////////
+         // if we're going to pending-update, and there are no insert automations or triggers, //
+         // then we may skip pending and go to okay.                                           //
+         ////////////////////////////////////////////////////////////////////////////////////////
+         return (true);
+      }
+      else
+      {
+         ///////////////////////////////////////////////////////////////////////////////////////////////////////
+         // if we're going to any other automation status - then we may never "skip pending" and go to okay - //
+         // because we weren't asked to go to pending!                                                        //
+         ///////////////////////////////////////////////////////////////////////////////////////////////////////
+         return (false);
+      }
    }
 
 
@@ -163,11 +260,29 @@ public class RecordAutomationStatusUpdater
          return (false);
       }
 
+      if(memoizeCheckForTriggers)
+      {
+         ///////////////////////////////////////////////////////////////////////////////////////
+         // as within the lookup method, error on the side of "yes, maybe there are triggers" //
+         ///////////////////////////////////////////////////////////////////////////////////////
+         Optional<Boolean> result = areThereTableTriggersForTableMemoization.getResult(new Key(table, triggerEvent), key -> lookupIfThereAreTriggersForTable(table, triggerEvent));
+         return result.orElse(true);
+      }
+      else
+      {
+         return lookupIfThereAreTriggersForTable(table, triggerEvent);
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private static Boolean lookupIfThereAreTriggersForTable(QTableMetaData table, TriggerEvent triggerEvent)
+   {
       try
       {
-         ///////////////////
-         // todo - cache? //
-         ///////////////////
          CountInput countInput = new CountInput();
          countInput.setTableName(TableTrigger.TABLE_NAME);
          countInput.setFilter(new QQueryFilter(
@@ -182,6 +297,7 @@ public class RecordAutomationStatusUpdater
          ///////////////////////////////////////////////////////////////////////////////////////////////////////////
          // if the count query failed, we're a bit safer to err on the side of "yeah, there might be automations" //
          ///////////////////////////////////////////////////////////////////////////////////////////////////////////
+         LOG.warn("Error looking if there are triggers for table", e, logPair("tableName", table.getName()));
          return (true);
       }
    }
@@ -192,12 +308,12 @@ public class RecordAutomationStatusUpdater
     ** for a list of records, update their automation status and actually Update the
     ** backend as well.
     *******************************************************************************/
-   public static void setAutomationStatusInRecordsAndUpdate(QInstance instance, QSession session, QTableMetaData table, List<QRecord> records, AutomationStatus automationStatus) throws QException
+   public static void setAutomationStatusInRecordsAndUpdate(QTableMetaData table, List<QRecord> records, AutomationStatus automationStatus, QBackendTransaction transaction) throws QException
    {
       QTableAutomationDetails automationDetails = table.getAutomationDetails();
       if(automationDetails != null && AutomationStatusTrackingType.FIELD_IN_TABLE.equals(automationDetails.getStatusTracking().getType()))
       {
-         boolean didSetStatusField = setAutomationStatusInRecords(session, table, records, automationStatus);
+         boolean didSetStatusField = setAutomationStatusInRecords(table, records, automationStatus, transaction, null);
          if(didSetStatusField)
          {
             UpdateInput updateInput = new UpdateInput();
@@ -212,6 +328,7 @@ public class RecordAutomationStatusUpdater
                .withValue(table.getPrimaryKeyField(), r.getValue(table.getPrimaryKeyField()))
                .withValue(automationDetails.getStatusTracking().getFieldName(), r.getValue(automationDetails.getStatusTracking().getFieldName()))).toList());
             updateInput.setAreAllValuesBeingUpdatedTheSame(true);
+            updateInput.setTransaction(transaction);
             updateInput.setOmitDmlAudit(true);
 
             new UpdateAction().execute(updateInput);
@@ -225,4 +342,8 @@ public class RecordAutomationStatusUpdater
       }
    }
 
+
+
+   private record Key(QTableMetaData table, TriggerEvent triggerEvent) {}
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/automation/RunRecordScriptAutomationHandler.java

@@ -42,6 +42,7 @@ import com.kingsrook.qqq.backend.core.model.data.QRecord;
 import com.kingsrook.qqq.backend.core.model.metadata.code.AdHocScriptCodeReference;
 import com.kingsrook.qqq.backend.core.model.scripts.Script;
 import com.kingsrook.qqq.backend.core.model.scripts.ScriptRevision;
+import com.kingsrook.qqq.backend.core.model.scripts.ScriptsMetaDataProvider;
 import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
 import com.kingsrook.qqq.backend.core.utils.ValueUtils;
 import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
@@ -74,7 +75,7 @@ public class RunRecordScriptAutomationHandler extends RecordAutomationHandler
       QueryInput queryInput = new QueryInput();
       queryInput.setTableName(ScriptRevision.TABLE_NAME);
       queryInput.setFilter(new QQueryFilter(new QFilterCriteria("scriptId", QCriteriaOperator.EQUALS, scriptId)));
-      queryInput.withQueryJoin(new QueryJoin(Script.TABLE_NAME).withBaseTableOrAlias(ScriptRevision.TABLE_NAME).withJoinMetaData(QContext.getQInstance().getJoin("currentScriptRevision")));
+      queryInput.withQueryJoin(new QueryJoin(Script.TABLE_NAME).withBaseTableOrAlias(ScriptRevision.TABLE_NAME).withJoinMetaData(QContext.getQInstance().getJoin(ScriptsMetaDataProvider.CURRENT_SCRIPT_REVISION_JOIN_NAME)));
       QueryOutput queryOutput = new QueryAction().execute(queryInput);
       if(CollectionUtils.nullSafeIsEmpty(queryOutput.getRecords()))
       {

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/automation/polling/PollingAutomationPerTableRunner.java

@@ -28,6 +28,7 @@ import java.util.Comparator;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
+import java.util.Optional;
 import java.util.function.Supplier;
 import java.util.stream.Collectors;
 import com.kingsrook.qqq.backend.core.actions.async.AsyncRecordPipeLoop;
@@ -39,12 +40,16 @@ import com.kingsrook.qqq.backend.core.actions.customizers.QCodeLoader;
 import com.kingsrook.qqq.backend.core.actions.processes.QProcessCallback;
 import com.kingsrook.qqq.backend.core.actions.processes.RunProcessAction;
 import com.kingsrook.qqq.backend.core.actions.reporting.RecordPipe;
+import com.kingsrook.qqq.backend.core.actions.tables.GetAction;
 import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
 import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.logging.LogPair;
 import com.kingsrook.qqq.backend.core.logging.QLogger;
 import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput;
 import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessOutput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetOutput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy;
@@ -56,16 +61,22 @@ import com.kingsrook.qqq.backend.core.model.automation.TableTrigger;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
 import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
 import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.DynamicDefaultValueBehavior;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.AutomationStatusTrackingType;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.QTableAutomationDetails;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.TableAutomationAction;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.TriggerEvent;
+import com.kingsrook.qqq.backend.core.model.savedviews.SavedView;
 import com.kingsrook.qqq.backend.core.model.session.QSession;
 import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
+import com.kingsrook.qqq.backend.core.utils.JsonUtils;
 import com.kingsrook.qqq.backend.core.utils.StringUtils;
 import com.kingsrook.qqq.backend.core.utils.collections.MapBuilder;
 import org.apache.commons.lang.NotImplementedException;
+import org.json.JSONObject;
+import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
 
 
 /*******************************************************************************
@@ -81,8 +92,9 @@ public class PollingAutomationPerTableRunner implements Runnable
 {
    private static final QLogger LOG = QLogger.getLogger(PollingAutomationPerTableRunner.class);
 
-   private final TableActions tableActions;
-   private final String       name;
+   private final TableActionsInterface tableActions;
+
+   private String name;
 
    private QInstance          instance;
    private Supplier<QSession> sessionSupplier;
@@ -110,10 +122,56 @@ public class PollingAutomationPerTableRunner implements Runnable
 
 
    /*******************************************************************************
-    **
+    ** Interface to be used by 2 records in this class - normal TableActions, and
+    ** ShardedTableActions.
     *******************************************************************************/
-   public record TableActions(String tableName, AutomationStatus status)
+   public interface TableActionsInterface
    {
+      /*******************************************************************************
+       **
+       *******************************************************************************/
+      String tableName();
+
+      /*******************************************************************************
+       **
+       *******************************************************************************/
+      QTableAutomationDetails tableAutomationDetails();
+
+      /*******************************************************************************
+       **
+       *******************************************************************************/
+      AutomationStatus status();
+   }
+
+
+
+   /*******************************************************************************
+    ** Wrapper for a pair of (tableName, automationStatus)
+    *******************************************************************************/
+   public record TableActions(String tableName, QTableAutomationDetails tableAutomationDetails, AutomationStatus status) implements TableActionsInterface
+   {
+      /*******************************************************************************
+       **
+       *******************************************************************************/
+      public void noopToFakeTestCoverage()
+      {
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** extended version of TableAction, for sharding use-case - adds the shard
+    ** details.
+    *******************************************************************************/
+   public record ShardedTableActions(String tableName, QTableAutomationDetails tableAutomationDetails, AutomationStatus status, String shardByFieldName, Serializable shardValue, String shardLabel) implements TableActionsInterface
+   {
+      /*******************************************************************************
+       **
+       *******************************************************************************/
+      public void noopToFakeTestCoverage()
+      {
+      }
    }
 
 
@@ -122,16 +180,46 @@ public class PollingAutomationPerTableRunner implements Runnable
     ** basically just get a list of tables which at least *could* have automations
     ** run - either meta-data automations, or table-triggers (data/user defined).
     *******************************************************************************/
-   public static List<TableActions> getTableActions(QInstance instance, String providerName)
+   public static List<TableActionsInterface> getTableActions(QInstance instance, String providerName)
    {
-      List<TableActions> tableActionList = new ArrayList<>();
+      List<TableActionsInterface> tableActionList = new ArrayList<>();
 
       for(QTableMetaData table : instance.getTables().values())
       {
-         if(table.getAutomationDetails() != null && providerName.equals(table.getAutomationDetails().getProviderName()))
+         QTableAutomationDetails automationDetails = table.getAutomationDetails();
+         if(automationDetails != null && providerName.equals(automationDetails.getProviderName()))
          {
-            tableActionList.add(new TableActions(table.getName(), AutomationStatus.PENDING_INSERT_AUTOMATIONS));
-            tableActionList.add(new TableActions(table.getName(), AutomationStatus.PENDING_UPDATE_AUTOMATIONS));
+            if(StringUtils.hasContent(automationDetails.getShardByFieldName()))
+            {
+               //////////////////////////////////////////////////////////////////////////////////////////////
+               // for sharded automations, add a tableAction (of the sharded subtype) for each shard-value //
+               //////////////////////////////////////////////////////////////////////////////////////////////
+               try
+               {
+                  QueryInput queryInput = new QueryInput();
+                  queryInput.setTableName(automationDetails.getShardSourceTableName());
+                  QueryOutput queryOutput = new QueryAction().execute(queryInput);
+                  for(QRecord record : queryOutput.getRecords())
+                  {
+                     Serializable shardId = record.getValue(automationDetails.getShardIdFieldName());
+                     String       label   = record.getValueString(automationDetails.getShardLabelFieldName());
+                     tableActionList.add(new ShardedTableActions(table.getName(), automationDetails, AutomationStatus.PENDING_INSERT_AUTOMATIONS, automationDetails.getShardByFieldName(), shardId, label));
+                     tableActionList.add(new ShardedTableActions(table.getName(), automationDetails, AutomationStatus.PENDING_UPDATE_AUTOMATIONS, automationDetails.getShardByFieldName(), shardId, label));
+                  }
+               }
+               catch(Exception e)
+               {
+                  LOG.error("Error getting sharded table automation actions for a table", e, new LogPair("tableName", table.getName()));
+               }
+            }
+            else
+            {
+               //////////////////////////////////////////////////////////////////
+               // for non-sharded, we just need table name & automation status //
+               //////////////////////////////////////////////////////////////////
+               tableActionList.add(new TableActions(table.getName(), automationDetails, AutomationStatus.PENDING_INSERT_AUTOMATIONS));
+               tableActionList.add(new TableActions(table.getName(), automationDetails, AutomationStatus.PENDING_UPDATE_AUTOMATIONS));
+            }
          }
       }
 
@@ -143,12 +231,17 @@ public class PollingAutomationPerTableRunner implements Runnable
    /*******************************************************************************
     **
     *******************************************************************************/
-   public PollingAutomationPerTableRunner(QInstance instance, String providerName, Supplier<QSession> sessionSupplier, TableActions tableActions)
+   public PollingAutomationPerTableRunner(QInstance instance, String providerName, Supplier<QSession> sessionSupplier, TableActionsInterface tableActions)
    {
       this.instance = instance;
       this.sessionSupplier = sessionSupplier;
       this.tableActions = tableActions;
       this.name = providerName + ">" + tableActions.tableName() + ">" + tableActions.status().getInsertOrUpdate();
+
+      if(tableActions instanceof ShardedTableActions shardedTableActions)
+      {
+         this.name += ":" + shardedTableActions.shardLabel();
+      }
    }
 
 
@@ -167,12 +260,11 @@ public class PollingAutomationPerTableRunner implements Runnable
 
       try
       {
-         QSession session = sessionSupplier != null ? sessionSupplier.get() : new QSession();
-         processTableInsertOrUpdate(instance.getTable(tableActions.tableName()), session, tableActions.status());
+         processTableInsertOrUpdate(instance.getTable(tableActions.tableName()), tableActions.status());
       }
       catch(Exception e)
       {
-         LOG.warn("Error running automations", e);
+         LOG.warn("Error running automations", e, logPair("tableName", tableActions.tableName()), logPair("status", tableActions.status()));
       }
       finally
       {
@@ -186,7 +278,7 @@ public class PollingAutomationPerTableRunner implements Runnable
    /*******************************************************************************
     ** Query for and process records that have a PENDING_INSERT or PENDING_UPDATE status on a given table.
     *******************************************************************************/
-   public void processTableInsertOrUpdate(QTableMetaData table, QSession session, AutomationStatus automationStatus) throws QException
+   public void processTableInsertOrUpdate(QTableMetaData table, AutomationStatus automationStatus) throws QException
    {
       /////////////////////////////////////////////////////////////////////////
       // get the actions to run against this table in this automation status //
@@ -216,19 +308,30 @@ public class PollingAutomationPerTableRunner implements Runnable
             AutomationStatusTrackingType statusTrackingType = automationDetails.getStatusTracking().getType();
             if(AutomationStatusTrackingType.FIELD_IN_TABLE.equals(statusTrackingType))
             {
-               queryInput.setFilter(new QQueryFilter().withCriteria(new QFilterCriteria(automationDetails.getStatusTracking().getFieldName(), QCriteriaOperator.EQUALS, List.of(automationStatus.getId()))));
+               QQueryFilter filter = new QQueryFilter().withCriteria(new QFilterCriteria(automationDetails.getStatusTracking().getFieldName(), QCriteriaOperator.EQUALS, List.of(automationStatus.getId())));
+               addOrderByToQueryFilter(table, automationStatus, filter);
+               queryInput.setFilter(filter);
             }
             else
             {
                throw (new NotImplementedException("Automation Status Tracking type [" + statusTrackingType + "] is not yet implemented in here."));
             }
 
+            if(tableActions instanceof ShardedTableActions shardedTableActions)
+            {
+               //////////////////////////////////////////////////////////////
+               // for sharded actions, add the shardBy field as a criteria //
+               //////////////////////////////////////////////////////////////
+               QQueryFilter filter = queryInput.getFilter();
+               filter.addCriteria(new QFilterCriteria(shardedTableActions.shardByFieldName(), QCriteriaOperator.EQUALS, shardedTableActions.shardValue()));
+            }
+
             queryInput.setRecordPipe(recordPipe);
             return (new QueryAction().execute(queryInput));
          }, () ->
          {
             List<QRecord> records = recordPipe.consumeAvailableRecords();
-            applyActionsToRecords(session, table, records, actions, automationStatus);
+            applyActionsToRecords(table, records, actions, automationStatus);
             return (records.size());
          }
       );
@@ -236,6 +339,38 @@ public class PollingAutomationPerTableRunner implements Runnable
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   static void addOrderByToQueryFilter(QTableMetaData table, AutomationStatus automationStatus, QQueryFilter filter)
+   {
+      ////////////////////////////////////////////////////////////////////////////////////
+      // look for a field in the table with either create-date or modify-date behavior, //
+      // based on if doing insert or update automations                                 //
+      ////////////////////////////////////////////////////////////////////////////////////
+      DynamicDefaultValueBehavior dynamicDefaultValueBehavior = automationStatus.equals(AutomationStatus.PENDING_INSERT_AUTOMATIONS) ? DynamicDefaultValueBehavior.CREATE_DATE : DynamicDefaultValueBehavior.MODIFY_DATE;
+      Optional<QFieldMetaData> field = table.getFields().values().stream()
+         .filter(f -> dynamicDefaultValueBehavior.equals(f.getBehaviorOrDefault(QContext.getQInstance(), DynamicDefaultValueBehavior.class)))
+         .findFirst();
+
+      if(field.isPresent())
+      {
+         //////////////////////////////////////////////////////////////////////
+         // if a create/modify date field was found, order by it (ascending) //
+         //////////////////////////////////////////////////////////////////////
+         filter.addOrderBy(new QFilterOrderBy(field.get().getName()));
+      }
+      else
+      {
+         ////////////////////////////////////
+         // else, order by the primary key //
+         ////////////////////////////////////
+         filter.addOrderBy(new QFilterOrderBy(table.getPrimaryKeyField()));
+      }
+   }
+
+
+
    /*******************************************************************************
     ** get the actions to run against a table in an automation status.  both from
     ** metaData and tableTriggers/data.
@@ -252,7 +387,23 @@ public class PollingAutomationPerTableRunner implements Runnable
       {
          if(action.getTriggerEvent().equals(triggerEvent))
          {
-            rs.add(action);
+            ///////////////////////////////////////////////////////////
+            // for sharded configs, only run if the shard id matches //
+            ///////////////////////////////////////////////////////////
+            if(tableActions instanceof ShardedTableActions shardedTableActions)
+            {
+               if(shardedTableActions.shardValue().equals(action.getShardId()))
+               {
+                  rs.add(action);
+               }
+            }
+            else
+            {
+               ////////////////////////////////////////////
+               // for non-sharded, always add the action //
+               ////////////////////////////////////////////
+               rs.add(action);
+            }
          }
       }
 
@@ -270,15 +421,41 @@ public class PollingAutomationPerTableRunner implements Runnable
          QueryOutput queryOutput = new QueryAction().execute(queryInput);
          for(QRecord record : queryOutput.getRecords())
          {
-            // todo - get filter if there is/was one
-            rs.add(new TableAutomationAction()
-               .withName("Script:" + record.getValue("scriptId"))
-               .withFilter(null)
-               .withTriggerEvent(triggerEvent)
-               .withPriority(record.getValueInteger("priority"))
-               .withCodeReference(new QCodeReference(RunRecordScriptAutomationHandler.class))
-               .withValues(MapBuilder.of("scriptId", record.getValue("scriptId")))
-            );
+            TableTrigger tableTrigger = new TableTrigger(record);
+
+            try
+            {
+               QQueryFilter filter   = null;
+               Integer      filterId = tableTrigger.getFilterId();
+               if(filterId != null)
+               {
+                  GetInput getInput = new GetInput();
+                  getInput.setTableName(SavedView.TABLE_NAME);
+                  getInput.setPrimaryKey(filterId);
+                  GetOutput getOutput = new GetAction().execute(getInput);
+                  if(getOutput.getRecord() != null)
+                  {
+                     SavedView  savedView   = new SavedView(getOutput.getRecord());
+                     JSONObject viewJson    = new JSONObject(savedView.getViewJson());
+                     JSONObject queryFilter = viewJson.getJSONObject("queryFilter");
+                     filter = JsonUtils.toObject(queryFilter.toString(), QQueryFilter.class);
+                  }
+               }
+
+               rs.add(new TableAutomationAction()
+                  .withName("Script:" + tableTrigger.getScriptId())
+                  .withFilter(filter)
+                  .withTriggerEvent(triggerEvent)
+                  .withPriority(tableTrigger.getPriority())
+                  .withCodeReference(new QCodeReference(RunRecordScriptAutomationHandler.class))
+                  .withValues(MapBuilder.of("scriptId", tableTrigger.getScriptId()))
+                  .withIncludeRecordAssociations(true)
+               );
+            }
+            catch(Exception e)
+            {
+               LOG.error("Error setting up table trigger", e, logPair("tableTriggerId", tableTrigger.getId()));
+            }
          }
       }
 
@@ -294,7 +471,7 @@ public class PollingAutomationPerTableRunner implements Runnable
     ** table's actions against them - IF they are found to match the action's filter
     ** (assuming it has one - if it doesn't, then all records match).
     *******************************************************************************/
-   private void applyActionsToRecords(QSession session, QTableMetaData table, List<QRecord> records, List<TableAutomationAction> actions, AutomationStatus automationStatus) throws QException
+   private void applyActionsToRecords(QTableMetaData table, List<QRecord> records, List<TableAutomationAction> actions, AutomationStatus automationStatus) throws QException
    {
       if(CollectionUtils.nullSafeIsEmpty(records))
       {
@@ -304,7 +481,7 @@ public class PollingAutomationPerTableRunner implements Runnable
       ///////////////////////////////////////////////////
       // mark the records as RUNNING their automations //
       ///////////////////////////////////////////////////
-      RecordAutomationStatusUpdater.setAutomationStatusInRecordsAndUpdate(instance, session, table, records, pendingToRunningStatusMap.get(automationStatus));
+      RecordAutomationStatusUpdater.setAutomationStatusInRecordsAndUpdate(table, records, pendingToRunningStatusMap.get(automationStatus), null);
 
       ///////////////////////////////////////////////////////////////////////////////////////////////////////////////
       // foreach action - run it against the records (but only if they match the action's filter, if there is one) //
@@ -312,22 +489,9 @@ public class PollingAutomationPerTableRunner implements Runnable
       boolean anyActionsFailed = false;
       for(TableAutomationAction action : actions)
       {
-         try
+         boolean hadError = applyActionToRecords(table, records, action);
+         if(hadError)
          {
-            //////////////////////////////////////////////////////////////////////////////////////////////////////////////
-            // note - this method - will re-query the objects, so we should have confidence that their data is fresh... //
-            //////////////////////////////////////////////////////////////////////////////////////////////////////////////
-            List<QRecord> matchingQRecords = getRecordsMatchingActionFilter(table, records, action);
-            LOG.debug("Of the {} records that were pending automations, {} of them match the filter on the action {}", records.size(), matchingQRecords.size(), action);
-            if(CollectionUtils.nullSafeHasContents(matchingQRecords))
-            {
-               LOG.debug("  Processing " + matchingQRecords.size() + " records in " + table + " for action " + action);
-               applyActionToMatchingRecords(table, matchingQRecords, action);
-            }
-         }
-         catch(Exception e)
-         {
-            LOG.warn("Caught exception processing records on " + table + " for action " + action, e);
             anyActionsFailed = true;
          }
       }
@@ -335,13 +499,46 @@ public class PollingAutomationPerTableRunner implements Runnable
       ////////////////////////////////////////
       // update status on all these records //
       ////////////////////////////////////////
-      if(anyActionsFailed)
+      AutomationStatus statusToUpdateTo = anyActionsFailed ? pendingToFailedStatusMap.get(automationStatus) : AutomationStatus.OK;
+      try
       {
-         RecordAutomationStatusUpdater.setAutomationStatusInRecordsAndUpdate(instance, session, table, records, pendingToFailedStatusMap.get(automationStatus));
+         RecordAutomationStatusUpdater.setAutomationStatusInRecordsAndUpdate(table, records, statusToUpdateTo, null);
       }
-      else
+      catch(Exception e)
       {
-         RecordAutomationStatusUpdater.setAutomationStatusInRecordsAndUpdate(instance, session, table, records, AutomationStatus.OK);
+         LOG.warn("Error updating automationStatus after running automations", logPair("tableName", table), logPair("count", records.size()), logPair("status", statusToUpdateTo));
+         throw (e);
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** Run one action over a list of records (if they match the action's filter).
+    **
+    ** @return hadError - true if an exception was caught; false if all OK.
+    *******************************************************************************/
+   protected boolean applyActionToRecords(QTableMetaData table, List<QRecord> records, TableAutomationAction action)
+   {
+      try
+      {
+         //////////////////////////////////////////////////////////////////////////////////////////////////////////////
+         // note - this method - will re-query the objects, so we should have confidence that their data is fresh... //
+         //////////////////////////////////////////////////////////////////////////////////////////////////////////////
+         List<QRecord> matchingQRecords = getRecordsMatchingActionFilter(table, records, action);
+         LOG.debug("Of the [" + records.size() + "] records that were pending automations, [" + matchingQRecords.size() + "] of them match the filter on the action:" +  action);
+         if(CollectionUtils.nullSafeHasContents(matchingQRecords))
+         {
+            LOG.debug("  Processing " + matchingQRecords.size() + " records in " + table + " for action " + action);
+            applyActionToMatchingRecords(table, matchingQRecords, action);
+         }
+
+         return (false);
+      }
+      catch(Exception e)
+      {
+         LOG.warn("Caught exception processing automations", e, logPair("tableName", table), logPair("action", action.getName()));
+         return (true);
       }
    }
 
@@ -364,26 +561,29 @@ public class PollingAutomationPerTableRunner implements Runnable
       QueryInput queryInput = new QueryInput();
       queryInput.setTableName(table.getName());
 
+      ///////////////////////////////////////////////////////////////////////////////////////
+      // set up a filter that is for the primary keys IN the list that we identified above //
+      ///////////////////////////////////////////////////////////////////////////////////////
       QQueryFilter filter = new QQueryFilter();
+      filter.addCriteria(new QFilterCriteria(table.getPrimaryKeyField(), QCriteriaOperator.IN, records.stream().map(r -> r.getValue(table.getPrimaryKeyField())).toList()));
 
-      /////////////////////////////////////////////////////////////////////////////////////////////////////
-      // copy filter criteria from the action's filter to a new filter that we'll run here.              //
-      // Critically - don't modify the filter object on the action!  as that object has a long lifespan. //
-      /////////////////////////////////////////////////////////////////////////////////////////////////////
       if(action.getFilter() != null)
       {
-         if(action.getFilter().getCriteria() != null)
-         {
-            action.getFilter().getCriteria().forEach(filter::addCriteria);
-         }
+         /////////////////////////////////////////////////////////////////////////////////////////////////////
+         // if the action defines a filter of its own, add that to the filter we'll run now as a sub-filter //
+         // not entirely clear if this needs to be a clone, but, it feels safe and cheap enough             //
+         /////////////////////////////////////////////////////////////////////////////////////////////////////
+         filter.addSubFilter(action.getFilter().clone());
+
+         ////////////////////////////////////////////////////////////////////////////////////////////////////////////
+         // we also want to set order-bys from the action into our filter (since they only apply at the top-level) //
+         ////////////////////////////////////////////////////////////////////////////////////////////////////////////
          if(action.getFilter().getOrderBys() != null)
          {
             action.getFilter().getOrderBys().forEach(filter::addOrderBy);
          }
       }
 
-      filter.addCriteria(new QFilterCriteria(table.getPrimaryKeyField(), QCriteriaOperator.IN, records.stream().map(r -> r.getValue(table.getPrimaryKeyField())).toList()));
-
       /////////////////////////////////////////////////////////////////////////////////////////////
       // always add order-by the primary key, to give more predictable/consistent results        //
       // todo - in future - if this becomes a source of slowness, make this a config to opt-out? //
@@ -392,6 +592,8 @@ public class PollingAutomationPerTableRunner implements Runnable
 
       queryInput.setFilter(filter);
 
+      queryInput.setIncludeAssociations(action.getIncludeRecordAssociations());
+
       return (new QueryAction().execute(queryInput).getRecords());
    }
 
@@ -399,7 +601,7 @@ public class PollingAutomationPerTableRunner implements Runnable
 
    /*******************************************************************************
     ** Finally, actually run action code against a list of known matching records.
-    ** todo not commit - move to somewhere genericer
+    **
     *******************************************************************************/
    public static void applyActionToMatchingRecords(QTableMetaData table, List<QRecord> records, TableAutomationAction action) throws Exception
    {
@@ -418,7 +620,7 @@ public class PollingAutomationPerTableRunner implements Runnable
             @Override
             public QQueryFilter getQueryFilter()
             {
-               List<Serializable> recordIds = records.stream().map(r -> r.getValueInteger(table.getPrimaryKeyField())).collect(Collectors.toList());
+               List<Serializable> recordIds = records.stream().map(r -> r.getValue(table.getPrimaryKeyField())).collect(Collectors.toList());
                return (new QQueryFilter().withCriteria(new QFilterCriteria(table.getPrimaryKeyField(), QCriteriaOperator.IN, recordIds)));
             }
          });

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/AbstractPostDeleteCustomizer.java

@@ -0,0 +1,94 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.actions.customizers;
+
+
+import java.util.List;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+
+
+/*******************************************************************************
+ ** Abstract class that a table can specify an implementation of, to provide
+ ** custom actions after a delete takes place.
+ **
+ ** General implementation would be, to iterate over the records (ones which didn't
+ ** have a delete error), and look at their values:
+ ** - possibly adding Errors (`addError`) or Warnings (`addWarning`) to the records?
+ ** - possibly throwing an exception - though doing so won't stop the delete, and instead
+ **   will just set a warning on all of the deleted records...
+ ** - doing "whatever else" you may want to do.
+ ** - returning the list of records (can be the input list) that you want to go back
+ **   to the caller - this is how errors and warnings are propagated .
+ **
+ ** Note that the full deleteInput is available as a field in this class.
+ **
+ ** A future enhancement here may be to take (as fields in this class) the list of
+ ** records that the delete action marked in error - the user might want to do
+ ** something special with them (idk, try some other way to delete them?)
+ *******************************************************************************/
+public abstract class AbstractPostDeleteCustomizer implements TableCustomizerInterface
+{
+   protected DeleteInput deleteInput;
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public List<QRecord> postDelete(DeleteInput deleteInput, List<QRecord> records) throws QException
+   {
+      this.deleteInput = deleteInput;
+      return apply(records);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public abstract List<QRecord> apply(List<QRecord> records) throws QException;
+
+
+
+   /*******************************************************************************
+    ** Getter for deleteInput
+    **
+    *******************************************************************************/
+   public DeleteInput getDeleteInput()
+   {
+      return deleteInput;
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for deleteInput
+    **
+    *******************************************************************************/
+   public void setDeleteInput(DeleteInput deleteInput)
+   {
+      this.deleteInput = deleteInput;
+   }
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/AbstractPostInsertCustomizer.java

@@ -23,14 +23,26 @@ package com.kingsrook.qqq.backend.core.actions.customizers;
 
 
 import java.util.List;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
 
 
 /*******************************************************************************
+ ** Abstract class that a table can specify an implementation of, to provide
+ ** custom actions after an insert takes place.
  **
+ ** General implementation would be, to iterate over the records (the outputs of
+ ** the insert action), and look at their values:
+ ** - possibly adding Errors (`addError`) or Warnings (`addWarning`) to the records
+ ** - possibly throwing an exception - though doing so won't stop the update, and instead
+ **   will just set a warning on all of the updated records...
+ ** - doing "whatever else" you may want to do.
+ ** - returning the list of records (can be the input list) that you want to go back to the caller.
+ **
+ ** Note that the full insertInput is available as a field in this class.
  *******************************************************************************/
-public abstract class AbstractPostInsertCustomizer
+public abstract class AbstractPostInsertCustomizer implements TableCustomizerInterface
 {
    protected InsertInput insertInput;
 
@@ -39,7 +51,19 @@ public abstract class AbstractPostInsertCustomizer
    /*******************************************************************************
     **
     *******************************************************************************/
-   public abstract List<QRecord> apply(List<QRecord> records);
+   @Override
+   public List<QRecord> postInsert(InsertInput insertInput, List<QRecord> records) throws QException
+   {
+      this.insertInput = insertInput;
+      return (apply(records));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public abstract List<QRecord> apply(List<QRecord> records) throws QException;
 
 
 

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/AbstractPostQueryCustomizer.java

@@ -23,16 +23,29 @@ package com.kingsrook.qqq.backend.core.actions.customizers;
 
 
 import java.util.List;
-import com.kingsrook.qqq.backend.core.model.actions.AbstractTableActionInput;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.actions.tables.QueryOrGetInputInterface;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
 
 
 /*******************************************************************************
  **
  *******************************************************************************/
-public abstract class AbstractPostQueryCustomizer
+public abstract class AbstractPostQueryCustomizer implements TableCustomizerInterface
 {
-   protected AbstractTableActionInput input;
+   protected QueryOrGetInputInterface input;
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public List<QRecord> postQuery(QueryOrGetInputInterface queryInput, List<QRecord> records) throws QException
+   {
+      input = queryInput;
+      return apply(records);
+   }
 
 
 
@@ -47,7 +60,7 @@ public abstract class AbstractPostQueryCustomizer
     ** Getter for input
     **
     *******************************************************************************/
-   public AbstractTableActionInput getInput()
+   public QueryOrGetInputInterface getInput()
    {
       return (input);
    }
@@ -58,7 +71,7 @@ public abstract class AbstractPostQueryCustomizer
     ** Setter for input
     **
     *******************************************************************************/
-   public void setInput(AbstractTableActionInput input)
+   public void setInput(QueryOrGetInputInterface input)
    {
       this.input = input;
    }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/AbstractPostUpdateCustomizer.java

@@ -0,0 +1,144 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.actions.customizers;
+
+
+import java.io.Serializable;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+
+
+/*******************************************************************************
+ ** Abstract class that a table can specify an implementation of, to provide
+ ** custom actions after an update takes place.
+ **
+ ** General implementation would be, to iterate over the records (the outputs of
+ ** the update action), and look at their values:
+ ** - possibly adding Errors (`addError`) or Warnings (`addWarning`) to the records?
+ ** - possibly throwing an exception - though doing so won't stop the update, and instead
+ **   will just set a warning on all of the updated records...
+ ** - doing "whatever else" you may want to do.
+ ** - returning the list of records (can be the input list) that you want to go back to the caller.
+ **
+ ** Note that the full updateInput is available as a field in this class, and the
+ ** "old records" (e.g., with values freshly fetched from the backend) will be
+ ** available (if the backend supports it) - both as a list (`getOldRecordList`)
+ ** and as a memoized (by this class) map of primaryKey to record (`getOldRecordMap`).
+ *******************************************************************************/
+public abstract class AbstractPostUpdateCustomizer implements TableCustomizerInterface
+{
+   protected UpdateInput   updateInput;
+   protected List<QRecord> oldRecordList;
+
+   private Map<Serializable, QRecord> oldRecordMap = null;
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public List<QRecord> postUpdate(UpdateInput updateInput, List<QRecord> records, Optional<List<QRecord>> oldRecordList) throws QException
+   {
+      this.updateInput = updateInput;
+      this.oldRecordList = oldRecordList.orElse(null);
+      return apply(records);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public abstract List<QRecord> apply(List<QRecord> records) throws QException;
+
+
+
+   /*******************************************************************************
+    ** Getter for updateInput
+    **
+    *******************************************************************************/
+   public UpdateInput getUpdateInput()
+   {
+      return updateInput;
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for updateInput
+    **
+    *******************************************************************************/
+   public void setUpdateInput(UpdateInput updateInput)
+   {
+      this.updateInput = updateInput;
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public void setOldRecordList(List<QRecord> oldRecordList)
+   {
+      this.oldRecordList = oldRecordList;
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public List<QRecord> getOldRecordList()
+   {
+      return oldRecordList;
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   protected Map<Serializable, QRecord> getOldRecordMap()
+   {
+      if(oldRecordMap == null)
+      {
+         oldRecordMap = new HashMap<>();
+
+         if(oldRecordList != null && updateInput != null)
+         {
+            for(QRecord qRecord : oldRecordList)
+            {
+               oldRecordMap.put(qRecord.getValue(updateInput.getTable().getPrimaryKeyField()), qRecord);
+            }
+         }
+      }
+
+      return (oldRecordMap);
+   }
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/AbstractPreDeleteCustomizer.java

@@ -0,0 +1,132 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.actions.customizers;
+
+
+import java.util.List;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+
+
+/*******************************************************************************
+ ** Abstract class that a table can specify an implementation of, to provide
+ ** custom actions before a delete takes place.
+ **
+ ** It's important for implementations to be aware of the isPreview field, which
+ ** is set to true when the code is running to give users advice, e.g., on a review
+ ** screen - vs. being false when the action is ACTUALLY happening.  So, if you're doing
+ ** things like storing data, you don't want to do that if isPreview is true!!
+ **
+ ** General implementation would be, to iterate over the records (which the DeleteAction
+ ** would look up based on the inputs to the delete action), and look at their values:
+ ** - possibly adding Errors (`addError`) or Warnings (`addWarning`) to the records
+ ** - possibly throwing an exception - if you really don't want the delete operation to continue.
+ ** - doing "whatever else" you may want to do.
+ ** - returning the list of records (can be the input list) - this is how errors
+ **   and warnings are propagated to the DeleteAction.  Note that any records with
+ **   an error will NOT proceed to the backend's delete interface - but those with
+ **   warnings will.
+ **
+ ** Note that the full deleteInput is available as a field in this class.
+ **
+ *******************************************************************************/
+public abstract class AbstractPreDeleteCustomizer implements TableCustomizerInterface
+{
+   protected DeleteInput deleteInput;
+
+   protected boolean isPreview = false;
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public List<QRecord> preDelete(DeleteInput deleteInput, List<QRecord> records, boolean isPreview) throws QException
+   {
+      this.deleteInput = deleteInput;
+      this.isPreview = isPreview;
+      return apply(records);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public abstract List<QRecord> apply(List<QRecord> records) throws QException;
+
+
+
+   /*******************************************************************************
+    ** Getter for deleteInput
+    **
+    *******************************************************************************/
+   public DeleteInput getDeleteInput()
+   {
+      return deleteInput;
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for deleteInput
+    **
+    *******************************************************************************/
+   public void setDeleteInput(DeleteInput deleteInput)
+   {
+      this.deleteInput = deleteInput;
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for isPreview
+    *******************************************************************************/
+   public boolean getIsPreview()
+   {
+      return (this.isPreview);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for isPreview
+    *******************************************************************************/
+   public void setIsPreview(boolean isPreview)
+   {
+      this.isPreview = isPreview;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for isPreview
+    *******************************************************************************/
+   public AbstractPreDeleteCustomizer withIsPreview(boolean isPreview)
+   {
+      this.isPreview = isPreview;
+      return (this);
+   }
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/AbstractPreInsertCustomizer.java

@@ -0,0 +1,165 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.actions.customizers;
+
+
+import java.util.List;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+
+
+/*******************************************************************************
+ ** Abstract class that a table can specify an implementation of, to provide
+ ** custom actions before an insert takes place.
+ **
+ ** It's important for implementations to be aware of the isPreview field, which
+ ** is set to true when the code is running to give users advice, e.g., on a review
+ ** screen - vs. being false when the action is ACTUALLY happening.  So, if you're doing
+ ** things like storing data, you don't want to do that if isPreview is true!!
+ **
+ ** General implementation would be, to iterate over the records (the inputs to
+ ** the insert action), and look at their values:
+ ** - possibly adding Errors (`addError`) or Warnings (`addWarning`) to the records
+ ** - possibly manipulating values (`setValue`)
+ ** - possibly throwing an exception - if you really don't want the insert operation to continue.
+ ** - doing "whatever else" you may want to do.
+ ** - returning the list of records (can be the input list) that you want to go on to the backend implementation class.
+ **
+ ** Note that the full insertInput is available as a field in this class.
+ *******************************************************************************/
+public abstract class AbstractPreInsertCustomizer implements TableCustomizerInterface
+{
+   protected InsertInput insertInput;
+
+   protected boolean isPreview = false;
+
+
+
+   /////////////////////////////////////////////////////////////////////////////////
+   // allow the customizer to specify when it should be executed as part of the   //
+   // insert action.  default (per method in this class) is AFTER_ALL_VALIDATIONS //
+   /////////////////////////////////////////////////////////////////////////////////
+   public enum WhenToRun
+   {
+      BEFORE_ALL_VALIDATIONS,
+      BEFORE_UNIQUE_KEY_CHECKS,
+      BEFORE_REQUIRED_FIELD_CHECKS,
+      BEFORE_SECURITY_CHECKS,
+      AFTER_ALL_VALIDATIONS
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public List<QRecord> preInsert(InsertInput insertInput, List<QRecord> records, boolean isPreview) throws QException
+   {
+      this.insertInput = insertInput;
+      this.isPreview = isPreview;
+      return (apply(records));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public WhenToRun whenToRunPreInsert(InsertInput insertInput, boolean isPreview)
+   {
+      return getWhenToRun();
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public abstract List<QRecord> apply(List<QRecord> records) throws QException;
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public WhenToRun getWhenToRun()
+   {
+      return (WhenToRun.AFTER_ALL_VALIDATIONS);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for insertInput
+    **
+    *******************************************************************************/
+   public InsertInput getInsertInput()
+   {
+      return insertInput;
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for insertInput
+    **
+    *******************************************************************************/
+   public void setInsertInput(InsertInput insertInput)
+   {
+      this.insertInput = insertInput;
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for isPreview
+    *******************************************************************************/
+   public boolean getIsPreview()
+   {
+      return (this.isPreview);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for isPreview
+    *******************************************************************************/
+   public void setIsPreview(boolean isPreview)
+   {
+      this.isPreview = isPreview;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for isPreview
+    *******************************************************************************/
+   public AbstractPreInsertCustomizer withIsPreview(boolean isPreview)
+   {
+      this.isPreview = isPreview;
+      return (this);
+   }
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/AbstractPreUpdateCustomizer.java

@@ -0,0 +1,178 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.actions.customizers;
+
+
+import java.io.Serializable;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+
+
+/*******************************************************************************
+ ** Abstract class that a table can specify an implementation of, to provide
+ ** custom actions before an update takes place.
+ **
+ ** It's important for implementations to be aware of the isPreview field, which
+ ** is set to true when the code is running to give users advice, e.g., on a review
+ ** screen - vs. being false when the action is ACTUALLY happening.  So, if you're doing
+ ** things like storing data, you don't want to do that if isPreview is true!!
+ **
+ ** General implementation would be, to iterate over the records (the inputs to
+ ** the update action), and look at their values:
+ ** - possibly adding Errors (`addError`) or Warnings (`addWarning`) to the records
+ ** - possibly manipulating values (`setValue`)
+ ** - possibly throwing an exception - if you really don't want the update operation to continue.
+ ** - doing "whatever else" you may want to do.
+ ** - returning the list of records (can be the input list) that you want to go on to the backend implementation class.
+ **
+ ** Note that the full updateInput is available as a field in this class, and the
+ ** "old records" (e.g., with values freshly fetched from the backend) will be
+ ** available (if the backend supports it) - both as a list (`getOldRecordList`)
+ ** and as a memoized (by this class) map of primaryKey to record (`getOldRecordMap`).
+ *******************************************************************************/
+public abstract class AbstractPreUpdateCustomizer implements TableCustomizerInterface
+{
+   protected UpdateInput   updateInput;
+   protected List<QRecord> oldRecordList;
+   protected boolean       isPreview = false;
+
+   private Map<Serializable, QRecord> oldRecordMap = null;
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public List<QRecord> preUpdate(UpdateInput updateInput, List<QRecord> records, boolean isPreview, Optional<List<QRecord>> oldRecordList) throws QException
+   {
+      this.updateInput = updateInput;
+      this.isPreview = isPreview;
+      this.oldRecordList = oldRecordList.orElse(null);
+      return apply(records);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public abstract List<QRecord> apply(List<QRecord> records) throws QException;
+
+
+
+   /*******************************************************************************
+    ** Getter for updateInput
+    **
+    *******************************************************************************/
+   public UpdateInput getUpdateInput()
+   {
+      return updateInput;
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for updateInput
+    **
+    *******************************************************************************/
+   public void setUpdateInput(UpdateInput updateInput)
+   {
+      this.updateInput = updateInput;
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public void setOldRecordList(List<QRecord> oldRecordList)
+   {
+      this.oldRecordList = oldRecordList;
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public List<QRecord> getOldRecordList()
+   {
+      return oldRecordList;
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   protected Map<Serializable, QRecord> getOldRecordMap()
+   {
+      if(oldRecordMap == null)
+      {
+         oldRecordMap = new HashMap<>();
+         for(QRecord qRecord : oldRecordList)
+         {
+            oldRecordMap.put(qRecord.getValue(updateInput.getTable().getPrimaryKeyField()), qRecord);
+         }
+      }
+
+      return (oldRecordMap);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for isPreview
+    *******************************************************************************/
+   public boolean getIsPreview()
+   {
+      return (this.isPreview);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for isPreview
+    *******************************************************************************/
+   public void setIsPreview(boolean isPreview)
+   {
+      this.isPreview = isPreview;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for isPreview
+    *******************************************************************************/
+   public AbstractPreUpdateCustomizer withIsPreview(boolean isPreview)
+   {
+      this.isPreview = isPreview;
+      return (this);
+   }
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/ChildInserterPostInsertCustomizer.java

@@ -29,28 +29,30 @@ import java.util.List;
 import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
 import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.exceptions.QRuntimeException;
 import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertOutput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+import com.kingsrook.qqq.backend.core.model.statusmessages.QStatusMessage;
+import com.kingsrook.qqq.backend.core.model.statusmessages.QWarningMessage;
+import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
 
 
 /*******************************************************************************
  ** Standard/re-usable post-insert customizer, for the use case where, when we
  ** do an insert into table "parent", we want a record automatically inserted into
- ** table "child", and there's a foreign key in "parent", pointed at "child"
- ** e.g., named: "parent.childId".
+ ** table "child".  Optionally (based on RelationshipType), there can be a foreign
+ ** key in "parent", pointed at "child".  e.g., named: "parent.childId".
  **
- ** A similar use-case would have the foreign key in the child table - in which case,
- ** we could add a "Type" enum, plus abstract method to get our "Type", then logic
- ** to switch behavior based on type.  See existing type enum, but w/ only 1 case :)
  *******************************************************************************/
 public abstract class ChildInserterPostInsertCustomizer extends AbstractPostInsertCustomizer
 {
    public enum RelationshipType
    {
-      PARENT_POINTS_AT_CHILD
+      PARENT_POINTS_AT_CHILD,
+      CHILD_POINTS_AT_PARENT
    }
 
 
@@ -65,10 +67,17 @@ public abstract class ChildInserterPostInsertCustomizer extends AbstractPostInse
     *******************************************************************************/
    public abstract String getChildTableName();
 
+
+
    /*******************************************************************************
     **
     *******************************************************************************/
-   public abstract String getForeignKeyFieldName();
+   public String getForeignKeyFieldName()
+   {
+      return (null);
+   }
+
+
 
    /*******************************************************************************
     **
@@ -85,7 +94,7 @@ public abstract class ChildInserterPostInsertCustomizer extends AbstractPostInse
    {
       try
       {
-         List<QRecord>  rs               = new ArrayList<>();
+         List<QRecord>  rs               = records;
          List<QRecord>  childrenToInsert = new ArrayList<>();
          QTableMetaData table            = getInsertInput().getTable();
          QTableMetaData childTable       = getInsertInput().getInstance().getTable(getChildTableName());
@@ -94,12 +103,37 @@ public abstract class ChildInserterPostInsertCustomizer extends AbstractPostInse
          // iterate over the inserted records, building a list child records to insert //
          // for ones missing a value in the foreign key field.                         //
          ////////////////////////////////////////////////////////////////////////////////
-         for(QRecord record : records)
+         switch(getRelationshipType())
          {
-            if(record.getValue(getForeignKeyFieldName()) == null)
+            case PARENT_POINTS_AT_CHILD ->
             {
-               childrenToInsert.add(buildChildForRecord(record));
+               String foreignKeyFieldName = getForeignKeyFieldName();
+               try
+               {
+                  table.getField(foreignKeyFieldName);
+               }
+               catch(Exception e)
+               {
+                  throw new QRuntimeException("For RelationshipType.PARENT_POINTS_AT_CHILD, a valid foreignKeyFieldName in the parent table must be given.  "
+                     + "[" + foreignKeyFieldName + "] is not a valid field name in table [" + table.getName() + "]");
+               }
+
+               for(QRecord record : records)
+               {
+                  if(record.getValue(foreignKeyFieldName) == null)
+                  {
+                     childrenToInsert.add(buildChildForRecord(record));
+                  }
+               }
             }
+            case CHILD_POINTS_AT_PARENT ->
+            {
+               for(QRecord record : records)
+               {
+                  childrenToInsert.add(buildChildForRecord(record));
+               }
+            }
+            default -> throw new IllegalStateException("Unexpected value: " + getRelationshipType());
          }
 
          ///////////////////////////////////////////////////////////////////////////////////
@@ -120,38 +154,76 @@ public abstract class ChildInserterPostInsertCustomizer extends AbstractPostInse
          InsertOutput      insertOutput           = new InsertAction().execute(insertInput);
          Iterator<QRecord> insertedRecordIterator = insertOutput.getRecords().iterator();
 
+         /////////////////////////////////////////////////////////////////////////////////
+         // check for any errors when inserting the children, if any errors were found, //
+         // then set a warning in the parent with the details of the problem            //
+         /////////////////////////////////////////////////////////////////////////////////
+
          //////////////////////////////////////////////////////////////////////////////////////////////////////
+         // for the PARENT_POINTS_AT_CHILD relationship type:
          // iterate over the original list of records again - for any that need a child (e.g., are missing   //
          // foreign key), set their foreign key to a newly inserted child's key, and add them to be updated. //
          //////////////////////////////////////////////////////////////////////////////////////////////////////
-         List<QRecord> recordsToUpdate = new ArrayList<>();
-         for(QRecord record : records)
+         switch(getRelationshipType())
          {
-            Serializable primaryKey = record.getValue(table.getPrimaryKeyField());
-            if(record.getValue(getForeignKeyFieldName()) == null)
+            case PARENT_POINTS_AT_CHILD ->
             {
-               Serializable foreignKey = insertedRecordIterator.next().getValue(childTable.getPrimaryKeyField());
-               recordsToUpdate.add(new QRecord().withValue(table.getPrimaryKeyField(), primaryKey).withValue(getForeignKeyFieldName(), foreignKey));
-               record.setValue(getForeignKeyFieldName(), foreignKey);
-               rs.add(record);
+               rs = new ArrayList<>();
+               List<QRecord> recordsToUpdate = new ArrayList<>();
+               for(QRecord record : records)
+               {
+                  Serializable primaryKey = record.getValue(table.getPrimaryKeyField());
+                  if(record.getValue(getForeignKeyFieldName()) == null)
+                  {
+                     ///////////////////////////////////////////////////////////////////////////////////////////////////
+                     // get the corresponding child record, if it has any errors, set that as a warning in the parent //
+                     ///////////////////////////////////////////////////////////////////////////////////////////////////
+                     QRecord childRecord = insertedRecordIterator.next();
+                     if(CollectionUtils.nullSafeHasContents(childRecord.getErrors()))
+                     {
+                        for(QStatusMessage error : childRecord.getErrors())
+                        {
+                           record.addWarning(new QWarningMessage("Error creating child " + childTable.getLabel() + " (" + error.toString() + ")"));
+                        }
+                        rs.add(record);
+                        continue;
+                     }
+
+                     Serializable foreignKey = childRecord.getValue(childTable.getPrimaryKeyField());
+                     recordsToUpdate.add(new QRecord().withValue(table.getPrimaryKeyField(), primaryKey).withValue(getForeignKeyFieldName(), foreignKey));
+                     record.setValue(getForeignKeyFieldName(), foreignKey);
+                     rs.add(record);
+                  }
+                  else
+                  {
+                     rs.add(record);
+                  }
+               }
+
+               ////////////////////////////////////////////////////////////////////////////
+               // update the originally inserted records to reference their new children //
+               ////////////////////////////////////////////////////////////////////////////
+               UpdateInput updateInput = new UpdateInput();
+               updateInput.setTableName(getInsertInput().getTableName());
+               updateInput.setRecords(recordsToUpdate);
+               updateInput.setTransaction(this.insertInput.getTransaction());
+               new UpdateAction().execute(updateInput);
             }
-            else
+            case CHILD_POINTS_AT_PARENT ->
             {
-               rs.add(record);
+               ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+               // todo - some version of looking at the inserted children to confirm that they were inserted, and updating the parents with warnings if they weren't //
+               ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
             }
+            default -> throw new IllegalStateException("Unexpected value: " + getRelationshipType());
          }
 
-         ////////////////////////////////////////////////////////////////////////////
-         // update the originally inserted records to reference their new children //
-         ////////////////////////////////////////////////////////////////////////////
-         UpdateInput updateInput = new UpdateInput();
-         updateInput.setTableName(getInsertInput().getTableName());
-         updateInput.setRecords(recordsToUpdate);
-         updateInput.setTransaction(this.insertInput.getTransaction());
-         new UpdateAction().execute(updateInput);
-
          return (rs);
       }
+      catch(RuntimeException re)
+      {
+         throw (re);
+      }
       catch(Exception e)
       {
          throw new RuntimeException("Error inserting new child records for new parent records", e);

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/QCodeLoader.java

@@ -22,6 +22,7 @@
 package com.kingsrook.qqq.backend.core.actions.customizers;
 
 
+import java.lang.reflect.Constructor;
 import java.util.Optional;
 import java.util.function.Function;
 import com.kingsrook.qqq.backend.core.actions.automation.RecordAutomationHandler;
@@ -34,42 +35,35 @@ import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeType;
 import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSource;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.TableAutomationAction;
+import com.kingsrook.qqq.backend.core.utils.lambdas.UnsafeFunction;
+import com.kingsrook.qqq.backend.core.utils.memoization.Memoization;
 import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
 
 
 /*******************************************************************************
  ** Utility to load code for running QQQ customizers.
+ **
+ ** TODO - redo all to go through method that memoizes class & constructor
+ ** lookup.  That memoziation causes 1,000,000 such calls to go from ~500ms
+ ** to ~100ms.
  *******************************************************************************/
 public class QCodeLoader
 {
    private static final QLogger LOG = QLogger.getLogger(QCodeLoader.class);
 
-
-
-   /*******************************************************************************
-    **
-    *******************************************************************************/
-   public static <T, R> Optional<Function<T, R>> getTableCustomizerFunction(QTableMetaData table, String customizerName)
-   {
-      Optional<QCodeReference> codeReference = table.getCustomizer(customizerName);
-      if(codeReference.isPresent())
-      {
-         return (Optional.ofNullable(QCodeLoader.getFunction(codeReference.get())));
-      }
-      return (Optional.empty());
-   }
+   private static Memoization<String, Constructor<?>> constructorMemoization = new Memoization<>();
 
 
 
    /*******************************************************************************
     **
     *******************************************************************************/
-   public static <T> Optional<T> getTableCustomizer(Class<T> expectedClass, QTableMetaData table, String customizerName)
+   public static Optional<TableCustomizerInterface> getTableCustomizer(QTableMetaData table, String customizerName)
    {
       Optional<QCodeReference> codeReference = table.getCustomizer(customizerName);
       if(codeReference.isPresent())
       {
-         return (Optional.ofNullable(QCodeLoader.getAdHoc(expectedClass, codeReference.get())));
+         return (Optional.ofNullable(QCodeLoader.getAdHoc(TableCustomizerInterface.class, codeReference.get())));
       }
       return (Optional.empty());
    }
@@ -102,7 +96,7 @@ public class QCodeLoader
       }
       catch(Exception e)
       {
-         LOG.error("Error initializing customizer", logPair("codeReference", codeReference), e);
+         LOG.error("Error initializing customizer", e, logPair("codeReference", codeReference));
 
          //////////////////////////////////////////////////////////////////////////////////////////////////////////
          // return null here - under the assumption that during normal run-time operations, we'll never hit here //
@@ -141,7 +135,7 @@ public class QCodeLoader
       }
       catch(Exception e)
       {
-         LOG.error("Error initializing customizer", logPair("codeReference", codeReference), e);
+         LOG.error("Error initializing customizer", e, logPair("codeReference", codeReference));
 
          //////////////////////////////////////////////////////////////////////////////////////////////////////////
          // return null here - under the assumption that during normal run-time operations, we'll never hit here //
@@ -175,12 +169,25 @@ public class QCodeLoader
 
       try
       {
-         Class<?> customizerClass = Class.forName(codeReference.getName());
-         return ((T) customizerClass.getConstructor().newInstance());
+         Optional<Constructor<?>> constructor = constructorMemoization.getResultThrowing(codeReference.getName(), (UnsafeFunction<String, Constructor<?>, Exception>) s ->
+         {
+            Class<?> customizerClass = Class.forName(codeReference.getName());
+            return customizerClass.getConstructor();
+         });
+
+         if(constructor.isPresent())
+         {
+            return ((T) constructor.get().newInstance());
+         }
+         else
+         {
+            LOG.error("Could not get constructor for code reference", logPair("codeReference", codeReference));
+            return (null);
+         }
       }
       catch(Exception e)
       {
-         LOG.error("Error initializing customizer", logPair("codeReference", codeReference), e);
+         LOG.error("Error initializing customizer", e, logPair("codeReference", codeReference));
 
          //////////////////////////////////////////////////////////////////////////////////////////////////////////
          // return null here - under the assumption that during normal run-time operations, we'll never hit here //

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/RecordCustomizerUtilityInterface.java

@@ -0,0 +1,149 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2023.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.actions.customizers;
+
+
+import java.io.Serializable;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Objects;
+import com.kingsrook.qqq.backend.core.context.QContext;
+import com.kingsrook.qqq.backend.core.logging.QLogger;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+import com.kingsrook.qqq.backend.core.model.statusmessages.BadInputStatusMessage;
+import com.kingsrook.qqq.backend.core.utils.StringUtils;
+import com.kingsrook.qqq.backend.core.utils.ValueUtils;
+import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
+
+
+/*******************************************************************************
+ ** Interface with utility methods that pre insert/update/delete customizers
+ ** may want to use.
+ *******************************************************************************/
+public interface RecordCustomizerUtilityInterface
+{
+   QLogger LOG = QLogger.getLogger(RecordCustomizerUtilityInterface.class);
+
+
+
+   /*******************************************************************************
+    ** Container for an old value and a new value.
+    *******************************************************************************/
+   @SuppressWarnings("checkstyle:MethodName")
+   record Change(Serializable oldValue, Serializable newValue)
+   {
+   }
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   default Map<String, Change> getChanges(String tableName, QRecord oldRecord, QRecord newRecord)
+   {
+      Map<String, Change> rs = new HashMap<>();
+
+      QTableMetaData table = QContext.getQInstance().getTable(tableName);
+      for(Map.Entry<String, Serializable> entry : newRecord.getValues().entrySet())
+      {
+         String       fieldName = entry.getKey();
+         Serializable newValue  = entry.getValue();
+         Serializable oldValue  = oldRecord.getValue(fieldName);
+
+         try
+         {
+            QFieldMetaData field         = table.getField(fieldName);
+            Serializable   newTypedValue = ValueUtils.getValueAsFieldType(field.getType(), newValue);
+            Serializable   oldTypedValue = ValueUtils.getValueAsFieldType(field.getType(), oldValue);
+
+            if(!Objects.equals(oldTypedValue, newTypedValue))
+            {
+               rs.put(fieldName, new Change(oldTypedValue, newTypedValue));
+            }
+         }
+         catch(Exception e)
+         {
+            LOG.info("Error getting a value as field's type", e, logPair("fieldName", fieldName), logPair("oldValue", oldValue), logPair("newValue", newValue));
+         }
+      }
+
+      return (rs);
+   }
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   default void errorIfNoValue(Serializable value, QRecord record, String errorMessage)
+   {
+      errorIf(!StringUtils.hasContent(ValueUtils.getValueAsString(value)), record, errorMessage);
+   }
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   default void errorIfEditedValue(QRecord oldRecord, QRecord newRecord, String fieldName, String errorMessage)
+   {
+      if(newRecord.getValues().containsKey(fieldName))
+      {
+         errorIf(isChangedValue(oldRecord.getValue(fieldName), newRecord.getValue(fieldName)), newRecord, errorMessage);
+      }
+   }
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   default boolean isChangedValue(Serializable oldValue, Serializable newValue)
+   {
+      //////////////////////////////////////////////
+      // todo - probably ... some type "coercion" //
+      //////////////////////////////////////////////
+      return (!Objects.equals(oldValue, newValue));
+   }
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   default void errorIfAnyValue(Serializable value, QRecord record, String errorMessage)
+   {
+      if(StringUtils.hasContent(ValueUtils.getValueAsString(value)))
+      {
+         record.addError(new BadInputStatusMessage(errorMessage));
+      }
+   }
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   default void errorIf(boolean condition, QRecord record, String errorMessage)
+   {
+      if(condition)
+      {
+         record.addError(new BadInputStatusMessage(errorMessage));
+      }
+   }
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/TableCustomizerInterface.java

@@ -0,0 +1,202 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.actions.customizers;
+
+
+import java.util.List;
+import java.util.Optional;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.logging.QLogger;
+import com.kingsrook.qqq.backend.core.model.actions.tables.QueryOrGetInputInterface;
+import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
+
+
+/*******************************************************************************
+ ** Common interface used by all (core) TableCustomizer types (e.g., post-query,
+ ** and {pre,post}-{insert,update,delete}.
+ **
+ ** Note that the abstract-base classes for each action still exist, though have
+ ** been back-ported to be implementors of this interface.  The action classes
+ ** will now expect this type, and call this type's methods.
+ **
+ *******************************************************************************/
+public interface TableCustomizerInterface
+{
+   QLogger LOG = QLogger.getLogger(TableCustomizerInterface.class);
+
+
+   /*******************************************************************************
+    ** custom actions to run after a query (or get!) takes place.
+    **
+    *******************************************************************************/
+   default List<QRecord> postQuery(QueryOrGetInputInterface queryInput, List<QRecord> records) throws QException
+   {
+      LOG.info("A default implementation of postQuery is running...  Probably not expected!", logPair("tableName", queryInput.getTableName()));
+      return (records);
+   }
+
+
+   /*******************************************************************************
+    ** custom actions before an insert takes place.
+    **
+    ** It's important for implementations to be aware of the isPreview field, which
+    ** is set to true when the code is running to give users advice, e.g., on a review
+    ** screen - vs. being false when the action is ACTUALLY happening.  So, if you're doing
+    ** things like storing data, you don't want to do that if isPreview is true!!
+    **
+    ** General implementation would be, to iterate over the records (the inputs to
+    ** the insert action), and look at their values:
+    ** - possibly adding Errors (`addError`) or Warnings (`addWarning`) to the records
+    ** - possibly manipulating values (`setValue`)
+    ** - possibly throwing an exception - if you really don't want the insert operation to continue.
+    ** - doing "whatever else" you may want to do.
+    ** - returning the list of records (can be the input list) that you want to go on to the backend implementation class.
+    *******************************************************************************/
+   default List<QRecord> preInsert(InsertInput insertInput, List<QRecord> records, boolean isPreview) throws QException
+   {
+      LOG.info("A default implementation of preInsert is running...  Probably not expected!", logPair("tableName", insertInput.getTableName()));
+      return (records);
+   }
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   default AbstractPreInsertCustomizer.WhenToRun whenToRunPreInsert(InsertInput insertInput, boolean isPreview)
+   {
+      return (AbstractPreInsertCustomizer.WhenToRun.AFTER_ALL_VALIDATIONS);
+   }
+
+
+   /*******************************************************************************
+    ** custom actions after an insert takes place.
+    **
+    ** General implementation would be, to iterate over the records (the outputs of
+    ** the insert action), and look at their values:
+    ** - possibly adding Errors (`addError`) or Warnings (`addWarning`) to the records
+    ** - possibly throwing an exception - though doing so won't stop the update, and instead
+    **   will just set a warning on all of the updated records...
+    ** - doing "whatever else" you may want to do.
+    ** - returning the list of records (can be the input list) that you want to go back to the caller.
+    *******************************************************************************/
+   default List<QRecord> postInsert(InsertInput insertInput, List<QRecord> records) throws QException
+   {
+      LOG.info("A default implementation of postInsert is running...  Probably not expected!", logPair("tableName", insertInput.getTableName()));
+      return (records);
+   }
+
+
+   /*******************************************************************************
+    ** custom actions before an update takes place.
+    **
+    ** It's important for implementations to be aware of the isPreview field, which
+    ** is set to true when the code is running to give users advice, e.g., on a review
+    ** screen - vs. being false when the action is ACTUALLY happening.  So, if you're doing
+    ** things like storing data, you don't want to do that if isPreview is true!!
+    **
+    ** General implementation would be, to iterate over the records (the inputs to
+    ** the update action), and look at their values:
+    ** - possibly adding Errors (`addError`) or Warnings (`addWarning`) to the records
+    ** - possibly manipulating values (`setValue`)
+    ** - possibly throwing an exception - if you really don't want the update operation to continue.
+    ** - doing "whatever else" you may want to do.
+    ** - returning the list of records (can be the input list) that you want to go on to the backend implementation class.
+    **
+    ** Note, "old records" (e.g., with values freshly fetched from the backend) will be
+    ** available (if the backend supports it)
+    *******************************************************************************/
+   default List<QRecord> preUpdate(UpdateInput updateInput, List<QRecord> records, boolean isPreview, Optional<List<QRecord>> oldRecordList) throws QException
+   {
+      LOG.info("A default implementation of preUpdate is running...  Probably not expected!", logPair("tableName", updateInput.getTableName()));
+      return (records);
+   }
+
+
+   /*******************************************************************************
+    ** custom actions after an update takes place.
+    **
+    ** General implementation would be, to iterate over the records (the outputs of
+    ** the update action), and look at their values:
+    ** - possibly adding Errors (`addError`) or Warnings (`addWarning`) to the records?
+    ** - possibly throwing an exception - though doing so won't stop the update, and instead
+    **   will just set a warning on all of the updated records...
+    ** - doing "whatever else" you may want to do.
+    ** - returning the list of records (can be the input list) that you want to go back to the caller.
+    **
+    ** Note, "old records" (e.g., with values freshly fetched from the backend) will be
+    ** available (if the backend supports it).
+    *******************************************************************************/
+   default List<QRecord> postUpdate(UpdateInput updateInput, List<QRecord> records, Optional<List<QRecord>> oldRecordList) throws QException
+   {
+      LOG.info("A default implementation of postUpdate is running...  Probably not expected!", logPair("tableName", updateInput.getTableName()));
+      return (records);
+   }
+
+
+   /*******************************************************************************
+    ** Custom actions before a delete takes place.
+    **
+    ** It's important for implementations to be aware of the isPreview param, which
+    ** is set to true when the code is running to give users advice, e.g., on a review
+    ** screen - vs. being false when the action is ACTUALLY happening.  So, if you're doing
+    ** things like storing data, you don't want to do that if isPreview is true!!
+    **
+    ** General implementation would be, to iterate over the records (which the DeleteAction
+    ** would look up based on the inputs to the delete action), and look at their values:
+    ** - possibly adding Errors (`addError`) or Warnings (`addWarning`) to the records
+    ** - possibly throwing an exception - if you really don't want the delete operation to continue.
+    ** - doing "whatever else" you may want to do.
+    ** - returning the list of records (can be the input list) - this is how errors
+    **   and warnings are propagated to the DeleteAction.  Note that any records with
+    **   an error will NOT proceed to the backend's delete interface - but those with
+    **   warnings will.
+    *******************************************************************************/
+   default List<QRecord> preDelete(DeleteInput deleteInput, List<QRecord> records, boolean isPreview) throws QException
+   {
+      LOG.info("A default implementation of preDelete is running...  Probably not expected!", logPair("tableName", deleteInput.getTableName()));
+      return (records);
+   }
+
+
+   /*******************************************************************************
+    ** Custom actions after a delete takes place.
+    **
+    ** General implementation would be, to iterate over the records (ones which didn't
+    ** have a delete error), and look at their values:
+    ** - possibly adding Errors (`addError`) or Warnings (`addWarning`) to the records?
+    ** - possibly throwing an exception - though doing so won't stop the delete, and instead
+    **   will just set a warning on all of the deleted records...
+    ** - doing "whatever else" you may want to do.
+    ** - returning the list of records (can be the input list) that you want to go back
+    **   to the caller - this is how errors and warnings are propagated .
+    *******************************************************************************/
+   default List<QRecord> postDelete(DeleteInput deleteInput, List<QRecord> records) throws QException
+   {
+      LOG.info("A default implementation of postDelete is running...  Probably not expected!", logPair("tableName", deleteInput.getTableName()));
+      return (records);
+   }
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/TableCustomizers.java

@@ -26,33 +26,30 @@ package com.kingsrook.qqq.backend.core.actions.customizers;
  ** Enum definition of possible table customizers - "roles" for custom code that
  ** can be applied to tables.
  **
- ** Works with TableCustomizer (singular version of this name) objects, during
- ** instance validation, to provide validation of the referenced code (and to
- ** make such validation from sub-backend-modules possible in the future).
- **
- ** The idea of the 3rd argument here is to provide a way that we can enforce
- ** the type-parameters for the custom code.  E.g., if it's a Function - how
- ** can we check at run-time that the type-params are correct?  We couldn't find
- ** how to do this "reflectively", so we can instead try to run the custom code,
- ** passing it objects of the type that this customizer expects, and a validation
- ** error will raise upon ClassCastException...  This maybe could improve!
  *******************************************************************************/
 public enum TableCustomizers
 {
-   POST_QUERY_RECORD(new TableCustomizer("postQueryRecord", AbstractPostQueryCustomizer.class)),
-   POST_INSERT_RECORD(new TableCustomizer("postInsertRecord", AbstractPostInsertCustomizer.class));
+   POST_QUERY_RECORD("postQueryRecord", TableCustomizerInterface.class),
+   PRE_INSERT_RECORD("preInsertRecord", TableCustomizerInterface.class),
+   POST_INSERT_RECORD("postInsertRecord", TableCustomizerInterface.class),
+   PRE_UPDATE_RECORD("preUpdateRecord", TableCustomizerInterface.class),
+   POST_UPDATE_RECORD("postUpdateRecord", TableCustomizerInterface.class),
+   PRE_DELETE_RECORD("preDeleteRecord", TableCustomizerInterface.class),
+   POST_DELETE_RECORD("postDeleteRecord", TableCustomizerInterface.class);
 
 
-   private final TableCustomizer tableCustomizer;
+   private final String   role;
+   private final Class<?> expectedType;
 
 
 
    /*******************************************************************************
     **
     *******************************************************************************/
-   TableCustomizers(TableCustomizer tableCustomizer)
+   TableCustomizers(String role, Class<?> expectedType)
    {
-      this.tableCustomizer = tableCustomizer;
+      this.role = role;
+      this.expectedType = expectedType;
    }
 
 
@@ -65,7 +62,7 @@ public enum TableCustomizers
    {
       for(TableCustomizers value : values())
       {
-         if(value.tableCustomizer.getRole().equals(name))
+         if(value.role.equals(name))
          {
             return (value);
          }
@@ -76,24 +73,23 @@ public enum TableCustomizers
 
 
 
-   /*******************************************************************************
-    ** Getter for tableCustomizer
-    **
-    *******************************************************************************/
-   public TableCustomizer getTableCustomizer()
-   {
-      return tableCustomizer;
-   }
-
-
-
    /*******************************************************************************
     ** get the role from the tableCustomizer
     **
     *******************************************************************************/
    public String getRole()
    {
-      return (tableCustomizer.getRole());
+      return (role);
    }
 
+
+
+   /*******************************************************************************
+    ** Getter for expectedType
+    **
+    *******************************************************************************/
+   public Class<?> getExpectedType()
+   {
+      return expectedType;
+   }
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/dashboard/AbstractHTMLWidgetRenderer.java

@@ -41,6 +41,7 @@ import com.kingsrook.qqq.backend.core.model.actions.widgets.RenderWidgetInput;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.DisplayFormat;
 import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
 import com.kingsrook.qqq.backend.core.utils.JsonUtils;
+import com.kingsrook.qqq.backend.core.utils.QQueryFilterDeduper;
 import com.kingsrook.qqq.backend.core.utils.StringUtils;
 
 
@@ -176,11 +177,13 @@ public abstract class AbstractHTMLWidgetRenderer extends AbstractWidgetRenderer
       {
          return (totalString);
       }
+      filter = QQueryFilterDeduper.dedupeFilter(filter);
       return ("<a href='" + tablePath + "?filter=" + JsonUtils.toJson(filter) + "'>" + totalString + "</a>");
    }
 
 
 
+
    /*******************************************************************************
     **
     *******************************************************************************/
@@ -192,6 +195,7 @@ public abstract class AbstractHTMLWidgetRenderer extends AbstractWidgetRenderer
          return;
       }
 
+      filter = QQueryFilterDeduper.dedupeFilter(filter);
       urls.add(tablePath + "?filter=" + JsonUtils.toJson(filter));
    }
 
@@ -208,6 +212,7 @@ public abstract class AbstractHTMLWidgetRenderer extends AbstractWidgetRenderer
          return (null);
       }
 
+      filter = QQueryFilterDeduper.dedupeFilter(filter);
       return (tablePath + "?filter=" + JsonUtils.toJson(filter));
    }
 
@@ -216,7 +221,7 @@ public abstract class AbstractHTMLWidgetRenderer extends AbstractWidgetRenderer
    /*******************************************************************************
     **
     *******************************************************************************/
-   public static String linkTableFilter(RenderWidgetInput input, String tableName, QQueryFilter filter) throws QException
+   public static String linkTableFilter(String tableName, QQueryFilter filter) throws QException
    {
       String tablePath = QContext.getQInstance().getTablePath(tableName);
       if(tablePath == null)
@@ -224,6 +229,7 @@ public abstract class AbstractHTMLWidgetRenderer extends AbstractWidgetRenderer
          return (null);
       }
 
+      filter = QQueryFilterDeduper.dedupeFilter(filter);
       return (tablePath + "?filter=" + URLEncoder.encode(JsonUtils.toJson(filter), Charset.defaultCharset()));
    }
 
@@ -232,9 +238,9 @@ public abstract class AbstractHTMLWidgetRenderer extends AbstractWidgetRenderer
    /*******************************************************************************
     **
     *******************************************************************************/
-   public static String aHrefTableFilterNoOfRecords(RenderWidgetInput input, String tableName, QQueryFilter filter, Integer noOfRecords, String singularLabel, String pluralLabel) throws QException
+   public static String aHrefTableFilterNoOfRecords(String tableName, QQueryFilter filter, Integer noOfRecords, String singularLabel, String pluralLabel) throws QException
    {
-      return (aHrefTableFilterNoOfRecords(input, tableName, filter, noOfRecords, singularLabel, pluralLabel, false));
+      return (aHrefTableFilterNoOfRecords(tableName, filter, noOfRecords, singularLabel, pluralLabel, false));
    }
 
 
@@ -242,7 +248,7 @@ public abstract class AbstractHTMLWidgetRenderer extends AbstractWidgetRenderer
    /*******************************************************************************
     **
     *******************************************************************************/
-   public static String aHrefTableFilterNoOfRecords(RenderWidgetInput input, String tableName, QQueryFilter filter, Integer noOfRecords, String singularLabel, String pluralLabel, boolean onlyLinkCount) throws QException
+   public static String aHrefTableFilterNoOfRecords(String tableName, QQueryFilter filter, Integer noOfRecords, String singularLabel, String pluralLabel, boolean onlyLinkCount) throws QException
    {
       String plural      = StringUtils.plural(noOfRecords, singularLabel, pluralLabel);
       String countString = QValueFormatter.formatValue(DisplayFormat.COMMAS, noOfRecords);
@@ -253,7 +259,7 @@ public abstract class AbstractHTMLWidgetRenderer extends AbstractWidgetRenderer
          return (countString + displayText);
       }
 
-      String href = linkTableFilter(input, tableName, filter);
+      String href = linkTableFilter(tableName, filter);
       if(onlyLinkCount)
       {
          return ("<a href=\"" + href + "\">" + countString + "</a>" + displayText);
@@ -269,7 +275,7 @@ public abstract class AbstractHTMLWidgetRenderer extends AbstractWidgetRenderer
    /*******************************************************************************
     **
     *******************************************************************************/
-   public static String aHrefViewRecord(RenderWidgetInput input, String tableName, Serializable id, String linkText) throws QException
+   public static String aHrefViewRecord(String tableName, Serializable id, String linkText) throws QException
    {
       String tablePath = QContext.getQInstance().getTablePath(tableName);
       if(tablePath == null)
@@ -277,7 +283,7 @@ public abstract class AbstractHTMLWidgetRenderer extends AbstractWidgetRenderer
          return (linkText);
       }
 
-      return ("<a href=\"" + linkRecordView(input, tableName, id) + "\">" + linkText + "</a>");
+      return ("<a href=\"" + linkRecordView(tableName, id) + "\">" + linkText + "</a>");
    }
 
 
@@ -296,7 +302,7 @@ public abstract class AbstractHTMLWidgetRenderer extends AbstractWidgetRenderer
    /*******************************************************************************
     **
     *******************************************************************************/
-   public static String linkRecordView(AbstractActionInput input, String tableName, Serializable recordId) throws QException
+   public static String linkRecordView(String tableName, Serializable recordId) throws QException
    {
       String tablePath = QContext.getQInstance().getTablePath(tableName);
       if(tablePath == null)
@@ -326,6 +332,7 @@ public abstract class AbstractHTMLWidgetRenderer extends AbstractWidgetRenderer
       }
 
       String tablePath = QContext.getQInstance().getTablePath(tableName);
+      filter = QQueryFilterDeduper.dedupeFilter(filter);
       return (tablePath + "/" + processName + "?recordsParam=filterJSON&filterJSON=" + URLEncoder.encode(JsonUtils.toJson(filter), StandardCharsets.UTF_8));
    }
 

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/dashboard/widgets/AbstractWidgetRenderer.java

@@ -42,10 +42,12 @@ import com.kingsrook.qqq.backend.core.model.actions.widgets.RenderWidgetOutput;
 import com.kingsrook.qqq.backend.core.model.dashboard.widgets.QWidgetData;
 import com.kingsrook.qqq.backend.core.model.metadata.dashboard.QWidgetMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.dashboard.WidgetDropdownData;
+import com.kingsrook.qqq.backend.core.model.metadata.dashboard.WidgetDropdownType;
 import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValue;
 import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSource;
 import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
 import com.kingsrook.qqq.backend.core.utils.StringUtils;
+import com.kingsrook.qqq.backend.core.utils.collections.MapBuilder;
 
 
 /*******************************************************************************
@@ -71,80 +73,104 @@ public abstract class AbstractWidgetRenderer
     *******************************************************************************/
    protected boolean setupDropdowns(RenderWidgetInput input, QWidgetMetaData metaData, QWidgetData widgetData) throws QException
    {
-      List<List<Map<String, String>>> pvsData                   = new ArrayList<>();
-      List<String>                    pvsLabels                 = new ArrayList<>();
-      List<String>                    pvsNames                  = new ArrayList<>();
+      List<List<Map<String, String>>> dataList                  = new ArrayList<>();
+      List<String>                    labelList                 = new ArrayList<>();
+      List<String>                    nameList                  = new ArrayList<>();
       List<String>                    missingRequiredSelections = new ArrayList<>();
       for(WidgetDropdownData dropdownData : CollectionUtils.nonNullList(metaData.getDropdowns()))
       {
-         String               possibleValueSourceName = dropdownData.getPossibleValueSourceName();
-         QPossibleValueSource possibleValueSource     = input.getInstance().getPossibleValueSource(possibleValueSourceName);
-
-         /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-         // this looks complicated, but is just look for a label in the dropdown data and if found use it,                                                                                  //
-         // otherwise look for label in PVS and if found use that, otherwise just use the PVS name                                                                                          //
-         /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-         String pvsLabel = dropdownData.getLabel() != null ? dropdownData.getLabel() : (possibleValueSource.getLabel() != null ? possibleValueSource.getLabel() : possibleValueSourceName);
-         pvsLabels.add(pvsLabel);
-         pvsNames.add(possibleValueSourceName);
-
-         SearchPossibleValueSourceInput pvsInput = new SearchPossibleValueSourceInput();
-         pvsInput.setPossibleValueSourceName(possibleValueSourceName);
-
-         if(dropdownData.getForeignKeyFieldName() != null)
+         if(WidgetDropdownType.DATE_PICKER.equals(dropdownData.getType()))
          {
-            ////////////////////////////////////////
-            // look for an id in the query params //
-            ////////////////////////////////////////
-            Integer id = null;
-            if(input.getQueryParams() != null && input.getQueryParams().containsKey("id") && StringUtils.hasContent(input.getQueryParams().get("id")))
+            String name = dropdownData.getName();
+            nameList.add(name);
+            labelList.add(dropdownData.getLabel());
+            dataList.add(new ArrayList<>());
+
+            ////////////////////////////////////////////////////////////////////////////////////////////////////////////
+            // sure that something has been selected, and if not, display a message that a selection needs made       //
+            ////////////////////////////////////////////////////////////////////////////////////////////////////////////
+            if(dropdownData.getIsRequired())
             {
-               id = Integer.parseInt(input.getQueryParams().get("id"));
-            }
-            if(id != null)
-            {
-               pvsInput.setDefaultQueryFilter(new QQueryFilter().withCriteria(
-                  new QFilterCriteria(
-                     dropdownData.getForeignKeyFieldName(),
-                     QCriteriaOperator.EQUALS,
-                     id)));
+               if(!input.getQueryParams().containsKey(name) || !StringUtils.hasContent(input.getQueryParams().get(name)))
+               {
+                  missingRequiredSelections.add(dropdownData.getLabel());
+               }
             }
          }
-
-         SearchPossibleValueSourceOutput output = new SearchPossibleValueSourceAction().execute(pvsInput);
-
-         List<Map<String, String>> dropdownOptionList = new ArrayList<>();
-         pvsData.add(dropdownOptionList);
-
-         //////////////////////////////////////////
-         // sort results, dedupe, and add to map //
-         //////////////////////////////////////////
-         Set<String> exists = new HashSet<>();
-         output.getResults().removeIf(pvs -> !exists.add(pvs.getLabel()));
-         for(QPossibleValue<?> possibleValue : output.getResults())
+         else
          {
-            dropdownOptionList.add(Map.of(
-               "id", String.valueOf(possibleValue.getId()),
-               "label", possibleValue.getLabel()
-            ));
-         }
-
-         ////////////////////////////////////////////////////////////////////////////////////////////////////////////
-         // because we know the dropdowns and what the field names will be when something is selected, we can make //
-         // sure that something has been selected, and if not, display a message that a selection needs made       //
-         ////////////////////////////////////////////////////////////////////////////////////////////////////////////
-         if(dropdownData.getIsRequired())
-         {
-            if(!input.getQueryParams().containsKey(possibleValueSourceName) || !StringUtils.hasContent(input.getQueryParams().get(possibleValueSourceName)))
+            String possibleValueSourceName = dropdownData.getPossibleValueSourceName();
+            if(possibleValueSourceName != null)
             {
-               missingRequiredSelections.add(pvsLabel);
+               QPossibleValueSource possibleValueSource = input.getInstance().getPossibleValueSource(possibleValueSourceName);
+
+               /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+               // this looks complicated, but is just look for a label in the dropdown data and if found use it,                                                                                  //
+               // otherwise look for label in PVS and if found use that, otherwise just use the PVS name                                                                                          //
+               /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+               String pvsLabel = dropdownData.getLabel() != null ? dropdownData.getLabel() : (possibleValueSource.getLabel() != null ? possibleValueSource.getLabel() : possibleValueSourceName);
+               labelList.add(pvsLabel);
+               nameList.add(possibleValueSourceName);
+
+               SearchPossibleValueSourceInput pvsInput = new SearchPossibleValueSourceInput();
+               pvsInput.setPossibleValueSourceName(possibleValueSourceName);
+
+               if(dropdownData.getForeignKeyFieldName() != null)
+               {
+                  ////////////////////////////////////////
+                  // look for an id in the query params //
+                  ////////////////////////////////////////
+                  Integer id = null;
+                  if(input.getQueryParams() != null && input.getQueryParams().containsKey("id") && StringUtils.hasContent(input.getQueryParams().get("id")))
+                  {
+                     id = Integer.parseInt(input.getQueryParams().get("id"));
+                  }
+                  if(id != null)
+                  {
+                     pvsInput.setDefaultQueryFilter(new QQueryFilter().withCriteria(
+                        new QFilterCriteria(
+                           dropdownData.getForeignKeyFieldName(),
+                           QCriteriaOperator.EQUALS,
+                           id)));
+                  }
+               }
+
+               SearchPossibleValueSourceOutput output = new SearchPossibleValueSourceAction().execute(pvsInput);
+
+               List<Map<String, String>> dropdownOptionList = new ArrayList<>();
+               dataList.add(dropdownOptionList);
+
+               //////////////////////////////////////////
+               // sort results, dedupe, and add to map //
+               //////////////////////////////////////////
+               Set<String> exists = new HashSet<>();
+               output.getResults().removeIf(pvs -> !exists.add(pvs.getLabel()));
+               for(QPossibleValue<?> possibleValue : output.getResults())
+               {
+                  dropdownOptionList.add(MapBuilder.of(
+                     "id", String.valueOf(possibleValue.getId()),
+                     "label", possibleValue.getLabel()
+                  ));
+               }
+
+               ////////////////////////////////////////////////////////////////////////////////////////////////////////////
+               // because we know the dropdowns and what the field names will be when something is selected, we can make //
+               // sure that something has been selected, and if not, display a message that a selection needs made       //
+               ////////////////////////////////////////////////////////////////////////////////////////////////////////////
+               if(dropdownData.getIsRequired())
+               {
+                  if(!input.getQueryParams().containsKey(possibleValueSourceName) || !StringUtils.hasContent(input.getQueryParams().get(possibleValueSourceName)))
+                  {
+                     missingRequiredSelections.add(pvsLabel);
+                  }
+               }
             }
          }
       }
 
-      widgetData.setDropdownNameList(pvsNames);
-      widgetData.setDropdownLabelList(pvsLabels);
-      widgetData.setDropdownDataList(pvsData);
+      widgetData.setDropdownNameList(nameList);
+      widgetData.setDropdownLabelList(labelList);
+      widgetData.setDropdownDataList(dataList);
 
       ////////////////////////////////////////////////////////////////////////////////
       // if there are any missing required dropdowns, build up a message to display //

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/dashboard/widgets/Aggregate2DTableWidgetRenderer.java

@@ -0,0 +1,202 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.actions.dashboard.widgets;
+
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.LinkedHashMap;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import com.kingsrook.qqq.backend.core.actions.tables.AggregateAction;
+import com.kingsrook.qqq.backend.core.context.QContext;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.logging.QLogger;
+import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.Aggregate;
+import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateOperator;
+import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateOutput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateResult;
+import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.GroupBy;
+import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.QFilterOrderByAggregate;
+import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.QFilterOrderByGroupBy;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
+import com.kingsrook.qqq.backend.core.model.actions.widgets.RenderWidgetInput;
+import com.kingsrook.qqq.backend.core.model.actions.widgets.RenderWidgetOutput;
+import com.kingsrook.qqq.backend.core.model.dashboard.widgets.TableData;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+import com.kingsrook.qqq.backend.core.utils.StringUtils;
+import com.kingsrook.qqq.backend.core.utils.ValueUtils;
+
+
+/*******************************************************************************
+ ** Generic widget that does an aggregate query, and presents its results
+ ** as a table, using group-by values as both row & column labels.
+ *******************************************************************************/
+public class Aggregate2DTableWidgetRenderer extends AbstractWidgetRenderer
+{
+   private static final QLogger LOG = QLogger.getLogger(Aggregate2DTableWidgetRenderer.class);
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public RenderWidgetOutput render(RenderWidgetInput input) throws QException
+   {
+      Map<String, Serializable> values = input.getWidgetMetaData().getDefaultValues();
+
+      String         tableName   = ValueUtils.getValueAsString(values.get("tableName"));
+      String         valueField  = ValueUtils.getValueAsString(values.get("valueField"));
+      String         rowField    = ValueUtils.getValueAsString(values.get("rowField"));
+      String         columnField = ValueUtils.getValueAsString(values.get("columnField"));
+      QTableMetaData table       = QContext.getQInstance().getTable(tableName);
+
+      AggregateInput aggregateInput = new AggregateInput();
+      aggregateInput.setTableName(tableName);
+
+      // todo - allow input of "list of columns" (e.g., in case some miss sometimes, or as a version of filter)
+      // todo - max rows, max cols?
+
+      // todo - from input map
+      QQueryFilter filter = new QQueryFilter();
+      aggregateInput.setFilter(filter);
+
+      Aggregate aggregate = new Aggregate(valueField, AggregateOperator.COUNT);
+      aggregateInput.withAggregate(aggregate);
+
+      GroupBy rowGroupBy    = new GroupBy(table.getField(rowField));
+      GroupBy columnGroupBy = new GroupBy(table.getField(columnField));
+      aggregateInput.withGroupBy(rowGroupBy);
+      aggregateInput.withGroupBy(columnGroupBy);
+
+      String orderBys = ValueUtils.getValueAsString(values.get("orderBys"));
+      if(StringUtils.hasContent(orderBys))
+      {
+         for(String orderBy : orderBys.split(","))
+         {
+            switch(orderBy)
+            {
+               case "row" -> filter.addOrderBy(new QFilterOrderByGroupBy(rowGroupBy));
+               case "column" -> filter.addOrderBy(new QFilterOrderByGroupBy(columnGroupBy));
+               case "value" -> filter.addOrderBy(new QFilterOrderByAggregate(aggregate));
+               default -> LOG.warn("Unrecognized orderBy: " + orderBy);
+            }
+         }
+      }
+
+      AggregateOutput aggregateOutput = new AggregateAction().execute(aggregateInput);
+
+      Map<Serializable, Map<Serializable, Serializable>> data       = new LinkedHashMap<>();
+      Set<Serializable>                                  columnsSet = new LinkedHashSet<>();
+
+      for(AggregateResult result : aggregateOutput.getResults())
+      {
+         Serializable column = result.getGroupByValue(columnGroupBy);
+         Serializable row    = result.getGroupByValue(rowGroupBy);
+         Serializable value  = result.getAggregateValue(aggregate);
+
+         Map<Serializable, Serializable> rowMap = data.computeIfAbsent(row, (k) -> new LinkedHashMap<>());
+         rowMap.put(column, value);
+         columnsSet.add(column);
+      }
+
+      // todo - possible values from rows, cols
+
+      ////////////////////////////////////
+      // setup datastructures for table //
+      ////////////////////////////////////
+      List<Map<String, Object>> tableRows    = new ArrayList<>();
+      List<TableData.Column>    tableColumns = new ArrayList<>();
+      tableColumns.add(new TableData.Column("default", table.getField(rowField).getLabel(), "_row", "2fr", "left"));
+
+      for(Serializable column : columnsSet)
+      {
+         tableColumns.add(new TableData.Column("default", String.valueOf(column) /* todo display value */, String.valueOf(column), "1fr", "right"));
+      }
+
+      tableColumns.add(new TableData.Column("default", "Total", "_total", "1fr", "right"));
+
+      TableData tableData = new TableData(null, tableColumns, tableRows)
+         .withRowsPerPage(100)
+         .withFixedStickyLastRow(false)
+         .withHidePaginationDropdown(true);
+
+      Map<Serializable, Integer> columnSums = new HashMap<>();
+      int                        grandTotal = 0;
+      for(Map.Entry<Serializable, Map<Serializable, Serializable>> rowEntry : data.entrySet())
+      {
+         Map<String, Object> rowMap = new HashMap<>();
+         tableRows.add(rowMap);
+
+         rowMap.put("_row", rowEntry.getKey() /* todo display value */);
+         int rowTotal = 0;
+         for(Serializable column : columnsSet)
+         {
+            Serializable value = rowEntry.getValue().get(column);
+            if(value == null)
+            {
+               value = 0; // todo?
+            }
+
+            Integer valueAsInteger = Objects.requireNonNullElse(ValueUtils.getValueAsInteger(value), 0);
+            rowTotal += valueAsInteger;
+            columnSums.putIfAbsent(column, 0);
+            columnSums.put(column, columnSums.get(column) + valueAsInteger);
+
+            rowMap.put(String.valueOf(column), value); // todo format commas?
+         }
+
+         rowMap.put("_total", rowTotal);
+         grandTotal += rowTotal;
+      }
+
+      ///////////////
+      // total row //
+      ///////////////
+      Map<String, Object> totalRowMap = new HashMap<>();
+      tableRows.add(totalRowMap);
+
+      totalRowMap.put("_row", "Total");
+      int rowTotal = 0;
+      for(Serializable column : columnsSet)
+      {
+         Serializable value = columnSums.get(column);
+         if(value == null)
+         {
+            value = 0; // todo?
+         }
+
+         totalRowMap.put(String.valueOf(column), value); // todo format commas?
+      }
+
+      totalRowMap.put("_total", grandTotal);
+
+      return (new RenderWidgetOutput(tableData));
+   }
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/dashboard/widgets/ChildRecordListRenderer.java

@@ -30,10 +30,13 @@ import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
+import com.kingsrook.qqq.backend.core.actions.tables.CountAction;
 import com.kingsrook.qqq.backend.core.actions.tables.GetAction;
 import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
+import com.kingsrook.qqq.backend.core.actions.values.QValueFormatter;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.exceptions.QNotFoundException;
+import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetOutput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
@@ -72,7 +75,7 @@ public class ChildRecordListRenderer extends AbstractWidgetRenderer
       return (new Builder(new QWidgetMetaData()
          .withName(join.getName())
          .withIsCard(true)
-         .withCodeReference(new QCodeReference(ChildRecordListRenderer.class, null))
+         .withCodeReference(new QCodeReference(ChildRecordListRenderer.class))
          .withType(WidgetType.CHILD_RECORD_LIST.getType())
          .withDefaultValue("joinName", join.getName())));
    }
@@ -134,7 +137,7 @@ public class ChildRecordListRenderer extends AbstractWidgetRenderer
        *******************************************************************************/
       public Builder withCanAddChildRecord(boolean b)
       {
-         widgetMetaData.withDefaultValue("canAddChildRecord", true);
+         widgetMetaData.withDefaultValue("canAddChildRecord", b);
          return (this);
       }
 
@@ -148,6 +151,17 @@ public class ChildRecordListRenderer extends AbstractWidgetRenderer
          widgetMetaData.withDefaultValue("disabledFieldsForNewChildRecords", new HashSet<>(disabledFieldsForNewChildRecords));
          return (this);
       }
+
+
+
+      /*******************************************************************************
+       **
+       *******************************************************************************/
+      public Builder withManageAssociationName(String manageAssociationName)
+      {
+         widgetMetaData.withDefaultValue("manageAssociationName", manageAssociationName);
+         return (this);
+      }
    }
 
 
@@ -158,56 +172,83 @@ public class ChildRecordListRenderer extends AbstractWidgetRenderer
    @Override
    public RenderWidgetOutput render(RenderWidgetInput input) throws QException
    {
-      String        widgetLabel = input.getQueryParams().get("widgetLabel");
-      String        joinName    = input.getQueryParams().get("joinName");
-      QJoinMetaData join        = input.getInstance().getJoin(joinName);
-      String        id          = input.getQueryParams().get("id");
+      String         widgetLabel = input.getQueryParams().get("widgetLabel");
+      String         joinName    = input.getQueryParams().get("joinName");
+      QJoinMetaData  join        = input.getInstance().getJoin(joinName);
+      String         id          = input.getQueryParams().get("id");
+      QTableMetaData leftTable   = input.getInstance().getTable(join.getLeftTable());
+      QTableMetaData rightTable  = input.getInstance().getTable(join.getRightTable());
 
       Integer maxRows = null;
       if(StringUtils.hasContent(input.getQueryParams().get("maxRows")))
       {
          maxRows = ValueUtils.getValueAsInteger(input.getQueryParams().get("maxRows"));
       }
-
-      ////////////////////////////////////////////////////////
-      // fetch the record that we're getting children for.  //
-      // e.g., the left-side of the join, with the input id //
-      ////////////////////////////////////////////////////////
-      GetInput getInput = new GetInput();
-      getInput.setTableName(join.getLeftTable());
-      getInput.setPrimaryKey(id);
-      GetOutput getOutput = new GetAction().execute(getInput);
-      QRecord   record    = getOutput.getRecord();
-
-      if(record == null)
+      else if(input.getWidgetMetaData().getDefaultValues().containsKey("maxRows"))
       {
-         QTableMetaData table = input.getInstance().getTable(join.getLeftTable());
-         throw (new QNotFoundException("Could not find " + (table == null ? "" : table.getLabel()) + " with primary key " + id));
+         maxRows = ValueUtils.getValueAsInteger(input.getWidgetMetaData().getDefaultValues().containsKey("maxRows"));
       }
 
-      ////////////////////////////////////////////////////////////////////
-      // set up the query - for the table on the right side of the join //
-      ////////////////////////////////////////////////////////////////////
-      QQueryFilter filter = new QQueryFilter();
-      for(JoinOn joinOn : join.getJoinOns())
+      //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      // fetch the record that we're getting children for. e.g., the left-side of the join, with the input id                                     //
+      // but - only try this if we were given an id.  note, this widget could be called for on an INSERT screen, where we don't have a record yet //
+      // but we still want to be able to return all the other data in here that otherwise comes from the widget meta data, join, etc.             //
+      //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      int          totalRows     = 0;
+      QRecord      primaryRecord = null;
+      QQueryFilter filter        = null;
+      QueryOutput  queryOutput   = new QueryOutput(new QueryInput());
+      if(StringUtils.hasContent(id))
       {
-         filter.addCriteria(new QFilterCriteria(joinOn.getRightField(), QCriteriaOperator.EQUALS, List.of(record.getValue(joinOn.getLeftField()))));
+         GetInput getInput = new GetInput();
+         getInput.setTableName(join.getLeftTable());
+         getInput.setPrimaryKey(id);
+         GetOutput getOutput = new GetAction().execute(getInput);
+         primaryRecord = getOutput.getRecord();
+
+         if(primaryRecord == null)
+         {
+            throw (new QNotFoundException("Could not find " + (leftTable == null ? "" : leftTable.getLabel()) + " with primary key " + id));
+         }
+
+         ////////////////////////////////////////////////////////////////////
+         // set up the query - for the table on the right side of the join //
+         ////////////////////////////////////////////////////////////////////
+         filter = new QQueryFilter();
+         for(JoinOn joinOn : join.getJoinOns())
+         {
+            filter.addCriteria(new QFilterCriteria(joinOn.getRightField(), QCriteriaOperator.EQUALS, List.of(primaryRecord.getValue(joinOn.getLeftField()))));
+         }
+         filter.setOrderBys(join.getOrderBys());
+         filter.setLimit(maxRows);
+
+         QueryInput queryInput = new QueryInput();
+         queryInput.setTableName(join.getRightTable());
+         queryInput.setShouldTranslatePossibleValues(true);
+         queryInput.setShouldGenerateDisplayValues(true);
+         queryInput.setFilter(filter);
+         queryOutput = new QueryAction().execute(queryInput);
+
+         QValueFormatter.setBlobValuesToDownloadUrls(rightTable, queryOutput.getRecords());
+
+         totalRows = queryOutput.getRecords().size();
+         if(maxRows != null && (queryOutput.getRecords().size() == maxRows))
+         {
+            /////////////////////////////////////////////////////////////////////////////////////
+            // if the input said to only do some max, and the # of results we got is that max, //
+            // then do a count query, for displaying 1-n of <count>                            //
+            /////////////////////////////////////////////////////////////////////////////////////
+            CountInput countInput = new CountInput();
+            countInput.setTableName(join.getRightTable());
+            countInput.setFilter(filter);
+            totalRows = new CountAction().execute(countInput).getCount();
+         }
       }
-      filter.setOrderBys(join.getOrderBys());
 
-      QueryInput queryInput = new QueryInput();
-      queryInput.setTableName(join.getRightTable());
-      queryInput.setShouldTranslatePossibleValues(true);
-      queryInput.setShouldGenerateDisplayValues(true);
-      queryInput.setFilter(filter);
-      queryInput.setLimit(maxRows);
-      QueryOutput queryOutput = new QueryAction().execute(queryInput);
+      String tablePath   = input.getInstance().getTablePath(rightTable.getName());
+      String viewAllLink = tablePath == null ? null : (tablePath + "?filter=" + URLEncoder.encode(JsonUtils.toJson(filter), Charset.defaultCharset()));
 
-      QTableMetaData table       = input.getInstance().getTable(join.getRightTable());
-      String         tablePath   = input.getInstance().getTablePath(table.getName());
-      String         viewAllLink = tablePath == null ? null : (tablePath + "?filter=" + URLEncoder.encode(JsonUtils.toJson(filter), Charset.defaultCharset()));
-
-      ChildRecordListData widgetData = new ChildRecordListData(widgetLabel, queryOutput, table, tablePath, viewAllLink);
+      ChildRecordListData widgetData = new ChildRecordListData(widgetLabel, queryOutput, rightTable, tablePath, viewAllLink, totalRows);
 
       if(BooleanUtils.isTrue(ValueUtils.getValueAsBoolean(input.getQueryParams().get("canAddChildRecord"))))
       {
@@ -217,10 +258,14 @@ public class ChildRecordListRenderer extends AbstractWidgetRenderer
          // new child records must have values from the join-ons //
          //////////////////////////////////////////////////////////
          Map<String, Serializable> defaultValuesForNewChildRecords = new HashMap<>();
-         for(JoinOn joinOn : join.getJoinOns())
+         if(primaryRecord != null)
          {
-            defaultValuesForNewChildRecords.put(joinOn.getRightField(), record.getValue(joinOn.getLeftField()));
+            for(JoinOn joinOn : join.getJoinOns())
+            {
+               defaultValuesForNewChildRecords.put(joinOn.getRightField(), primaryRecord.getValue(joinOn.getLeftField()));
+            }
          }
+
          widgetData.setDefaultValuesForNewChildRecords(defaultValuesForNewChildRecords);
 
          Map<String, Serializable> widgetValues = input.getWidgetMetaData().getDefaultValues();
@@ -228,6 +273,22 @@ public class ChildRecordListRenderer extends AbstractWidgetRenderer
          {
             widgetData.setDisabledFieldsForNewChildRecords((Set<String>) widgetValues.get("disabledFieldsForNewChildRecords"));
          }
+         else
+         {
+            /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+            // if there are no disabled fields specified - then normally any fields w/ a default value get implicitly disabled //
+            // but - if we didn't look-up the primary record, then we'll want to explicit disable fields from joins            //
+            /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+            if(primaryRecord == null)
+            {
+               Set<String> implicitlyDisabledFields = new HashSet<>();
+               widgetData.setDisabledFieldsForNewChildRecords(implicitlyDisabledFields);
+               for(JoinOn joinOn : join.getJoinOns())
+               {
+                  implicitlyDisabledFields.add(joinOn.getRightField());
+               }
+            }
+         }
       }
 
       return (new RenderWidgetOutput(widgetData));

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/dashboard/widgets/DateTimeGroupBy.java

@@ -99,7 +99,17 @@ public enum DateTimeGroupBy
    public String getSqlExpression()
    {
       ZoneId sessionOrInstanceZoneId = ValueUtils.getSessionOrInstanceZoneId();
-      String targetTimezone          = sessionOrInstanceZoneId.toString();
+      return (getSqlExpression(sessionOrInstanceZoneId));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public String getSqlExpression(ZoneId targetZoneId)
+   {
+      String targetTimezone = targetZoneId.toString();
 
       if("Z".equals(targetTimezone) || !StringUtils.hasContent(targetTimezone))
       {
@@ -158,7 +168,18 @@ public enum DateTimeGroupBy
     *******************************************************************************/
    public String makeSelectedString(Instant time)
    {
-      ZonedDateTime zoned = time.atZone(ValueUtils.getSessionOrInstanceZoneId());
+      return (makeSelectedString(time, ValueUtils.getSessionOrInstanceZoneId()));
+   }
+
+
+
+   /*******************************************************************************
+    ** Make an Instant into a string that will match what came out of the database's
+    ** DATE_FORMAT() function
+    *******************************************************************************/
+   public String makeSelectedString(Instant time, ZoneId zoneId)
+   {
+      ZonedDateTime zoned = time.atZone(zoneId);
 
       if(this == WEEK)
       {
@@ -182,7 +203,17 @@ public enum DateTimeGroupBy
     *******************************************************************************/
    public String makeHumanString(Instant instant)
    {
-      ZonedDateTime zoned = instant.atZone(ValueUtils.getSessionOrInstanceZoneId());
+      return (makeHumanString(instant, ValueUtils.getSessionOrInstanceZoneId()));
+   }
+
+
+
+   /*******************************************************************************
+    ** Make a string to show to a user
+    *******************************************************************************/
+   public String makeHumanString(Instant instant, ZoneId zoneId)
+   {
+      ZonedDateTime zoned = instant.atZone(zoneId);
       if(this.equals(WEEK))
       {
          DateTimeFormatter dateTimeFormatter = DateTimeFormatter.ofPattern("M'/'d");
@@ -215,25 +246,35 @@ public enum DateTimeGroupBy
    /*******************************************************************************
     **
     *******************************************************************************/
-   @SuppressWarnings("checkstyle:indentation")
    public Instant roundDown(Instant instant)
    {
-      ZonedDateTime zoned = instant.atZone(ValueUtils.getSessionOrInstanceZoneId());
+      return roundDown(instant, ValueUtils.getSessionOrInstanceZoneId());
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @SuppressWarnings("checkstyle:indentation")
+   public Instant roundDown(Instant instant, ZoneId zoneId)
+   {
+      ZonedDateTime zoned = instant.atZone(zoneId);
       return switch(this)
+      {
+         case YEAR -> zoned.with(TemporalAdjusters.firstDayOfYear()).truncatedTo(ChronoUnit.DAYS).toInstant();
+         case MONTH -> zoned.with(TemporalAdjusters.firstDayOfMonth()).truncatedTo(ChronoUnit.DAYS).toInstant();
+         case WEEK ->
          {
-            case YEAR -> zoned.with(TemporalAdjusters.firstDayOfYear()).truncatedTo(ChronoUnit.DAYS).toInstant();
-            case MONTH -> zoned.with(TemporalAdjusters.firstDayOfMonth()).truncatedTo(ChronoUnit.DAYS).toInstant();
-            case WEEK ->
+            while(zoned.get(ChronoField.DAY_OF_WEEK) != DayOfWeek.SUNDAY.getValue())
             {
-               while(zoned.get(ChronoField.DAY_OF_WEEK) != DayOfWeek.SUNDAY.getValue())
-               {
-                  zoned = zoned.minusDays(1);
-               }
-               yield (zoned.truncatedTo(ChronoUnit.DAYS).toInstant());
+               zoned = zoned.minusDays(1);
             }
-            case DAY -> zoned.truncatedTo(ChronoUnit.DAYS).toInstant();
-            case HOUR -> zoned.truncatedTo(ChronoUnit.HOURS).toInstant();
-         };
+            yield (zoned.truncatedTo(ChronoUnit.DAYS).toInstant());
+         }
+         case DAY -> zoned.truncatedTo(ChronoUnit.DAYS).toInstant();
+         case HOUR -> zoned.truncatedTo(ChronoUnit.HOURS).toInstant();
+      };
    }
 
 
@@ -243,7 +284,34 @@ public enum DateTimeGroupBy
     *******************************************************************************/
    public Instant increment(Instant instant)
    {
-      ZonedDateTime zoned = instant.atZone(ValueUtils.getSessionOrInstanceZoneId());
+      return (increment(instant, ValueUtils.getSessionOrInstanceZoneId()));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public Instant increment(Instant instant, ZoneId zoneId)
+   {
+      ZonedDateTime zoned = instant.atZone(zoneId);
       return (zoned.plus(noOfChronoUnitsToAdd, chronoUnitToAdd).toInstant());
    }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public static DateTimeFormatter sqlDateFormatToSelectedDateTimeFormatter(String sqlDateFormat)
+   {
+      for(DateTimeGroupBy value : values())
+      {
+         if(value.sqlDateFormat.equals(sqlDateFormat))
+         {
+            return (value.selectedStringFormatter);
+         }
+      }
+      return null;
+   }
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/dashboard/widgets/NoCodeWidgetVelocityUtils.java

@@ -64,14 +64,24 @@ public class NoCodeWidgetVelocityUtils
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public String icon(String iconName, String color)
+   {
+      return String.format("""
+         <span class="material-icons-round notranslate MuiIcon-root MuiIcon-fontSizeInherit" style="color: %s; position: relative; top: 6px;" aria-hidden="true">%s</span>
+         """, color, iconName);
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/
    public String helpIcon()
    {
-      return ("""
-         <span class="material-icons-round notranslate MuiIcon-root MuiIcon-fontSizeInherit" style="color: blue; position: relative; top: 6px;" aria-hidden="true">help_outline</span>
-         """);
+      return (icon("help_outline", "blue"));
    }
 
 
@@ -81,9 +91,7 @@ public class NoCodeWidgetVelocityUtils
     *******************************************************************************/
    public String errorIcon()
    {
-      return ("""
-         <span class="material-icons-round notranslate MuiIcon-root MuiIcon-fontSizeInherit" style="color: red; position: relative; top: 6px;" aria-hidden="true">error_outline</span>
-         """);
+      return (icon("error_outline", "red"));
    }
 
 
@@ -93,9 +101,7 @@ public class NoCodeWidgetVelocityUtils
     *******************************************************************************/
    public String warningIcon()
    {
-      return ("""
-         <span class="material-icons-round notranslate MuiIcon-root MuiIcon-fontSizeInherit" style="color: orange; position: relative; top: 6px;" aria-hidden="true">warning</span>
-         """);
+      return (icon("warning", "orange"));
    }
 
 
@@ -105,9 +111,7 @@ public class NoCodeWidgetVelocityUtils
     *******************************************************************************/
    public String checkIcon()
    {
-      return ("""
-         <span class="material-icons-round notranslate MuiIcon-root MuiIcon-fontSizeInherit" style="color: green; position: relative; top: 6px;" aria-hidden="true">check</span>
-         """);
+      return (icon("check", "green"));
    }
 
 
@@ -117,9 +121,7 @@ public class NoCodeWidgetVelocityUtils
     *******************************************************************************/
    public String pendingIcon()
    {
-      return ("""
-         <span class="material-icons-round notranslate MuiIcon-root MuiIcon-fontSizeInherit" style="color: #0062ff; position: relative; top: 6px;" aria-hidden="true">pending</span>
-         """);
+      return (icon("pending", "#0062ff"));
    }
 
 
@@ -288,7 +290,7 @@ public class NoCodeWidgetVelocityUtils
          WidgetCount  widgetCount = (WidgetCount) context.get(countVariableName + ".source");
          Integer      count       = ValueUtils.getValueAsInteger(context.get(countVariableName));
          QQueryFilter filter      = widgetCount.getEffectiveFilter(input);
-         return (AbstractHTMLWidgetRenderer.aHrefTableFilterNoOfRecords(null, widgetCount.getTableName(), filter, count, singular, plural));
+         return (AbstractHTMLWidgetRenderer.aHrefTableFilterNoOfRecords(widgetCount.getTableName(), filter, count, singular, plural));
       }
       catch(Exception e)
       {

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/dashboard/widgets/ParentWidgetRenderer.java

@@ -60,6 +60,8 @@ public class ParentWidgetRenderer extends AbstractWidgetRenderer
             widgetData.setChildWidgetNameList(metaData.getChildWidgetNameList());
          }
 
+         widgetData.setLayoutType(metaData.getLayoutType());
+
          return (new RenderWidgetOutput(widgetData));
       }
       catch(Exception e)

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/interfaces/AggregateInterface.java

@@ -31,7 +31,7 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateOu
  ** Interface for the Aggregate action.
  **
  *******************************************************************************/
-public interface AggregateInterface
+public interface AggregateInterface extends BaseQueryInterface
 {
    /*******************************************************************************
     **

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/interfaces/BaseQueryInterface.java

@@ -0,0 +1,81 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2023.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.actions.interfaces;
+
+
+import java.time.Instant;
+import com.kingsrook.qqq.backend.core.model.querystats.QueryStat;
+
+
+/*******************************************************************************
+ ** Base class for "query" (e.g., read-operations) action interfaces (query, count, aggregate).
+ ** Initially just here for the QueryStat methods - if we expand those to apply
+ ** to insert/update/delete, well, then rename this maybe to BaseActionInterface?
+ *******************************************************************************/
+public interface BaseQueryInterface
+{
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   default void setQueryStat(QueryStat queryStat)
+   {
+      //////////
+      // noop //
+      //////////
+   }
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   default QueryStat getQueryStat()
+   {
+      return (null);
+   }
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   default void setQueryStatFirstResultTime()
+   {
+      QueryStat queryStat = getQueryStat();
+      if(queryStat != null)
+      {
+         if(queryStat.getFirstResultTimestamp() == null)
+         {
+            queryStat.setFirstResultTimestamp(Instant.now());
+         }
+      }
+   }
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   default void cancelAction()
+   {
+      //////////////////////////////////////////////
+      // initially at least, a noop in base class //
+      //////////////////////////////////////////////
+   }
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/interfaces/CountInterface.java

@@ -31,7 +31,7 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountOutput;
  ** Interface for the Count action.
  **
  *******************************************************************************/
-public interface CountInterface
+public interface CountInterface extends BaseQueryInterface
 {
    /*******************************************************************************
     **

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/interfaces/DeleteInterface.java

@@ -50,4 +50,13 @@ public interface DeleteInterface
       return (false);
    }
 
+   /*******************************************************************************
+    ** Specify whether this particular module's delete action can & should fetch
+    ** records before deleting them, e.g., for audits or "not-found-checks"
+    *******************************************************************************/
+   default boolean supportsPreFetchQuery()
+   {
+      return (true);
+   }
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/interfaces/GetInterface.java

@@ -28,6 +28,7 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetOutput;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.UniqueKey;
+import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
 
 
 /*******************************************************************************
@@ -55,7 +56,7 @@ public interface GetInterface
       {
          QTableMetaData table      = getInput.getTable();
          boolean        foundMatch = false;
-         for(UniqueKey uniqueKey : table.getUniqueKeys())
+         for(UniqueKey uniqueKey : CollectionUtils.nonNullList(table.getUniqueKeys()))
          {
             if(new HashSet<>(uniqueKey.getFieldNames()).equals(getInput.getUniqueKey().keySet()))
             {

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/interfaces/InsertInterface.java

@@ -31,7 +31,7 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertOutput;
  ** Interface for the Insert action.
  **
  *******************************************************************************/
-public interface InsertInterface extends QActionInterface
+public interface InsertInterface
 {
    /*******************************************************************************
     **

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/interfaces/QueryInterface.java

@@ -31,10 +31,11 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput;
  ** Interface for the Query action.
  **
  *******************************************************************************/
-public interface QueryInterface
+public interface QueryInterface extends BaseQueryInterface
 {
    /*******************************************************************************
     **
     *******************************************************************************/
    QueryOutput execute(QueryInput queryInput) throws QException;
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/interfaces/UpdateInterface.java

@@ -37,4 +37,14 @@ public interface UpdateInterface
     **
     *******************************************************************************/
    UpdateOutput execute(UpdateInput updateInput) throws QException;
+
+   /*******************************************************************************
+    ** Specify whether this particular module's update action can & should fetch
+    ** records before updating them, e.g., for audits or "not-found-checks"
+    *******************************************************************************/
+   default boolean supportsPreFetchQuery()
+   {
+      return (true);
+   }
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/metadata/JoinGraph.java

@@ -0,0 +1,336 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2023.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.actions.metadata;
+
+
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Objects;
+import java.util.Set;
+import java.util.TreeSet;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.joins.QJoinMetaData;
+import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
+import com.kingsrook.qqq.backend.core.utils.StringUtils;
+
+
+/*******************************************************************************
+ ** Object to represent the graph of joins in a QQQ Instance.  e.g., all of the
+ ** connections among tables through joins.
+ *******************************************************************************/
+public class JoinGraph
+{
+
+   private Set<Edge> edges = new HashSet<>();
+
+
+
+   /*******************************************************************************
+    ** Graph edge (no graph nodes needed in here)
+    *******************************************************************************/
+   private record Edge(String joinName, String leftTable, String rightTable)
+   {
+   }
+
+
+
+   /*******************************************************************************
+    ** In this class, we are treating joins as non-directional graph edges - so -
+    ** use this class to "normalize" what may otherwise be duplicated joins in the
+    ** qInstance (e.g., A -> B and B -> A -- in the instance, those are valid, but
+    ** in our graph here, we want to consider those the same).
+    *******************************************************************************/
+   private static class NormalizedJoin
+   {
+      private String tableA;
+      private String tableB;
+      private String joinFieldA;
+      private String joinFieldB;
+
+
+
+      /*******************************************************************************
+       **
+       *******************************************************************************/
+      public NormalizedJoin(QJoinMetaData joinMetaData)
+      {
+         boolean needFlip     = false;
+         int     tableCompare = joinMetaData.getLeftTable().compareTo(joinMetaData.getRightTable());
+         if(tableCompare < 0)
+         {
+            needFlip = true;
+         }
+         else if(tableCompare == 0)
+         {
+            int fieldCompare = joinMetaData.getJoinOns().get(0).getLeftField().compareTo(joinMetaData.getJoinOns().get(0).getRightField());
+            if(fieldCompare < 0)
+            {
+               needFlip = true;
+            }
+         }
+
+         if(needFlip)
+         {
+            joinMetaData = joinMetaData.flip();
+         }
+
+         tableA = joinMetaData.getLeftTable();
+         tableB = joinMetaData.getRightTable();
+         joinFieldA = joinMetaData.getJoinOns().get(0).getLeftField();
+         joinFieldB = joinMetaData.getJoinOns().get(0).getRightField();
+      }
+
+
+
+      /*******************************************************************************
+       **
+       *******************************************************************************/
+      @Override
+      public boolean equals(Object o)
+      {
+         if(this == o)
+         {
+            return true;
+         }
+         if(o == null || getClass() != o.getClass())
+         {
+            return false;
+         }
+         NormalizedJoin that = (NormalizedJoin) o;
+         return Objects.equals(tableA, that.tableA) && Objects.equals(tableB, that.tableB) && Objects.equals(joinFieldA, that.joinFieldA) && Objects.equals(joinFieldB, that.joinFieldB);
+      }
+
+
+
+      /*******************************************************************************
+       **
+       *******************************************************************************/
+      @Override
+      public int hashCode()
+      {
+         return Objects.hash(tableA, tableB, joinFieldA, joinFieldB);
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** Constructor
+    **
+    *******************************************************************************/
+   public JoinGraph(QInstance qInstance)
+   {
+      Set<NormalizedJoin> usedJoins = new HashSet<>();
+      for(QJoinMetaData join : CollectionUtils.nonNullMap(qInstance.getJoins()).values())
+      {
+         NormalizedJoin normalizedJoin = new NormalizedJoin(join);
+         if(usedJoins.contains(normalizedJoin))
+         {
+            continue;
+         }
+
+         usedJoins.add(normalizedJoin);
+         edges.add(new Edge(join.getName(), join.getLeftTable(), join.getRightTable()));
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public record JoinConnection(String joinTable, String viaJoinName) implements Comparable<JoinConnection>
+   {
+
+      /*******************************************************************************
+       **
+       *******************************************************************************/
+      @Override
+      public int compareTo(JoinConnection that)
+      {
+         Comparator<JoinConnection> comparator = Comparator.comparing((JoinConnection jc) -> jc.joinTable())
+            .thenComparing((JoinConnection jc) -> jc.viaJoinName());
+         return (comparator.compare(this, that));
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public record JoinConnectionList(List<JoinConnection> list) implements Comparable<JoinConnectionList>
+   {
+
+      /*******************************************************************************
+       **
+       *******************************************************************************/
+      public JoinConnectionList copy()
+      {
+         return new JoinConnectionList(new ArrayList<>(list));
+      }
+
+
+
+      /*******************************************************************************
+       **
+       *******************************************************************************/
+      public int compareTo(JoinConnectionList that)
+      {
+         if(this.equals(that))
+         {
+            return (0);
+         }
+
+         for(int i = 0; i < Math.min(this.list.size(), that.list.size()); i++)
+         {
+            int comp = this.list.get(i).compareTo(that.list.get(i));
+            if(comp != 0)
+            {
+               return (comp);
+            }
+         }
+
+         return (this.list.size() - that.list.size());
+      }
+
+
+
+      /*******************************************************************************
+       **
+       *******************************************************************************/
+      public boolean matchesJoinPath(List<String> joinPath)
+      {
+         if(list.size() != joinPath.size())
+         {
+            return (false);
+         }
+
+         for(int i = 0; i < list.size(); i++)
+         {
+            if(!list.get(i).viaJoinName().equals(joinPath.get(i)))
+            {
+               return (false);
+            }
+         }
+
+         return (true);
+      }
+
+
+
+      /*******************************************************************************
+       **
+       *******************************************************************************/
+      public String getJoinNamesAsString()
+      {
+         return (StringUtils.join(", ", list().stream().map(jc -> jc.viaJoinName()).toList()));
+      }
+
+
+
+      /*******************************************************************************
+       **
+       *******************************************************************************/
+      public List<String> getJoinNamesAsList()
+      {
+         return (list().stream().map(jc -> jc.viaJoinName()).toList());
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public Set<JoinConnectionList> getJoinConnections(String tableName)
+   {
+      Set<JoinConnectionList> rs = new TreeSet<>();
+      doGetJoinConnections(rs, tableName, new ArrayList<>(), new JoinConnectionList(new ArrayList<>()));
+      return (rs);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private void doGetJoinConnections(Set<JoinConnectionList> joinConnections, String tableName, List<String> path, JoinConnectionList connectionList)
+   {
+      for(Edge edge : edges)
+      {
+         if(edge.leftTable.equals(tableName) || edge.rightTable.equals(tableName))
+         {
+            if(path.contains(edge.joinName))
+            {
+               continue;
+            }
+
+            List<String> newPath = new ArrayList<>(path);
+            newPath.add(edge.joinName);
+            if(!joinConnectionsContain(joinConnections, newPath))
+            {
+               String otherTableName = null;
+               if(!edge.leftTable.equals(tableName))
+               {
+                  otherTableName = edge.leftTable;
+               }
+               else if(!edge.rightTable.equals(tableName))
+               {
+                  otherTableName = edge.rightTable;
+               }
+
+               if(otherTableName != null)
+               {
+
+                  JoinConnectionList newConnectionList = connectionList.copy();
+                  JoinConnection     joinConnection    = new JoinConnection(otherTableName, edge.joinName);
+                  newConnectionList.list.add(joinConnection);
+                  joinConnections.add(newConnectionList);
+                  doGetJoinConnections(joinConnections, otherTableName, new ArrayList<>(newPath), newConnectionList);
+               }
+            }
+         }
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private boolean joinConnectionsContain(Set<JoinConnectionList> joinPaths, List<String> newPath)
+   {
+      for(JoinConnectionList joinConnections : joinPaths)
+      {
+         List<String> joinConnectionJoins = joinConnections.list.stream().map(jc -> jc.viaJoinName).toList();
+         if(joinConnectionJoins.equals(newPath))
+         {
+            return (true);
+         }
+      }
+      return (false);
+   }
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/metadata/MetaDataAction.java

@@ -23,6 +23,7 @@ package com.kingsrook.qqq.backend.core.actions.metadata;
 
 
 import java.util.ArrayList;
+import java.util.Comparator;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
@@ -83,11 +84,14 @@ public class MetaDataAction
          }
 
          QBackendMetaData backendForTable = metaDataInput.getInstance().getBackendForTable(tableName);
-         tables.put(tableName, new QFrontendTableMetaData(metaDataInput, backendForTable, table, false));
+         tables.put(tableName, new QFrontendTableMetaData(metaDataInput, backendForTable, table, false, false));
          treeNodes.put(tableName, new AppTreeNode(table));
       }
       metaDataOutput.setTables(tables);
 
+      // addJoinsToTables(tables);
+      // addJoinedTablesToTables(tables);
+
       ////////////////////////////////////////
       // map processes to frontend metadata //
       ////////////////////////////////////////
@@ -147,14 +151,21 @@ public class MetaDataAction
       }
       metaDataOutput.setWidgets(widgets);
 
+      ///////////////////////////////////////////////////////
+      // sort apps - by sortOrder (integer), then by label //
+      ///////////////////////////////////////////////////////
+      List<QAppMetaData> sortedApps = metaDataInput.getInstance().getApps().values().stream()
+         .sorted(Comparator.comparing((QAppMetaData a) -> a.getSortOrder())
+            .thenComparing((QAppMetaData a) -> a.getLabel()))
+         .toList();
+
       ///////////////////////////////////
       // map apps to frontend metadata //
       ///////////////////////////////////
       Map<String, QFrontendAppMetaData> apps = new LinkedHashMap<>();
-      for(Map.Entry<String, QAppMetaData> entry : metaDataInput.getInstance().getApps().entrySet())
+      for(QAppMetaData app : sortedApps)
       {
-         String       appName = entry.getKey();
-         QAppMetaData app     = entry.getValue();
+         String appName = app.getName();
 
          PermissionCheckResult permissionResult = PermissionsHelper.getPermissionCheckResult(metaDataInput, app);
          if(permissionResult.equals(PermissionCheckResult.DENY_HIDE))
@@ -188,7 +199,7 @@ public class MetaDataAction
       // organize app tree nodes by their hierarchy //
       ////////////////////////////////////////////////
       List<AppTreeNode> appTree = new ArrayList<>();
-      for(QAppMetaData appMetaData : metaDataInput.getInstance().getApps().values())
+      for(QAppMetaData appMetaData : sortedApps)
       {
          if(appMetaData.getParentAppName() == null)
          {

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/metadata/TableMetaDataAction.java

@@ -54,7 +54,7 @@ public class TableMetaDataAction
          throw (new QNotFoundException("Table [" + tableMetaDataInput.getTableName() + "] was not found."));
       }
       QBackendMetaData backendForTable = tableMetaDataInput.getInstance().getBackendForTable(table.getName());
-      tableMetaDataOutput.setTable(new QFrontendTableMetaData(tableMetaDataInput, backendForTable, table, true));
+      tableMetaDataOutput.setTable(new QFrontendTableMetaData(tableMetaDataInput, backendForTable, table, true, true));
 
       // todo post-customization - can do whatever w/ the result if you want
 

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/permissions/PermissionsHelper.java

@@ -34,15 +34,18 @@ import com.kingsrook.qqq.backend.core.exceptions.QPermissionDeniedException;
 import com.kingsrook.qqq.backend.core.logging.QLogger;
 import com.kingsrook.qqq.backend.core.model.actions.AbstractActionInput;
 import com.kingsrook.qqq.backend.core.model.actions.AbstractTableActionInput;
+import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
 import com.kingsrook.qqq.backend.core.model.metadata.dashboard.QWidgetMetaDataInterface;
 import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.permissions.DenyBehavior;
 import com.kingsrook.qqq.backend.core.model.metadata.permissions.MetaDataWithName;
 import com.kingsrook.qqq.backend.core.model.metadata.permissions.MetaDataWithPermissionRules;
+import com.kingsrook.qqq.backend.core.model.metadata.permissions.PermissionLevel;
 import com.kingsrook.qqq.backend.core.model.metadata.permissions.QPermissionRules;
 import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.Capability;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 import com.kingsrook.qqq.backend.core.model.session.QSession;
 import com.kingsrook.qqq.backend.core.utils.StringUtils;
@@ -333,9 +336,25 @@ public class PermissionsHelper
          QPermissionRules rules    = getEffectivePermissionRules(tableMetaData, instance);
          String           baseName = getEffectivePermissionBaseName(rules, tableMetaData.getName());
 
-         for(TablePermissionSubType permissionSubType : TablePermissionSubType.values())
+         QBackendMetaData backend = instance.getBackend(tableMetaData.getBackendName());
+         if(tableMetaData.isCapabilityEnabled(backend, Capability.TABLE_INSERT))
          {
-            addEffectiveAvailablePermission(rules, permissionSubType, rs, baseName, tableMetaData, "Table");
+            addEffectiveAvailablePermission(rules, TablePermissionSubType.INSERT, rs, baseName, tableMetaData, "Table");
+         }
+
+         if(tableMetaData.isCapabilityEnabled(backend, Capability.TABLE_UPDATE))
+         {
+            addEffectiveAvailablePermission(rules, TablePermissionSubType.EDIT, rs, baseName, tableMetaData, "Table");
+         }
+
+         if(tableMetaData.isCapabilityEnabled(backend, Capability.TABLE_DELETE))
+         {
+            addEffectiveAvailablePermission(rules, TablePermissionSubType.DELETE, rs, baseName, tableMetaData, "Table");
+         }
+
+         if(tableMetaData.isCapabilityEnabled(backend, Capability.TABLE_QUERY) || tableMetaData.isCapabilityEnabled(backend, Capability.TABLE_GET))
+         {
+            addEffectiveAvailablePermission(rules, TablePermissionSubType.READ, rs, baseName, tableMetaData, "Table");
          }
       }
 
@@ -369,7 +388,10 @@ public class PermissionsHelper
       {
          QPermissionRules rules    = getEffectivePermissionRules(widgetMetaData, instance);
          String           baseName = getEffectivePermissionBaseName(rules, widgetMetaData.getName());
-         addEffectiveAvailablePermission(rules, PrivatePermissionSubType.HAS_ACCESS, rs, baseName, widgetMetaData, "Widget");
+         if(!rules.getLevel().equals(PermissionLevel.NOT_PROTECTED))
+         {
+            addEffectiveAvailablePermission(rules, PrivatePermissionSubType.HAS_ACCESS, rs, baseName, widgetMetaData, "Widget");
+         }
       }
 
       return (rs);

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/processes/QProcessCallbackFactory.java

@@ -0,0 +1,68 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.actions.processes;
+
+
+import java.io.Serializable;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
+
+
+/*******************************************************************************
+ ** Constructor for commonly used QProcessCallback's
+ *******************************************************************************/
+public class QProcessCallbackFactory
+{
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public static QProcessCallback forFilter(QQueryFilter filter)
+   {
+      return new QProcessCallback()
+      {
+         /*******************************************************************************
+          **
+          *******************************************************************************/
+         @Override
+         public QQueryFilter getQueryFilter()
+         {
+            return (filter);
+         }
+
+
+
+         /*******************************************************************************
+          **
+          *******************************************************************************/
+         @Override
+         public Map<String, Serializable> getFieldValues(List<QFieldMetaData> fields)
+         {
+            return (Collections.emptyMap());
+         }
+      };
+   }
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/processes/RunProcessAction.java

@@ -35,6 +35,7 @@ import com.kingsrook.qqq.backend.core.actions.dashboard.widgets.NoCodeWidgetRend
 import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
 import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
 import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction;
+import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.logging.QLogger;
 import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessState;
@@ -50,6 +51,7 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.processes.NoCodeWidgetFrontendComponentMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData;
@@ -57,6 +59,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendComponen
 import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendStepMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.processes.QStepMetaData;
+import com.kingsrook.qqq.backend.core.model.session.QSession;
 import com.kingsrook.qqq.backend.core.processes.implementations.basepull.BasepullConfiguration;
 import com.kingsrook.qqq.backend.core.state.InMemoryStateProvider;
 import com.kingsrook.qqq.backend.core.state.StateProviderInterface;
@@ -69,7 +72,7 @@ import org.apache.commons.lang.BooleanUtils;
 
 
 /*******************************************************************************
- ** Action handler for running q-processes (which are a sequence of q-functions).
+ ** Action handler for running q-processes (which are a sequence of q-steps).
  *
  *******************************************************************************/
 public class RunProcessAction
@@ -79,6 +82,7 @@ public class RunProcessAction
    public static final String BASEPULL_THIS_RUNTIME_KEY = "basepullThisRuntimeKey";
    public static final String BASEPULL_LAST_RUNTIME_KEY = "basepullLastRuntimeKey";
    public static final String BASEPULL_TIMESTAMP_FIELD  = "basepullTimestampField";
+   public static final String BASEPULL_CONFIGURATION = "basepullConfiguration";
 
    ////////////////////////////////////////////////////////////////////////////////////////////////
    // indicator that the timestamp field should be updated - e.g., the execute step is finished. //
@@ -483,6 +487,35 @@ public class RunProcessAction
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   protected String determineBasepullKeyValue(QProcessMetaData process, BasepullConfiguration basepullConfiguration) throws QException
+   {
+      String basepullKeyValue = (basepullConfiguration.getKeyValue() != null) ? basepullConfiguration.getKeyValue() : process.getName();
+
+      //////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      // if backend specifies that it uses variants, look for that data in the session and append to our basepull key //
+      //////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      if(process.getSchedule() != null && process.getVariantBackend() != null)
+      {
+         QSession         session         = QContext.getQSession();
+         QBackendMetaData backendMetaData = QContext.getQInstance().getBackend(process.getVariantBackend());
+         if(session.getBackendVariants() == null || !session.getBackendVariants().containsKey(backendMetaData.getVariantOptionsTableTypeValue()))
+         {
+            LOG.info("Could not find Backend Variant information for Backend '" + backendMetaData.getName() + "'");
+         }
+         else
+         {
+            basepullKeyValue += "-" + session.getBackendVariants().get(backendMetaData.getVariantOptionsTableTypeValue());
+         }
+      }
+
+      return (basepullKeyValue);
+   }
+
+
+
    /*******************************************************************************
     ** Insert or update the last runtime value for this basepull into the backend.
     *******************************************************************************/
@@ -491,7 +524,7 @@ public class RunProcessAction
       String basepullTableName            = basepullConfiguration.getTableName();
       String basepullKeyFieldName         = basepullConfiguration.getKeyField();
       String basepullLastRunTimeFieldName = basepullConfiguration.getLastRunTimeFieldName();
-      String basepullKeyValue             = (basepullConfiguration.getKeyValue() != null) ? basepullConfiguration.getKeyValue() : process.getName();
+      String basepullKeyValue             = determineBasepullKeyValue(process, basepullConfiguration);
 
       ///////////////////////////////////////
       // get the stored basepull timestamp //
@@ -571,7 +604,7 @@ public class RunProcessAction
       String  basepullKeyFieldName                 = basepullConfiguration.getKeyField();
       String  basepullLastRunTimeFieldName         = basepullConfiguration.getLastRunTimeFieldName();
       Integer basepullHoursBackForInitialTimestamp = basepullConfiguration.getHoursBackForInitialTimestamp();
-      String  basepullKeyValue                     = (basepullConfiguration.getKeyValue() != null) ? basepullConfiguration.getKeyValue() : process.getName();
+      String  basepullKeyValue                     = determineBasepullKeyValue(process, basepullConfiguration);
 
       ///////////////////////////////////////
       // get the stored basepull timestamp //
@@ -601,5 +634,6 @@ public class RunProcessAction
 
       runProcessInput.getValues().put(BASEPULL_LAST_RUNTIME_KEY, lastRunTime);
       runProcessInput.getValues().put(BASEPULL_TIMESTAMP_FIELD, basepullConfiguration.getTimestampField());
+      runProcessInput.getValues().put(BASEPULL_CONFIGURATION, basepullConfiguration);
    }
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/BufferedRecordPipe.java

@@ -24,6 +24,7 @@ package com.kingsrook.qqq.backend.core.actions.reporting;
 
 import java.util.ArrayList;
 import java.util.List;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
 
 
@@ -63,7 +64,7 @@ public class BufferedRecordPipe extends RecordPipe
     **
     *******************************************************************************/
    @Override
-   public void addRecord(QRecord record)
+   public void addRecord(QRecord record) throws QException
    {
       buffer.add(record);
       if(buffer.size() >= bufferSize)
@@ -78,7 +79,7 @@ public class BufferedRecordPipe extends RecordPipe
    /*******************************************************************************
     **
     *******************************************************************************/
-   public void finalFlush()
+   public void finalFlush() throws QException
    {
       if(!buffer.isEmpty())
       {

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/ExportAction.java

@@ -23,8 +23,12 @@ package com.kingsrook.qqq.backend.core.actions.reporting;
 
 
 import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
 import java.util.List;
+import java.util.Map;
 import java.util.Optional;
+import java.util.Set;
 import java.util.concurrent.TimeUnit;
 import com.kingsrook.qqq.backend.core.actions.ActionHelper;
 import com.kingsrook.qqq.backend.core.actions.async.AsyncJobManager;
@@ -32,6 +36,7 @@ import com.kingsrook.qqq.backend.core.actions.async.AsyncJobState;
 import com.kingsrook.qqq.backend.core.actions.async.AsyncJobStatus;
 import com.kingsrook.qqq.backend.core.actions.interfaces.CountInterface;
 import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
+import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.exceptions.QReportingException;
 import com.kingsrook.qqq.backend.core.exceptions.QUserFacingException;
@@ -41,10 +46,13 @@ import com.kingsrook.qqq.backend.core.model.actions.reporting.ExportOutput;
 import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportFormat;
 import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountOutput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryJoin;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.ExposedJoin;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleDispatcher;
 import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleInterface;
@@ -95,15 +103,25 @@ public class ExportAction
       ///////////////////////////////////
       if(CollectionUtils.nullSafeHasContents(exportInput.getFieldNames()))
       {
-         QTableMetaData table         = exportInput.getTable();
-         List<String>   badFieldNames = new ArrayList<>();
+         QTableMetaData              table        = exportInput.getTable();
+         Map<String, QTableMetaData> joinTableMap = getJoinTableMap(table);
+
+         List<String> badFieldNames = new ArrayList<>();
          for(String fieldName : exportInput.getFieldNames())
          {
             try
             {
-               table.getField(fieldName);
+               if(fieldName.contains("."))
+               {
+                  String[] parts = fieldName.split("\\.", 2);
+                  joinTableMap.get(parts[0]).getField(parts[1]);
+               }
+               else
+               {
+                  table.getField(fieldName);
+               }
             }
-            catch(IllegalArgumentException iae)
+            catch(Exception e)
             {
                badFieldNames.add(fieldName);
             }
@@ -128,6 +146,21 @@ public class ExportAction
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private static Map<String, QTableMetaData> getJoinTableMap(QTableMetaData table)
+   {
+      Map<String, QTableMetaData> joinTableMap = new HashMap<>();
+      for(ExposedJoin exposedJoin : CollectionUtils.nonNullList(table.getExposedJoins()))
+      {
+         joinTableMap.put(exposedJoin.getJoinTable(), QContext.getQInstance().getTable(exposedJoin.getJoinTable()));
+      }
+      return joinTableMap;
+   }
+
+
+
    /*******************************************************************************
     ** Run the report.
     *******************************************************************************/
@@ -151,8 +184,55 @@ public class ExportAction
       QueryInput  queryInput  = new QueryInput();
       queryInput.setTableName(exportInput.getTableName());
       queryInput.setFilter(exportInput.getQueryFilter());
-      queryInput.setLimit(exportInput.getLimit());
+
+      List<QueryJoin> queryJoins     = new ArrayList<>();
+      Set<String>     addedJoinNames = new HashSet<>();
+      if(CollectionUtils.nullSafeHasContents(exportInput.getFieldNames()))
+      {
+         for(String fieldName : exportInput.getFieldNames())
+         {
+            if(fieldName.contains("."))
+            {
+               String[] parts         = fieldName.split("\\.", 2);
+               String   joinTableName = parts[0];
+               if(!addedJoinNames.contains(joinTableName))
+               {
+                  QueryJoin queryJoin = new QueryJoin(joinTableName).withType(QueryJoin.Type.LEFT).withSelect(true);
+                  queryJoins.add(queryJoin);
+
+                  /////////////////////////////////////////////////////////////////////////////////////////////
+                  // in at least some cases, we need to let the queryJoin know what join-meta-data to use... //
+                  // This code basically mirrors what QFMD is doing right now, so it's better -              //
+                  // but shouldn't all of this just be in JoinsContext?  it does some of this...             //
+                  /////////////////////////////////////////////////////////////////////////////////////////////
+                  QTableMetaData        table               = exportInput.getTable();
+                  Optional<ExposedJoin> exposedJoinOptional = CollectionUtils.nonNullList(table.getExposedJoins()).stream().filter(ej -> ej.getJoinTable().equals(joinTableName)).findFirst();
+                  if(exposedJoinOptional.isEmpty())
+                  {
+                     throw (new QException("Could not find exposed join between base table " + table.getName() + " and requested join table " + joinTableName));
+                  }
+                  ExposedJoin exposedJoin = exposedJoinOptional.get();
+
+                  if(exposedJoin.getJoinPath().size() == 1)
+                  {
+                     queryJoin.setJoinMetaData(QContext.getQInstance().getJoin(exposedJoin.getJoinPath().get(exposedJoin.getJoinPath().size() - 1)));
+                  }
+
+                  addedJoinNames.add(joinTableName);
+               }
+            }
+         }
+      }
+
+      queryInput.setQueryJoins(queryJoins);
+
+      if(queryInput.getFilter() == null)
+      {
+         queryInput.setFilter(new QQueryFilter());
+      }
+      queryInput.getFilter().setLimit(exportInput.getLimit());
       queryInput.setShouldTranslatePossibleValues(true);
+      queryInput.withQueryHint(QueryInput.QueryHint.POTENTIALLY_LARGE_NUMBER_OF_RESULTS);
 
       /////////////////////////////////////////////////////////////////
       // tell this query that it needs to put its output into a pipe //
@@ -298,24 +378,51 @@ public class ExportAction
     *******************************************************************************/
    private List<QFieldMetaData> getFields(ExportInput exportInput)
    {
+      QTableMetaData              table        = exportInput.getTable();
+      Map<String, QTableMetaData> joinTableMap = getJoinTableMap(table);
+
       List<QFieldMetaData> fieldList;
-      QTableMetaData       table = exportInput.getTable();
       if(exportInput.getFieldNames() != null)
       {
-         fieldList = exportInput.getFieldNames().stream().map(table::getField).toList();
+         fieldList = new ArrayList<>();
+         for(String fieldName : exportInput.getFieldNames())
+         {
+            if(fieldName.contains("."))
+            {
+               String[]       parts     = fieldName.split("\\.", 2);
+               QTableMetaData joinTable = joinTableMap.get(parts[0]);
+               QFieldMetaData field     = joinTable.getField(parts[1]).clone();
+               field.setName(fieldName);
+               field.setLabel(joinTable.getLabel() + ": " + field.getLabel());
+               fieldList.add(field);
+            }
+            else
+            {
+               fieldList.add(table.getField(fieldName));
+            }
+         }
       }
       else
       {
          fieldList = new ArrayList<>(table.getFields().values());
       }
 
-      //////////////////////////////////////////
-      // add fields for possible value labels //
-      //////////////////////////////////////////
       List<QFieldMetaData> returnList = new ArrayList<>();
       for(QFieldMetaData field : fieldList)
       {
+         /////////////////////////////////////////////////////////////////////////////////////////
+         // skip heavy fields.  they aren't fetched, and we generally think we don't want them. //
+         /////////////////////////////////////////////////////////////////////////////////////////
+         if(field.getIsHeavy())
+         {
+            continue;
+         }
+
          returnList.add(field);
+
+         //////////////////////////////////////////
+         // add fields for possible value labels //
+         //////////////////////////////////////////
          if(StringUtils.hasContent(field.getPossibleValueSourceName()))
          {
             returnList.add(new QFieldMetaData(field.getName() + ":possibleValueLabel", QFieldType.STRING).withLabel(field.getLabel() + " Name"));

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/GenerateReportAction.java

@@ -73,6 +73,7 @@ import com.kingsrook.qqq.backend.core.utils.StringUtils;
 import com.kingsrook.qqq.backend.core.utils.aggregates.AggregatesInterface;
 import com.kingsrook.qqq.backend.core.utils.aggregates.BigDecimalAggregates;
 import com.kingsrook.qqq.backend.core.utils.aggregates.IntegerAggregates;
+import com.kingsrook.qqq.backend.core.utils.aggregates.LongAggregates;
 
 
 /*******************************************************************************
@@ -273,7 +274,7 @@ public class GenerateReportAction
       RunBackendStepOutput  transformStepOutput = null;
       if(tableView != null && tableView.getRecordTransformStep() != null)
       {
-         transformStep = QCodeLoader.getBackendStep(AbstractTransformStep.class, tableView.getRecordTransformStep());
+         transformStep = QCodeLoader.getAdHoc(AbstractTransformStep.class, tableView.getRecordTransformStep());
 
          transformStepInput = new RunBackendStepInput();
          transformStepInput.setValues(reportInput.getInputValues());
@@ -553,6 +554,12 @@ public class GenerateReportAction
             AggregatesInterface<Integer> fieldAggregates = (AggregatesInterface<Integer>) aggregatesMap.computeIfAbsent(field.getName(), (name) -> new IntegerAggregates());
             fieldAggregates.add(record.getValueInteger(field.getName()));
          }
+         else if(field.getType().equals(QFieldType.LONG))
+         {
+            @SuppressWarnings("unchecked")
+            AggregatesInterface<Long> fieldAggregates = (AggregatesInterface<Long>) aggregatesMap.computeIfAbsent(field.getName(), (name) -> new LongAggregates());
+            fieldAggregates.add(record.getValueLong(field.getName()));
+         }
          else if(field.getType().equals(QFieldType.DECIMAL))
          {
             @SuppressWarnings("unchecked")

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/RecordPipe.java

@@ -26,10 +26,11 @@ import java.util.ArrayList;
 import java.util.List;
 import java.util.concurrent.ArrayBlockingQueue;
 import java.util.concurrent.TimeUnit;
-import java.util.function.Consumer;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.logging.QLogger;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
 import com.kingsrook.qqq.backend.core.utils.SleepUtils;
+import com.kingsrook.qqq.backend.core.utils.lambdas.UnsafeConsumer;
 
 
 /*******************************************************************************
@@ -43,11 +44,12 @@ public class RecordPipe
    private static final long BLOCKING_SLEEP_MILLIS = 100;
    private static final long MAX_SLEEP_LOOP_MILLIS = 300_000; // 5 minutes
 
-   private ArrayBlockingQueue<QRecord> queue = new ArrayBlockingQueue<>(1_000);
+   private int capacity = 1_000;
+   private ArrayBlockingQueue<QRecord> queue = new ArrayBlockingQueue<>(capacity);
 
    private boolean isTerminated = false;
 
-   private Consumer<List<QRecord>> postRecordActions = null;
+   private UnsafeConsumer<List<QRecord>, QException> postRecordActions = null;
 
    /////////////////////////////////////
    // See usage below for explanation //
@@ -71,6 +73,7 @@ public class RecordPipe
     *******************************************************************************/
    public RecordPipe(Integer overrideCapacity)
    {
+      this.capacity = overrideCapacity;
       queue = new ArrayBlockingQueue<>(overrideCapacity);
    }
 
@@ -93,7 +96,7 @@ public class RecordPipe
    /*******************************************************************************
     ** Add a record to the pipe.  Will block if the pipe is full.  Will noop if pipe is terminated.
     *******************************************************************************/
-   public void addRecord(QRecord record)
+   public void addRecord(QRecord record) throws QException
    {
       if(isTerminated)
       {
@@ -109,7 +112,7 @@ public class RecordPipe
          // (which we'll create as a field in this class, to avoid always re-constructing) //
          ////////////////////////////////////////////////////////////////////////////////////
          singleRecordListForPostRecordActions.add(record);
-         postRecordActions.accept(singleRecordListForPostRecordActions);
+         postRecordActions.run(singleRecordListForPostRecordActions);
          record = singleRecordListForPostRecordActions.remove(0);
       }
 
@@ -135,7 +138,7 @@ public class RecordPipe
          {
             if(now - sleepLoopStartTime > MAX_SLEEP_LOOP_MILLIS)
             {
-               LOG.warn("Giving up adding record to pipe, due to pipe being full for more than {} millis", MAX_SLEEP_LOOP_MILLIS);
+               LOG.warn("Giving up adding record to pipe, due to pipe being full for more than " + MAX_SLEEP_LOOP_MILLIS + " millis");
                throw (new IllegalStateException("Giving up adding record to pipe, due to pipe staying full too long."));
             }
             LOG.trace("Record pipe.add failed (due to full pipe).  Blocking.");
@@ -152,11 +155,11 @@ public class RecordPipe
    /*******************************************************************************
     ** Add a list of records to the pipe.  Will block if the pipe is full.  Will noop if pipe is terminated.
     *******************************************************************************/
-   public void addRecords(List<QRecord> records)
+   public void addRecords(List<QRecord> records) throws QException
    {
       if(postRecordActions != null)
       {
-         postRecordActions.accept(records);
+         postRecordActions.run(records);
       }
 
       //////////////////////////////////////////////////////////////////////////////////////////////////
@@ -207,9 +210,19 @@ public class RecordPipe
    /*******************************************************************************
     **
     *******************************************************************************/
-   public void setPostRecordActions(Consumer<List<QRecord>> postRecordActions)
+   public void setPostRecordActions(UnsafeConsumer<List<QRecord>, QException> postRecordActions)
    {
       this.postRecordActions = postRecordActions;
    }
 
+
+
+   /*******************************************************************************
+    ** Getter for capacity
+    **
+    *******************************************************************************/
+   public int getCapacity()
+   {
+      return capacity;
+   }
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/RecordPipeBufferedWrapper.java

@@ -0,0 +1,79 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.actions.reporting;
+
+
+import java.util.List;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+
+
+/*******************************************************************************
+ ** Subclass of BufferedRecordPipe, which ultimately sends records down to an
+ ** original RecordPipe.
+ **
+ ** Meant to be used where: someone passed in a RecordPipe (so they have a reference
+ ** to it, and they are waiting to read from it), but the producer knows that
+ ** it will be better to buffer the records, so they want to use a buffered pipe
+ ** (but they still need the records to end up in the original pipe - thus -
+ ** it gets wrapped by an object of this class).
+ *******************************************************************************/
+public class RecordPipeBufferedWrapper extends BufferedRecordPipe
+{
+   private RecordPipe wrappedPipe;
+
+
+
+   /*******************************************************************************
+    ** Constructor - uses default buffer size
+    **
+    *******************************************************************************/
+   public RecordPipeBufferedWrapper(RecordPipe wrappedPipe)
+   {
+      this.wrappedPipe = wrappedPipe;
+   }
+
+
+
+   /*******************************************************************************
+    ** Constructor - customize buffer size.
+    **
+    *******************************************************************************/
+   public RecordPipeBufferedWrapper(Integer bufferSize, RecordPipe wrappedPipe)
+   {
+      super(bufferSize);
+      this.wrappedPipe = wrappedPipe;
+   }
+
+
+
+   /*******************************************************************************
+    ** when it's time to actually add records into the pipe, actually add them
+    ** into the wrapped pipe!
+    *******************************************************************************/
+   @Override
+   public void addRecords(List<QRecord> records) throws QException
+   {
+      wrappedPipe.addRecords(records);
+   }
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/scripts/AssociatedScriptContextPrimerInterface.java

@@ -19,26 +19,23 @@
  * along with this program.  If not, see <https://www.gnu.org/licenses/>.
  */
 
-package com.kingsrook.qqq.backend.core.actions.interfaces;
+package com.kingsrook.qqq.backend.core.actions.scripts;
 
 
-import com.kingsrook.qqq.backend.core.actions.QBackendTransaction;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
-import com.kingsrook.qqq.backend.core.model.actions.AbstractTableActionInput;
+import com.kingsrook.qqq.backend.core.model.actions.scripts.ExecuteCodeInput;
+import com.kingsrook.qqq.backend.core.model.scripts.ScriptRevision;
 
 
 /*******************************************************************************
  **
  *******************************************************************************/
-public interface QActionInterface
+public interface AssociatedScriptContextPrimerInterface
 {
 
    /*******************************************************************************
     **
     *******************************************************************************/
-   default QBackendTransaction openTransaction(AbstractTableActionInput input) throws QException
-   {
-      return (new QBackendTransaction());
-   }
+   void primeContext(ExecuteCodeInput executeCodeInput, ScriptRevision scriptRevision) throws QException;
 
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/scripts/ExecuteCodeAction.java

@@ -23,15 +23,36 @@ package com.kingsrook.qqq.backend.core.actions.scripts;
 
 
 import java.io.Serializable;
+import java.util.ArrayList;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
+import java.util.Objects;
 import com.kingsrook.qqq.backend.core.actions.scripts.logging.Log4jCodeExecutionLogger;
 import com.kingsrook.qqq.backend.core.actions.scripts.logging.QCodeExecutionLoggerInterface;
+import com.kingsrook.qqq.backend.core.actions.scripts.logging.ScriptExecutionLoggerInterface;
+import com.kingsrook.qqq.backend.core.actions.scripts.logging.StoreScriptLogAndScriptLogLineExecutionLogger;
+import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
+import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.exceptions.QCodeException;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.logging.QLogger;
+import com.kingsrook.qqq.backend.core.model.actions.scripts.AbstractRunScriptInput;
 import com.kingsrook.qqq.backend.core.model.actions.scripts.ExecuteCodeInput;
 import com.kingsrook.qqq.backend.core.model.actions.scripts.ExecuteCodeOutput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
 import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
+import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeType;
+import com.kingsrook.qqq.backend.core.model.scripts.ScriptRevision;
+import com.kingsrook.qqq.backend.core.model.scripts.ScriptRevisionFile;
+import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
+import com.kingsrook.qqq.backend.core.utils.ObjectUtils;
+import com.kingsrook.qqq.backend.core.utils.StringUtils;
 
 
 /*******************************************************************************
@@ -49,6 +70,9 @@ import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
  *******************************************************************************/
 public class ExecuteCodeAction
 {
+   private static final QLogger LOG = QLogger.getLogger(ExecuteCodeAction.class);
+
+
 
    /*******************************************************************************
     **
@@ -68,10 +92,10 @@ public class ExecuteCodeAction
       try
       {
          String languageExecutor = switch(codeReference.getCodeType())
-            {
-               case JAVA -> "com.kingsrook.qqq.backend.core.actions.scripts.QJavaExecutor";
-               case JAVA_SCRIPT -> "com.kingsrook.qqq.languages.javascript.QJavaScriptExecutor";
-            };
+         {
+            case JAVA -> "com.kingsrook.qqq.backend.core.actions.scripts.QJavaExecutor";
+            case JAVA_SCRIPT -> "com.kingsrook.qqq.languages.javascript.QJavaScriptExecutor";
+         };
 
          @SuppressWarnings("unchecked")
          Class<? extends QCodeExecutor> executorClass = (Class<? extends QCodeExecutor>) Class.forName(languageExecutor);
@@ -90,6 +114,22 @@ public class ExecuteCodeAction
             context.putAll(input.getInput());
          }
 
+         //////////////////////////////////////////
+         // safely always set the deploymentMode //
+         //////////////////////////////////////////
+         context.put("deploymentMode", ObjectUtils.tryAndRequireNonNullElse(() -> QContext.getQInstance().getDeploymentMode(), null));
+
+         /////////////////////////////////////////////////////////////////////////////////
+         // set the qCodeExecutor into any context objects which are QCodeExecutorAware //
+         /////////////////////////////////////////////////////////////////////////////////
+         for(Serializable value : context.values())
+         {
+            if(value instanceof QCodeExecutorAware qCodeExecutorAware)
+            {
+               qCodeExecutorAware.setQCodeExecutor(qCodeExecutor);
+            }
+         }
+
          Serializable codeOutput = qCodeExecutor.execute(codeReference, context, executionLogger);
          output.setOutput(codeOutput);
          executionLogger.acceptExecutionEnd(codeOutput);
@@ -108,6 +148,156 @@ public class ExecuteCodeAction
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public static ExecuteCodeInput setupExecuteCodeInput(AbstractRunScriptInput<?> input, ScriptRevision scriptRevision) throws QException
+   {
+      return setupExecuteCodeInput(input, scriptRevision, null);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public static ExecuteCodeInput setupExecuteCodeInput(AbstractRunScriptInput<?> input, ScriptRevision scriptRevision, String fileName) throws QException
+   {
+      ExecuteCodeInput executeCodeInput = new ExecuteCodeInput();
+      executeCodeInput.setInput(new HashMap<>(Objects.requireNonNullElseGet(input.getInputValues(), HashMap::new)));
+      executeCodeInput.setContext(new HashMap<>());
+
+      Map<String, Serializable> context = executeCodeInput.getContext();
+      if(input.getOutputObject() != null)
+      {
+         context.put("output", input.getOutputObject());
+      }
+
+      if(input.getScriptUtils() != null)
+      {
+         context.put("scriptUtils", input.getScriptUtils());
+      }
+
+      if(CollectionUtils.nullSafeIsEmpty(scriptRevision.getFiles()))
+      {
+         QueryInput queryInput = new QueryInput();
+         queryInput.setTableName(ScriptRevisionFile.TABLE_NAME);
+         queryInput.setFilter(new QQueryFilter(new QFilterCriteria("scriptRevisionId", QCriteriaOperator.EQUALS, scriptRevision.getId())));
+         QueryOutput queryOutput = new QueryAction().execute(queryInput);
+
+         scriptRevision.setFiles(new ArrayList<>());
+         for(QRecord record : queryOutput.getRecords())
+         {
+            scriptRevision.getFiles().add(new ScriptRevisionFile(record));
+         }
+      }
+
+      List<ScriptRevisionFile> files = scriptRevision.getFiles();
+      if(files == null || files.isEmpty())
+      {
+         throw (new QException("Script Revision " + scriptRevision.getId() + " had more than 1 associated ScriptRevisionFile (and the name to use was not specified)."));
+      }
+      else
+      {
+         String contents = null;
+         if(fileName == null || files.size() == 1)
+         {
+            contents = files.get(0).getContents();
+         }
+         else
+         {
+            for(ScriptRevisionFile file : files)
+            {
+               if(file.getFileName().equals(fileName))
+               {
+                  contents = file.getContents();
+               }
+            }
+            if(contents == null)
+            {
+               throw (new QException("Could not find file named " + fileName + " for Script Revision " + scriptRevision.getId()));
+            }
+         }
+
+         executeCodeInput.setCodeReference(new QCodeReference().withInlineCode(contents).withCodeType(QCodeType.JAVA_SCRIPT)); // todo - code type as attribute of script!!
+      }
+
+      ExecuteCodeAction.addApiUtilityToContext(context, scriptRevision);
+      context.put("qqq", new QqqScriptUtils());
+      ExecuteCodeAction.setExecutionLoggerInExecuteCodeInput(input, scriptRevision, executeCodeInput);
+
+      return (executeCodeInput);
+   }
+
+
+
+   /*******************************************************************************
+    ** Try to (dynamically) load the ApiScriptUtils object from the api middleware
+    ** module -- in case the runtime doesn't have that module deployed (e.g, not in
+    ** the project pom).
+    *******************************************************************************/
+   public static void addApiUtilityToContext(Map<String, Serializable> context, ScriptRevision scriptRevision)
+   {
+      addApiUtilityToContext(context, scriptRevision.getApiName(), scriptRevision.getApiVersion());
+   }
+
+
+
+   /*******************************************************************************
+    ** Try to (dynamically) load the ApiScriptUtils object from the api middleware
+    ** module -- in case the runtime doesn't have that module deployed (e.g, not in
+    ** the project pom).
+    *******************************************************************************/
+   public static void addApiUtilityToContext(Map<String, Serializable> context, String apiName, String apiVersion)
+   {
+      if(!StringUtils.hasContent(apiName) || !StringUtils.hasContent(apiVersion))
+      {
+         return;
+      }
+
+      try
+      {
+         Class<?> apiScriptUtilsClass  = Class.forName("com.kingsrook.qqq.api.utils.ApiScriptUtils");
+         Object   apiScriptUtilsObject = apiScriptUtilsClass.getConstructor(String.class, String.class).newInstance(apiName, apiVersion);
+         context.put("api", (Serializable) apiScriptUtilsObject);
+      }
+      catch(ClassNotFoundException e)
+      {
+         /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+         // this is the only exception we're kinda expecting here - so catch for it specifically, and just log.trace - others, warn //
+         /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+         LOG.trace("Couldn't load ApiScriptUtils class - qqq-middleware-api not on the classpath?");
+      }
+      catch(Exception e)
+      {
+         LOG.warn("Error adding api utility to script context", e);
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private static void setExecutionLoggerInExecuteCodeInput(AbstractRunScriptInput<?> input, ScriptRevision scriptRevision, ExecuteCodeInput executeCodeInput)
+   {
+      /////////////////////////////////////////////////////////////////////////////////////////////////
+      // let caller supply a logger, or by default use StoreScriptLogAndScriptLogLineExecutionLogger //
+      /////////////////////////////////////////////////////////////////////////////////////////////////
+      QCodeExecutionLoggerInterface executionLogger = Objects.requireNonNullElseGet(input.getLogger(), () -> new StoreScriptLogAndScriptLogLineExecutionLogger(scriptRevision.getScriptId(), scriptRevision.getId()));
+      executeCodeInput.setExecutionLogger(executionLogger);
+      if(executionLogger instanceof ScriptExecutionLoggerInterface scriptExecutionLoggerInterface)
+      {
+         ////////////////////////////////////////////////////////////////////////////////////////////////////
+         // if logger is aware of scripts (as opposed to a generic CodeExecution logger), give it the ids. //
+         ////////////////////////////////////////////////////////////////////////////////////////////////////
+         scriptExecutionLoggerInterface.setScriptId(scriptRevision.getScriptId());
+         scriptExecutionLoggerInterface.setScriptRevisionId(scriptRevision.getId());
+      }
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/scripts/QCodeExecutor.java

@@ -41,4 +41,24 @@ public interface QCodeExecutor
     *******************************************************************************/
    Serializable execute(QCodeReference codeReference, Map<String, Serializable> inputContext, QCodeExecutionLoggerInterface executionLogger) throws QCodeException;
 
+   /*******************************************************************************
+    ** Process an object from the script's language/runtime into a (more) native java object.
+    ** e.g., a Nashorn ScriptObjectMirror will end up as a "primitive", or a List or Map of such
+    **
+    *******************************************************************************/
+   default Object convertObjectToJava(Object object) throws QCodeException
+   {
+      return (object);
+   }
+
+   /*******************************************************************************
+    ** Convert a native java object into one for the script's language/runtime.
+    ** e.g., a java Instant to a Nashorn Date
+    **
+    *******************************************************************************/
+   default Object convertJavaObject(Object object, Object requestedTypeHint) throws QCodeException
+   {
+      return (object);
+   }
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/scripts/QCodeExecutorAware.java

@@ -0,0 +1,36 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2023.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.actions.scripts;
+
+
+/*******************************************************************************
+ ** Interface for classes that can accept a QCodeExecutor object via a setter.
+ *******************************************************************************/
+public interface QCodeExecutorAware
+{
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   void setQCodeExecutor(QCodeExecutor qCodeExecutor);
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/scripts/QqqScriptUtils.java

@@ -47,8 +47,17 @@ import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 /*******************************************************************************
  ** Object made available to scripts for access to qqq api (e.g., query, insert,
  ** etc, plus object constructors).
+ **
+ ** Before scripts knew about the API, this class made sense and was used.
+ ** But, when scripts gained knowledge of the API, then it felt like this class could
+ ** be deleted... but, what about, a QQQ deployment without the API module...
+ ** In that case, we might still want this class... think about it.
+ **
+ ** And/Or - it turns out - sometimes using QQQ directly is "better" (?) than using
+ ** an api - so - this object may be available for other use cases (e.g., getting
+ ** a record's backendDetails (e.g., for full json from a source backend api)).
  *******************************************************************************/
-public class ScriptApi implements Serializable
+public class QqqScriptUtils implements Serializable
 {
 
    /*******************************************************************************

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/scripts/RecordScriptTestInterface.java

@@ -0,0 +1,159 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2023.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.actions.scripts;
+
+
+import java.io.Serializable;
+import java.util.Collections;
+import java.util.List;
+import com.kingsrook.qqq.backend.core.actions.scripts.logging.BuildScriptLogAndScriptLogLineExecutionLogger;
+import com.kingsrook.qqq.backend.core.actions.tables.GetAction;
+import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
+import com.kingsrook.qqq.backend.core.context.QContext;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.actions.scripts.ExecuteCodeInput;
+import com.kingsrook.qqq.backend.core.model.actions.scripts.RunAdHocRecordScriptInput;
+import com.kingsrook.qqq.backend.core.model.actions.scripts.RunAdHocRecordScriptOutput;
+import com.kingsrook.qqq.backend.core.model.actions.scripts.TestScriptInput;
+import com.kingsrook.qqq.backend.core.model.actions.scripts.TestScriptOutput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.metadata.code.AdHocScriptCodeReference;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+import com.kingsrook.qqq.backend.core.model.scripts.Script;
+import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
+import com.kingsrook.qqq.backend.core.utils.StringUtils;
+import com.kingsrook.qqq.backend.core.utils.ValueUtils;
+
+
+/*******************************************************************************
+ **
+ *******************************************************************************/
+public class RecordScriptTestInterface implements TestScriptActionInterface
+{
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public void setupTestScriptInput(TestScriptInput testScriptInput, ExecuteCodeInput executeCodeInput) throws QException
+   {
+
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public void execute(TestScriptInput input, TestScriptOutput output) throws QException
+   {
+      try
+      {
+         Serializable scriptId = input.getInputValues().get("scriptId");
+         QRecord      script   = new GetAction().executeForRecord(new GetInput(Script.TABLE_NAME).withPrimaryKey(scriptId));
+
+         //////////////////////////////////////////////
+         // look up the records being tested against //
+         //////////////////////////////////////////////
+         String         tableName = script.getValueString("tableName");
+         QTableMetaData table     = QContext.getQInstance().getTable(tableName);
+         if(table == null)
+         {
+            throw (new QException("Could not find table [" + tableName + "] for script"));
+         }
+
+         String recordPrimaryKeyList = ValueUtils.getValueAsString(input.getInputValues().get("recordPrimaryKeyList"));
+         if(!StringUtils.hasContent(recordPrimaryKeyList))
+         {
+            throw (new QException("Record primary key list was not given."));
+         }
+
+         QueryOutput queryOutput = new QueryAction().execute(new QueryInput(tableName)
+            .withFilter(new QQueryFilter(new QFilterCriteria(table.getPrimaryKeyField(), QCriteriaOperator.IN, recordPrimaryKeyList.split(","))))
+            .withIncludeAssociations(true));
+         if(CollectionUtils.nullSafeIsEmpty(queryOutput.getRecords()))
+         {
+            throw (new QException("No records were found by the given primary keys."));
+         }
+
+         /////////////////////////////
+         // set up & run the action //
+         /////////////////////////////
+         RunAdHocRecordScriptInput runAdHocRecordScriptInput = new RunAdHocRecordScriptInput();
+         runAdHocRecordScriptInput.setRecordList(queryOutput.getRecords());
+
+         BuildScriptLogAndScriptLogLineExecutionLogger executionLogger = new BuildScriptLogAndScriptLogLineExecutionLogger(null, null);
+         runAdHocRecordScriptInput.setLogger(executionLogger);
+
+         runAdHocRecordScriptInput.setTableName(tableName);
+         runAdHocRecordScriptInput.setCodeReference((AdHocScriptCodeReference) input.getCodeReference());
+         RunAdHocRecordScriptOutput runAdHocRecordScriptOutput = new RunAdHocRecordScriptOutput();
+         new RunAdHocRecordScriptAction().run(runAdHocRecordScriptInput, runAdHocRecordScriptOutput);
+
+         /////////////////////////////////
+         // send outputs back to caller //
+         /////////////////////////////////
+         output.setScriptLog(executionLogger.getScriptLog());
+         output.setScriptLogLines(executionLogger.getScriptLogLines());
+         if(runAdHocRecordScriptOutput.getException().isPresent())
+         {
+            output.setException(runAdHocRecordScriptOutput.getException().get());
+         }
+      }
+      catch(QException e)
+      {
+         output.setException(e);
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public List<QFieldMetaData> getTestInputFields()
+   {
+      return (List.of(new QFieldMetaData("recordPrimaryKeyList", QFieldType.STRING).withLabel("Record Primary Key List")));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public List<QFieldMetaData> getTestOutputFields()
+   {
+      return (Collections.emptyList());
+   }
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/scripts/RunAdHocRecordScriptAction.java

@@ -22,15 +22,15 @@
 package com.kingsrook.qqq.backend.core.actions.scripts;
 
 
+import java.io.Serializable;
+import java.lang.reflect.Method;
 import java.util.ArrayList;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
-import java.util.Objects;
 import java.util.Optional;
 import com.kingsrook.qqq.backend.core.actions.ActionHelper;
-import com.kingsrook.qqq.backend.core.actions.scripts.logging.QCodeExecutionLoggerInterface;
-import com.kingsrook.qqq.backend.core.actions.scripts.logging.ScriptExecutionLoggerInterface;
-import com.kingsrook.qqq.backend.core.actions.scripts.logging.StoreScriptLogAndScriptLogLineExecutionLogger;
+import com.kingsrook.qqq.backend.core.actions.audits.DMLAuditAction;
 import com.kingsrook.qqq.backend.core.actions.tables.GetAction;
 import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
 import com.kingsrook.qqq.backend.core.context.QContext;
@@ -48,13 +48,14 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryJoin;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
 import com.kingsrook.qqq.backend.core.model.metadata.code.AdHocScriptCodeReference;
-import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
-import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeType;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 import com.kingsrook.qqq.backend.core.model.scripts.Script;
 import com.kingsrook.qqq.backend.core.model.scripts.ScriptRevision;
+import com.kingsrook.qqq.backend.core.model.scripts.ScriptsMetaDataProvider;
 import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
+import com.kingsrook.qqq.backend.core.utils.memoization.Memoization;
 
 
 /*******************************************************************************
@@ -67,6 +68,8 @@ public class RunAdHocRecordScriptAction
    private Map<Integer, ScriptRevision> scriptRevisionCacheByScriptRevisionId = new HashMap<>();
    private Map<Integer, ScriptRevision> scriptRevisionCacheByScriptId         = new HashMap<>();
 
+   private static Memoization<Integer, Script> scriptMemoizationById = new Memoization<>();
+
 
 
    /*******************************************************************************
@@ -87,6 +90,12 @@ public class RunAdHocRecordScriptAction
             throw (new QException("Script revision was not found."));
          }
 
+         Optional<Script> script = getScript(scriptRevision);
+
+         QContext.getQSession().setValue(DMLAuditAction.AUDIT_CONTEXT_FIELD_NAME, script.isPresent()
+            ? "via Script \"%s\"".formatted(script.get().getName())
+            : "via Script id " + scriptRevision.getScriptId());
+
          ////////////////////////////
          // figure out the records //
          ////////////////////////////
@@ -96,6 +105,7 @@ public class RunAdHocRecordScriptAction
             QueryInput queryInput = new QueryInput();
             queryInput.setTableName(input.getTableName());
             queryInput.setFilter(new QQueryFilter(new QFilterCriteria(table.getPrimaryKeyField(), QCriteriaOperator.IN, input.getRecordPrimaryKeyList())));
+            queryInput.setIncludeAssociations(true);
             QueryOutput queryOutput = new QueryAction().execute(queryInput);
             input.setRecordList(queryOutput.getRecords());
          }
@@ -112,48 +122,54 @@ public class RunAdHocRecordScriptAction
          /////////////
          // run it! //
          /////////////
-         ExecuteCodeInput executeCodeInput = new ExecuteCodeInput();
-         executeCodeInput.setInput(new HashMap<>(Objects.requireNonNullElseGet(input.getInputValues(), HashMap::new)));
-         executeCodeInput.getInput().put("records", new ArrayList<>(input.getRecordList()));
-         executeCodeInput.setContext(new HashMap<>());
-         if(input.getOutputObject() != null)
-         {
-            executeCodeInput.getContext().put("output", input.getOutputObject());
-         }
-
-         if(input.getScriptUtils() != null)
-         {
-            executeCodeInput.getContext().put("scriptUtils", input.getScriptUtils());
-         }
-
-         executeCodeInput.getContext().put("api", new ScriptApi());
-
-         executeCodeInput.setCodeReference(new QCodeReference().withInlineCode(scriptRevision.getContents()).withCodeType(QCodeType.JAVA_SCRIPT)); // todo - code type as attribute of script!!
-
-         /////////////////////////////////////////////////////////////////////////////////////////////////
-         // let caller supply a logger, or by default use StoreScriptLogAndScriptLogLineExecutionLogger //
-         /////////////////////////////////////////////////////////////////////////////////////////////////
-         QCodeExecutionLoggerInterface executionLogger = Objects.requireNonNullElseGet(input.getLogger(), () -> new StoreScriptLogAndScriptLogLineExecutionLogger(scriptRevision.getScriptId(), scriptRevision.getId()));
-         executeCodeInput.setExecutionLogger(executionLogger);
-         if(executionLogger instanceof ScriptExecutionLoggerInterface scriptExecutionLoggerInterface)
-         {
-            ////////////////////////////////////////////////////////////////////////////////////////////////////
-            // if logger is aware of scripts (as opposed to a generic CodeExecution logger), give it the ids. //
-            ////////////////////////////////////////////////////////////////////////////////////////////////////
-            scriptExecutionLoggerInterface.setScriptId(scriptRevision.getScriptId());
-            scriptExecutionLoggerInterface.setScriptRevisionId(scriptRevision.getId());
-         }
+         ExecuteCodeInput executeCodeInput = ExecuteCodeAction.setupExecuteCodeInput(input, scriptRevision);
+         executeCodeInput.getInput().put("records", getRecordsForScript(input, scriptRevision));
 
          ExecuteCodeOutput executeCodeOutput = new ExecuteCodeOutput();
          new ExecuteCodeAction().run(executeCodeInput, executeCodeOutput);
 
          output.setOutput(executeCodeOutput.getOutput());
-         output.setLogger(executionLogger);
+         output.setLogger(executeCodeInput.getExecutionLogger());
       }
       catch(Exception e)
       {
          output.setException(Optional.of(e));
       }
+      finally
+      {
+         QContext.getQSession().removeValue(DMLAuditAction.AUDIT_CONTEXT_FIELD_NAME);
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private static ArrayList<? extends Serializable> getRecordsForScript(RunAdHocRecordScriptInput input, ScriptRevision scriptRevision)
+   {
+      try
+      {
+         Class<?> apiScriptUtilsClass        = Class.forName("com.kingsrook.qqq.api.utils.ApiScriptUtils");
+         Method   qRecordListToApiRecordList = apiScriptUtilsClass.getMethod("qRecordListToApiRecordList", List.class, String.class, String.class, String.class);
+         Object   apiRecordList              = qRecordListToApiRecordList.invoke(null, input.getRecordList(), input.getTableName(), scriptRevision.getApiName(), scriptRevision.getApiVersion());
+
+         // noinspection unchecked
+         return (ArrayList<? extends Serializable>) apiRecordList;
+      }
+      catch(ClassNotFoundException e)
+      {
+         /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+         // this is the only exception we're kinda expecting here - so catch for it specifically, and just log.trace - others, warn //
+         /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+         LOG.trace("Couldn't load ApiScriptUtils class - qqq-middleware-api not on the classpath?");
+      }
+      catch(Exception e)
+      {
+         LOG.warn("Error converting QRecord list to api record list", e);
+      }
+
+      return (new ArrayList<>(input.getRecordList()));
    }
 
 
@@ -197,7 +213,7 @@ public class RunAdHocRecordScriptAction
             QueryInput queryInput = new QueryInput();
             queryInput.setTableName(ScriptRevision.TABLE_NAME);
             queryInput.setFilter(new QQueryFilter(new QFilterCriteria("script.id", QCriteriaOperator.EQUALS, codeReference.getScriptId())));
-            queryInput.withQueryJoin(new QueryJoin(Script.TABLE_NAME).withBaseTableOrAlias(ScriptRevision.TABLE_NAME).withJoinMetaData(QContext.getQInstance().getJoin("currentScriptRevision")));
+            queryInput.withQueryJoin(new QueryJoin(Script.TABLE_NAME).withBaseTableOrAlias(ScriptRevision.TABLE_NAME).withJoinMetaData(QContext.getQInstance().getJoin(ScriptsMetaDataProvider.CURRENT_SCRIPT_REVISION_JOIN_NAME)));
             QueryOutput queryOutput = new QueryAction().execute(queryInput);
 
             if(CollectionUtils.nullSafeHasContents(queryOutput.getRecords()))
@@ -216,4 +232,44 @@ public class RunAdHocRecordScriptAction
       throw (new QException("Code reference did not contain a scriptRevision, scriptRevisionId, or scriptId"));
    }
 
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private Optional<Script> getScript(ScriptRevision scriptRevision)
+   {
+      if(scriptRevision == null || scriptRevision.getScriptId() == null)
+      {
+         return (Optional.empty());
+      }
+
+      try
+      {
+         return scriptMemoizationById.getResult(scriptRevision.getScriptId(), scriptId ->
+         {
+            try
+            {
+               QRecord scriptRecord = new GetAction().executeForRecord(new GetInput(Script.TABLE_NAME).withPrimaryKey(scriptRevision.getScriptId()));
+               if(scriptRecord != null)
+               {
+                  Script script = new Script(scriptRecord);
+                  scriptMemoizationById.storeResult(scriptRevision.getScriptId(), script);
+                  return (script);
+               }
+            }
+            catch(Exception e)
+            {
+               LOG.info("");
+            }
+
+            return (null);
+         });
+      }
+      catch(Exception e)
+      {
+         return (Optional.empty());
+      }
+   }
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/scripts/RunAssociatedScriptAction.java

@@ -25,11 +25,7 @@ package com.kingsrook.qqq.backend.core.actions.scripts;
 import java.io.Serializable;
 import java.util.HashMap;
 import java.util.Map;
-import java.util.Objects;
 import com.kingsrook.qqq.backend.core.actions.ActionHelper;
-import com.kingsrook.qqq.backend.core.actions.scripts.logging.QCodeExecutionLoggerInterface;
-import com.kingsrook.qqq.backend.core.actions.scripts.logging.ScriptExecutionLoggerInterface;
-import com.kingsrook.qqq.backend.core.actions.scripts.logging.StoreScriptLogAndScriptLogLineExecutionLogger;
 import com.kingsrook.qqq.backend.core.actions.tables.GetAction;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.exceptions.QNotFoundException;
@@ -40,8 +36,6 @@ import com.kingsrook.qqq.backend.core.model.actions.scripts.RunAssociatedScriptO
 import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetOutput;
 import com.kingsrook.qqq.backend.core.model.metadata.code.AssociatedScriptCodeReference;
-import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
-import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeType;
 import com.kingsrook.qqq.backend.core.model.scripts.Script;
 import com.kingsrook.qqq.backend.core.model.scripts.ScriptRevision;
 
@@ -54,6 +48,7 @@ public class RunAssociatedScriptAction
    private Map<AssociatedScriptCodeReference, ScriptRevision> scriptRevisionCache = new HashMap<>();
 
 
+
    /*******************************************************************************
     **
     *******************************************************************************/
@@ -61,41 +56,19 @@ public class RunAssociatedScriptAction
    {
       ActionHelper.validateSession(input);
 
-      ScriptRevision scriptRevision = getScriptRevision(input);
+      ScriptRevision   scriptRevision   = getScriptRevision(input);
+      ExecuteCodeInput executeCodeInput = ExecuteCodeAction.setupExecuteCodeInput(input, scriptRevision);
 
-      ExecuteCodeInput executeCodeInput = new ExecuteCodeInput();
-      executeCodeInput.setInput(new HashMap<>(input.getInputValues()));
-      executeCodeInput.setContext(new HashMap<>());
-      if(input.getOutputObject() != null)
+      if(input.getAssociatedScriptContextPrimerInterface() != null)
       {
-         executeCodeInput.getContext().put("output", input.getOutputObject());
-      }
-
-      if(input.getScriptUtils() != null)
-      {
-         executeCodeInput.getContext().put("scriptUtils", input.getScriptUtils());
-      }
-
-      executeCodeInput.setCodeReference(new QCodeReference().withInlineCode(scriptRevision.getContents()).withCodeType(QCodeType.JAVA_SCRIPT)); // todo - code type as attribute of script!!
-
-      /////////////////////////////////////////////////////////////////////////////////////////////////
-      // let caller supply a logger, or by default use StoreScriptLogAndScriptLogLineExecutionLogger //
-      /////////////////////////////////////////////////////////////////////////////////////////////////
-      QCodeExecutionLoggerInterface executionLogger = Objects.requireNonNullElseGet(input.getLogger(), () -> new StoreScriptLogAndScriptLogLineExecutionLogger(scriptRevision.getScriptId(), scriptRevision.getId()));
-      executeCodeInput.setExecutionLogger(executionLogger);
-      if(executionLogger instanceof ScriptExecutionLoggerInterface scriptExecutionLoggerInterface)
-      {
-         ////////////////////////////////////////////////////////////////////////////////////////////////////
-         // if logger is aware of scripts (as opposed to a generic CodeExecution logger), give it the ids. //
-         ////////////////////////////////////////////////////////////////////////////////////////////////////
-         scriptExecutionLoggerInterface.setScriptId(scriptRevision.getScriptId());
-         scriptExecutionLoggerInterface.setScriptRevisionId(scriptRevision.getId());
+         input.getAssociatedScriptContextPrimerInterface().primeContext(executeCodeInput, scriptRevision);
       }
 
       ExecuteCodeOutput executeCodeOutput = new ExecuteCodeOutput();
       new ExecuteCodeAction().run(executeCodeInput, executeCodeOutput);
 
       output.setOutput(executeCodeOutput.getOutput());
+      output.setScriptRevisionId(scriptRevision.getId());
    }
 
 
@@ -138,6 +111,7 @@ public class RunAssociatedScriptAction
       GetInput getInput = new GetInput();
       getInput.setTableName("scriptRevision");
       getInput.setPrimaryKey(scriptRevisionId);
+      getInput.setIncludeAssociations(true);
       GetOutput getOutput = new GetAction().execute(getInput);
       if(getOutput.getRecord() == null)
       {

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/scripts/StoreAssociatedScriptAction.java

@@ -31,6 +31,8 @@ import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
 import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
 import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
+import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
 import com.kingsrook.qqq.backend.core.model.actions.scripts.StoreAssociatedScriptInput;
 import com.kingsrook.qqq.backend.core.model.actions.scripts.StoreAssociatedScriptOutput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetInput;
@@ -47,6 +49,7 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.AssociatedScript;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+import com.kingsrook.qqq.backend.core.processes.implementations.scripts.StoreScriptRevisionProcessStep;
 import com.kingsrook.qqq.backend.core.utils.StringUtils;
 
 
@@ -156,8 +159,7 @@ public class StoreAssociatedScriptAction
          queryInput.setFilter(new QQueryFilter()
             .withCriteria(new QFilterCriteria("scriptId", QCriteriaOperator.EQUALS, List.of(script.getValue("id"))))
             .withOrderBy(new QFilterOrderBy("sequenceNo", false))
-         );
-         queryInput.setLimit(1);
+            .withLimit(1));
          QueryOutput queryOutput = new QueryAction().execute(queryInput);
          if(!queryOutput.getRecords().isEmpty())
          {
@@ -181,39 +183,19 @@ public class StoreAssociatedScriptAction
          }
       }
 
-      QRecord scriptRevision = new QRecord()
-         .withValue("scriptId", script.getValue("id"))
-         .withValue("contents", input.getCode())
-         .withValue("commitMessage", commitMessage)
-         .withValue("sequenceNo", nextSequenceNo);
-
-      try
-      {
-         scriptRevision.setValue("author", input.getSession().getUser().getFullName());
-      }
-      catch(Exception e)
-      {
-         scriptRevision.setValue("author", "Unknown");
-      }
-
-      InsertInput insertInput = new InsertInput();
-      insertInput.setTableName("scriptRevision");
-      insertInput.setRecords(List.of(scriptRevision));
-      InsertOutput insertOutput = new InsertAction().execute(insertInput);
-      scriptRevision = insertOutput.getRecords().get(0);
-
-      ////////////////////////////////////////////////////
-      // update the script to point at the new revision //
-      ////////////////////////////////////////////////////
-      script.setValue("currentScriptRevisionId", scriptRevision.getValue("id"));
-      UpdateInput updateInput = new UpdateInput();
-      updateInput.setTableName("script");
-      updateInput.setRecords(List.of(script));
-      new UpdateAction().execute(updateInput);
+      RunBackendStepInput storeScriptRevisionInput = new RunBackendStepInput();
+      storeScriptRevisionInput.addValue("scriptId", script.getValue("id"));
+      storeScriptRevisionInput.addValue("commitMessage", commitMessage);
+      storeScriptRevisionInput.addValue("apiName", input.getApiName());
+      storeScriptRevisionInput.addValue("apiVersion", input.getApiVersion());
+      storeScriptRevisionInput.addValue("fileNames", "script");
+      storeScriptRevisionInput.addValue("fileContents:script", input.getCode());
+      RunBackendStepOutput storeScriptRevisionOutput = new RunBackendStepOutput();
+      new StoreScriptRevisionProcessStep().run(storeScriptRevisionInput, storeScriptRevisionOutput);
 
       output.setScriptId(script.getValueInteger("id"));
       output.setScriptName(script.getValueString("name"));
-      output.setScriptRevisionId(scriptRevision.getValueInteger("id"));
-      output.setScriptRevisionSequenceNo(scriptRevision.getValueInteger("sequenceNo"));
+      output.setScriptRevisionId(storeScriptRevisionOutput.getValueInteger("scriptRevisionId"));
+      output.setScriptRevisionSequenceNo(storeScriptRevisionOutput.getValueInteger("scriptRevisionSequenceNo"));
    }
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/scripts/TestScriptActionInterface.java

@@ -32,6 +32,7 @@ import com.kingsrook.qqq.backend.core.model.actions.scripts.ExecuteCodeOutput;
 import com.kingsrook.qqq.backend.core.model.actions.scripts.TestScriptInput;
 import com.kingsrook.qqq.backend.core.model.actions.scripts.TestScriptOutput;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
+import com.kingsrook.qqq.backend.core.model.scripts.ScriptRevision;
 
 
 /*******************************************************************************
@@ -47,7 +48,7 @@ public interface TestScriptActionInterface
     ** Note - such a method may want or need to put an "output" object into the
     ** executeCodeInput's context map.
     *******************************************************************************/
-   void setupTestScriptInput(TestScriptInput testScriptInput, ExecuteCodeInput executeCodeInput);
+   void setupTestScriptInput(TestScriptInput testScriptInput, ExecuteCodeInput executeCodeInput) throws QException;
 
 
    /*******************************************************************************
@@ -80,6 +81,9 @@ public interface TestScriptActionInterface
     *******************************************************************************/
    default void execute(TestScriptInput input, TestScriptOutput output) throws QException
    {
+      /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      // todo - could this be merged with the various other script runners, to use ExecuteCodeAction.setupExecuteCodeInput?? //
+      /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
       ExecuteCodeInput executeCodeInput = new ExecuteCodeInput();
       executeCodeInput.setContext(new HashMap<>());
 
@@ -87,12 +91,21 @@ public interface TestScriptActionInterface
       BuildScriptLogAndScriptLogLineExecutionLogger executionLogger = new BuildScriptLogAndScriptLogLineExecutionLogger(null, null);
       executeCodeInput.setExecutionLogger(executionLogger);
 
-      setupTestScriptInput(input, executeCodeInput);
-
-      ExecuteCodeOutput executeCodeOutput = new ExecuteCodeOutput();
-
       try
       {
+         setupTestScriptInput(input, executeCodeInput);
+
+         ScriptRevision scriptRevision = new ScriptRevision().withApiName(input.getApiName()).withApiVersion(input.getApiVersion());
+
+         if(this instanceof AssociatedScriptContextPrimerInterface associatedScriptContextPrimerInterface)
+         {
+            associatedScriptContextPrimerInterface.primeContext(executeCodeInput, scriptRevision);
+         }
+
+         ExecuteCodeOutput executeCodeOutput = new ExecuteCodeOutput();
+
+         ExecuteCodeAction.addApiUtilityToContext(executeCodeInput.getContext(), scriptRevision);
+
          new ExecuteCodeAction().run(executeCodeInput, executeCodeOutput);
          output.setOutputObject(processTestScriptOutput(executeCodeOutput));
       }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/scripts/logging/Log4jCodeExecutionLogger.java

@@ -41,6 +41,7 @@ public class Log4jCodeExecutionLogger implements QCodeExecutionLoggerInterface
    private QCodeReference qCodeReference;
    private String         uuid = UUID.randomUUID().toString();
 
+   private boolean includeUUID = true;
 
 
    /*******************************************************************************
@@ -52,7 +53,7 @@ public class Log4jCodeExecutionLogger implements QCodeExecutionLoggerInterface
       this.qCodeReference = executeCodeInput.getCodeReference();
 
       String inputString = StringUtils.safeTruncate(ValueUtils.getValueAsString(executeCodeInput.getInput()), 250, "...");
-      LOG.info("Starting script execution: " + qCodeReference.getName() + ", uuid: " + uuid + ", with input: " + inputString);
+      LOG.info("Starting script execution: " + qCodeReference.getName() + (includeUUID ? ", uuid: " + uuid : "") + ", with input: " + inputString);
    }
 
 
@@ -63,7 +64,7 @@ public class Log4jCodeExecutionLogger implements QCodeExecutionLoggerInterface
    @Override
    public void acceptLogLine(String logLine)
    {
-      LOG.info("Script log: " + uuid + ": " + logLine);
+      LOG.info("Script log: " + (includeUUID ? uuid + ": " : "") + logLine);
    }
 
 
@@ -74,7 +75,7 @@ public class Log4jCodeExecutionLogger implements QCodeExecutionLoggerInterface
    @Override
    public void acceptException(Exception exception)
    {
-      LOG.info("Script Exception: " + uuid, exception);
+      LOG.info("Script Exception: " + (includeUUID ? uuid : ""), exception);
    }
 
 
@@ -86,7 +87,38 @@ public class Log4jCodeExecutionLogger implements QCodeExecutionLoggerInterface
    public void acceptExecutionEnd(Serializable output)
    {
       String outputString = StringUtils.safeTruncate(ValueUtils.getValueAsString(output), 250, "...");
-      LOG.info("Finished script execution: " + qCodeReference.getName() + ", uuid: " + uuid + ", with output: " + outputString);
+      LOG.info("Finished script execution: " + qCodeReference.getName() + (includeUUID ? ", uuid: " + uuid : "") + ", with output: " + outputString);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for includeUUID
+    *******************************************************************************/
+   public boolean getIncludeUUID()
+   {
+      return (this.includeUUID);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for includeUUID
+    *******************************************************************************/
+   public void setIncludeUUID(boolean includeUUID)
+   {
+      this.includeUUID = includeUUID;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for includeUUID
+    *******************************************************************************/
+   public Log4jCodeExecutionLogger withIncludeUUID(boolean includeUUID)
+   {
+      this.includeUUID = includeUUID;
+      return (this);
    }
 
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/scripts/logging/QCodeExecutionLoggerInterface.java

@@ -29,7 +29,7 @@ import com.kingsrook.qqq.backend.core.model.actions.scripts.ExecuteCodeInput;
 /*******************************************************************************
  ** Interface to provide logging functionality to QCodeExecution (e.g., scripts)
  *******************************************************************************/
-public interface QCodeExecutionLoggerInterface
+public interface QCodeExecutionLoggerInterface extends Serializable
 {
 
    /*******************************************************************************

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/scripts/logging/SystemOutExecutionLogger.java

@@ -0,0 +1,88 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.actions.scripts.logging;
+
+
+import java.io.Serializable;
+import com.kingsrook.qqq.backend.core.model.actions.scripts.ExecuteCodeInput;
+import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
+import com.kingsrook.qqq.backend.core.utils.ValueUtils;
+
+
+/*******************************************************************************
+ ** Implementation of a code execution logger that logs to System.out and
+ ** System.err (for exceptions)
+ *******************************************************************************/
+public class SystemOutExecutionLogger implements QCodeExecutionLoggerInterface
+{
+   private QCodeReference qCodeReference;
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public void acceptExecutionStart(ExecuteCodeInput executeCodeInput)
+   {
+      this.qCodeReference = executeCodeInput.getCodeReference();
+
+      String inputString = ValueUtils.getValueAsString(executeCodeInput.getInput());
+      System.out.println("Starting script execution: " + qCodeReference.getName() + ", with input: " + inputString);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public void acceptLogLine(String logLine)
+   {
+      System.out.println("Script log: " + logLine);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public void acceptException(Exception exception)
+   {
+      System.out.println("Script Exception: " + exception.getMessage());
+      exception.printStackTrace();
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public void acceptExecutionEnd(Serializable output)
+   {
+      String outputString = ValueUtils.getValueAsString(output);
+      System.out.println("Finished script execution: " + qCodeReference.getName() + ", with output: " + outputString);
+   }
+
+}