ES-52: updated UpdateAction to act same as insert or gets for 'retryable server errors'

Add isEnabled method to meta-data producers; Put interface on top of …

.circleci/adjust-pom-version.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+
+if [ -z "$CIRCLE_BRANCH" ] && [ -z "$CIRCLE_TAG" ]; then
+   echo "Error:  env vars CIRCLE_BRANCH and CIRCLE_TAG were not set."
+   exit 1;
+fi
+
+if [ "$CIRCLE_BRANCH" == "dev" ] || [ "$CIRCLE_BRANCH" == "staging" ] || [ "$CIRCLE_BRANCH" == "main" ] || [ \! -z $(echo "$CIRCLE_TAG" | grep "^version-") ]; then
+   echo "On a primary branch or tag [${CIRCLE_BRANCH}${CIRCLE_TAG}] - will not edit the pom version.";
+   exit 0;
+fi
+
+if [ -n "$CIRCLE_BRANCH" ]; then
+   SLUG=$(echo $CIRCLE_BRANCH | sed 's/[^a-zA-Z0-9]/-/g')
+else
+   SLUG=$(echo $CIRCLE_TAG | sed 's/^snapshot-//g')
+fi
+
+POM=$(dirname $0)/../pom.xml
+
+echo "Updating $POM <revision> to: $SLUG-SNAPSHOT"
+sed -i "s/<revision>.*/<revision>$SLUG-SNAPSHOT<\/revision>/" $POM
+git diff $POM

.circleci/config.yml

@@ -26,6 +26,7 @@ commands:
             sudo rm /etc/alternatives/java
             sudo ln -s /usr/lib/jvm/java-17-openjdk-amd64/bin/java /etc/alternatives/java
       - run:
+          ## used by jacoco uncovered class reporting in pom.xml
           name: Install html2text
           command: |
             sudo apt-get update
@@ -51,10 +52,18 @@ commands:
           module: qqq-backend-module-filesystem
       - store_jacoco_site:
           module: qqq-backend-module-rdbms
+      - store_jacoco_site:
+          module: qqq-backend-module-api
+      - store_jacoco_site:
+          module: qqq-middleware-api
       - store_jacoco_site:
           module: qqq-middleware-javalin
       - store_jacoco_site:
           module: qqq-middleware-picocli
+      - store_jacoco_site:
+          module: qqq-middleware-slack
+      - store_jacoco_site:
+          module: qqq-language-support-javascript
       - store_jacoco_site:
           module: qqq-sample-project
       - run:
@@ -65,11 +74,6 @@ commands:
           when: always
       - store_test_results:
           path: ~/test-results
-      - run:
-          name: Find Un-tested Classes
-          command: |
-            set +o pipefail && for i in */target/site/jacoco/*/index.html; do html2text -width 500 -nobs $i | sed '1,/^Total/d;' | grep -v Created | sed 's/ \+/ /g' | sed 's/ [[:digit:]]$//' | grep -v 0$ | cut -d' ' -f1; done
-          when: always
       - save_cache:
           paths:
             - ~/.m2
@@ -78,6 +82,10 @@ commands:
   mvn_jar_deploy:
     steps:
       - checkout
+      - run:
+          name: Adjust pom version
+          command: |
+            .circleci/adjust-pom-version.sh
       - restore_cache:
           keys:
             - v1-dependencies-{{ checksum "pom.xml" }}
@@ -94,14 +102,14 @@ jobs:
   mvn_test:
     executor: localstack/default
     steps:
-      - localstack/startup
+      ## - localstack/startup
       - install_java17
       - mvn_verify
 
   mvn_deploy:
     executor: localstack/default
     steps:
-      - localstack/startup
+      ## - localstack/startup
       - install_java17
       - mvn_verify
       - mvn_jar_deploy

README.md

@@ -10,7 +10,7 @@ The bundle contains all of the sub-jars.  It is named:
 
 ```qqq-${version}.jar```
 
-You can also use fine grained jars:
+You can also use fine-grained jars:
 - `qqq-backend-core`: The core module.  Useful if you're developing other modules.
 - `qqq-backend-module-rdbms`: Backend module for working with Relational Databases.
 - `qqq-backend-module-filesystem`: Backend module for working with Filesystems (including AWS S3).
@@ -35,4 +35,3 @@ GNU Affero General Public License for more details.
 
 You should have received a copy of the GNU Affero General Public License
 along with this program.  If not, see <https://www.gnu.org/licenses/>.
-

docs/Introduction.adoc

@@ -0,0 +1,8 @@
+= Introduction
+
+QQQ is ...
+
+- Framework
+- Declarative
+- Easy thing easy; Hard thing possible
+- Customizable

docs/actions/QueryAction.adoc

@@ -0,0 +1,179 @@
+== QueryAction
+include::../variables.adoc[]
+
+The `*QueryAction*` is the basic action that is used to get records from a {link-table}.
+In SQL/RDBMS terms, it is analogous to a `SELECT` statement, where 0 or more records may be found and returned.
+
+=== Examples
+==== Basic Form
+[source,java]
+----
+QueryInput input = new QueryInput(qInstance);
+input.setSession(session);
+input.setTableName("orders");
+input.setFilter(new QQueryFilter()
+   .withCriteria(new QFilterCriteria("total", GREATER_THAN, new BigDecimal("3.50")))
+   .withOrderBy(new QFilterOrderBy("orderDate", false))
+);
+QueryOutput output = new QueryAction.execute(input);
+List<QRecord> records = output.getRecords();
+----
+
+=== QueryInput
+* `table` - *String, Required* - Name of the table being queried against.
+* `filter` - *QQueryFilter object* - Specification for what records should be returned, based on *QFilterCriteria* objects, and how they should be sorted, based on *QFilterOrderBy* objects.
+* `skip` - *Integer* - Optional number of records to be skipped at the beginning of the result set.
+e.g., for implementing pagination.
+* `limit` - *Integer* - Optional maximum number of records to be returned by the query.
+* `transaction` - *QBackendTransaction object* - Optional transaction object.
+** Behavior for this object is backend-dependant.
+In an RDBMS backend, this object is generally needed if you want your query to see data that may have been modified within the same transaction.
+* `recordPipe` - *RecordPipe object* - Optional object that records are placed into, for asynchronous processing.
+** If a *recordPipe* is used, then records cannot be retrieved from the *QueryOutput*.
+Rather, such records must be read from the pipe's `consumeAvailableRecords()` method.
+** A *recordPipe* should only be used when a *QueryAction* is running in a separate Thread from the record's consumer.
+* `shouldTranslatePossibleValues` - *boolean, default: false* - Controls whether any fields in the table with a *possibleValueSource* assigned to them should have those possible values looked up
+(e.g., to provide text translations in the generated records' `displayValues` map).
+** For example, if running a query to present results to a user, this would generally need to be *true*.
+But if running a query to provide data as part of a process, then this can generally be left as *false*.
+* `shouldGenerateDisplayValues` - *boolean, default: false* - Controls whether if field level *displayFormats* should be used to populate the generated records' `displayValues` map.
+** For example, if running a query to present results to a user, this would generally need to be *true*.
+But if running a query to provide data as part of a process, then this can generally be left as *false*.
+* `queryJoins` - *List of QueryJoin objects* - Optional list of tables to be joined with the main *table* specified in the *QueryInput*.
+See QueryJoin below for further details.
+
+==== QQueryFilter
+A key component of *QueryInput*, a *QQueryFilter* defines both what records should be included in a query's results (e.g., an SQL `WHERE`), as well as how those results should be sorted (SQL `ORDER BY`).
+
+* `criteria` - *List of QFilterCriteria* - Individual conditions or clauses to filter records.
+They are combined using the *booleanOperator* specified in the *QQueryFilter*.  See below for further details.
+* `orderBys` - *List of QFilterOrderBy* - List of fields (and directions) to control the sorting of query results.
+In general, multiple *orderBys* can be given (depending on backend implementations).
+* `booleanOperator` - *Enum of AND, OR, default: AND* - Specifies the logical joining operator used among individual criteria.
+* `subFilters` - *List of QQueryFilter* - To build arbitrarily complex queries, with nested boolean logic, 0 or more *subFilters* may be provided.
+** Each *subFilter* can include its own additional *subFilters*.
+** Each *subFilter* can specify a different *booleanOperator*.
+** For example, consider the following *QQueryFilter*, that uses two *subFilters*, and a mix of *booleanOperators*
+
+[source,java]
+----
+ queryInput.setFilter(new QQueryFilter()
+    .withBooleanOperator(OR)
+    .withSubFilters(List.of(
+       new QQueryFilter().withBooleanOperator(AND)
+          .withCriteria(new QFilterCriteria("firstName", EQUALS, "James"))
+          .withCriteria(new QFilterCriteria("lastName", EQUALS, "Maes")),
+       new QQueryFilter().withBooleanOperator(AND)
+          .withCriteria(new QFilterCriteria("firstName", EQUALS, "Darin"))
+          .withCriteria(new QFilterCriteria("lastName", EQUALS, "Kelkhoff"))
+    )));
+
+// which would generate the following WHERE clause in an RDBMS backend:
+   WHERE (first_name='James' AND last_name='Maes') OR (first_name='Darin' AND last_name='Kelkhoff')
+----
+
+===== QFilterCriteria
+* `fieldName` - *String, required* - Reference to a field on the table being queried.
+** Or, in the case of a query with *queryJoins*, a qualified name of a field from a join-table (where the qualifier would be the joined table's name or alias, followed by a dot)
+*** For example:  `orderLine.sku` or `orderBillToCustomer.firstName`
+* `operator` - *Enum of QCriteriaOperator, required* - Comparison operation to be applied to the field specified as *fieldName* and the *values* or *otherFieldName*.
+** e.g., `EQUALS`, `NOT_IN`, `GREATER_THAN`, `BETWEEN`, `IS_BLANK`, etc.
+* `values` - *List of values, conditional* - Provides the value(s) that the field is compared against.
+The number of values (0, 1, 2, or more) be driven based on the *operator* being used.
+If an *otherFieldName* is given, and the *operator* expects 1 value, then *values* is ignored, and *otherFieldName* is used.
+* `otherFieldName` - *String, conditional* - Specifies that the *fieldName* should be compared against another field in the records, rather than the values in the *values* property.
+Only used for *operators* that expect 1 value (e.g., `EQUALS` or `LESS_THAN_OR_EQUALS` - not `IS_NOT_BLANK` or `IN`).
+
+QFilterCriteria definition examples:
+[source,java]
+----
+// one-liners, via constructors that take (List<Serializable> values) or (Serializable... values) in 3rd position
+new QFilterCriteria("id", IN, List.of(1, 2, 3))
+new QFilterCriteria("name", IS_BLANK)
+new QFilterCriteria("orderNo", IN, orderNoList)
+new QFilterCriteria("state", EQUALS, "MO");
+
+// long-form, with fluent setters
+new QFilterCriteria()
+   .withFieldName("quantity")
+   .withOpeartor(QCriteriaOperator.GREATER_THAN)
+   .withValues(List.of(47));
+
+// to use otherFieldName, long-form must be used
+new QFilterCriteria()
+   .withFieldName("firstName")
+   .withOpeartor(QCriteriaOperator.EQUALS)
+   .withOtherFieldName("lastName");
+
+// using otherFieldName to build a criterion that looks at two fields from join tables
+new QFilterCriteria()
+   .withFieldName("billToCustomer.lastName")
+   .withOpeartor(QCriteriaOperator.NOT_EQUALS)
+   .withOtherFieldName("shipToCustomer.lastName");
+
+----
+
+===== QFilterOrderBy
+* `fieldName` - *String, required* - Reference to a field on the table being queried.
+** Or, in the case of a query with *queryJoins*, a qualified name of a field from a join-table (where the qualifier would be the joined table's name or alias, followed by a dot)
+* `isAscending` - *boolean, default: true* - Specify if the sort is ascending or descending.
+
+QFilterCriteria definition examples:
+
+[source,java]
+----
+// short-form, via constructors
+new QFilterOrderBy("id") // isAscending defaults to true.
+new QFilterOrderBy("name", false)
+
+// long-form, with fluent setters
+new QFilterOrderBy()
+   .withFieldName("birthDate")
+   .withIsAscending(true);
+----
+
+==== QueryJoin
+* `joinTable` - *String, required* - Name of the table that is being joined in to the existing query.
+** Will be inferred from *joinMetaData*, if *joinTable* is not set when *joinMetaData* gets set.
+* `baseTableOrAlias` - *String, required* - Name of a table (or an alias) already defined in the query, to which the *joinTable* will be joined.
+** Will be inferred from *joinMetaData*, if *baseTableOrAlias* is not set when *joinMetaData* gets set (which will only use the leftTableName from the joinMetaData - never an alias).
+* `joinMetaData` - *QJoinMetaData object* - Optional specification of a {link-join} in the current QInstance.
+If not set, will be looked up at runtime based on *baseTableOrAlias* and *joinTable*.
+** If set before *baseTableOrAlias* and *joinTable*, then they will be set based on the *leftTable* and *rightTable* in this object.
+* `alias` - *String* - Optional (unless multiple instances of the same table are being joined together, when it becomes required).
+Behavior based on SQL `FROM` clause aliases.
+If given, must be used as the part before the dot in field name specifications throughout the rest of the query input.
+* `select` - *boolean, default: false* - Specify whether fields from the *rightTable* should be selected by the query.
+If *true*, then the `QRecord` objects returned by this query will have values with corresponding to the (table-or-alias `.` field-name) form.
+* `type` - *Enum of INNER, LEFT, RIGHT, FULL, default: INNER* - specifies the SQL-style type of join being performed.
+
+QueryJoin definition examples:
+
+[source,java]
+----
+// selecting from an "orderLine" table - then join to its corresponding "order" table
+queryInput.withTableName("orderLine");
+queryInput.withQueryJoin(new QueryJoin("order").withSelect(true));
+...
+queryOutput.getRecords().get(0).getValueBigDecimal("order.grandTotal");
+
+// given an "order" table with 2 foreign keys to a customer table (billToCustomerId and shipToCustomerId)
+// Note, we must supply the JoinMetaData to the QueryJoin, to drive what fields to join on in each case.
+queryInput.withTableName("order");
+queryInput.withQueryJoins(List.of(
+   new QueryJoin(instance.getJoin("orderJoinShipToCustomer")
+       .withAlias("shipToCustomer")
+       .withSelect(true)),
+   new QueryJoin(instance.getJoin("orderJoinBillToCustomer")
+       .withAlias("billToCustomer")
+       .withSelect(true))));
+...
+record.getValueString("billToCustomer.firstName")
+   + " placed an order for "
+   + record.getValueString("shipToCustomer.firstName")
+
+----
+
+=== QueryOutput
+* `records` - *List of QRecord* - List of 0 or more records that match the query filter.
+** _Note:  If a *recordPipe* was supplied to the QueryInput, then calling `queryOutput.getRecords()` will result in an `IllegalStateException` being thrown - as the records were placed into the pipe as they were fetched, and cannot all be accessed as a single list._

docs/actions/RenderTemplateAction.adoc

@@ -0,0 +1,33 @@
+== RenderTemplateAction
+include::../variables.adoc[]
+
+The `*RenderTemplateAction*` performs the job of taking a template - that is, a string of code, in a templating language, such as https://velocity.apache.org/engine/1.7/user-guide.html[Velocity], and merging it with a set of data (known as a context), to produce some using-facing output, such as a String of HTML.
+
+=== Examples
+==== Canonical Form
+[source,java]
+----
+RenderTemplateInput input = new RenderTemplateInput(qInstance);
+input.setSession(session);
+input.setCode("Hello, ${name}");
+input.setTemplateType(TemplateType.VELOCITY);
+input.setContext(Map.of("name", "Darin"));
+RenderTemplateOutput output = new RenderTemplateAction.execute(input);
+String result = output.getResult();
+assertEquals("Hello, Darin", result);
+----
+
+==== Convenient Form
+[source,java]
+----
+String result = RenderTemplateAction.renderVelocity(input, Map.of("name", "Darin"), "Hello, ${name}");
+assertEquals("Hello, Darin", result);
+----
+
+=== RenderTemplateInput
+* `code` - *String, Required* - String of template code to be rendered, in the templating language specified by the `type` parameter.
+* `type` - *Enum of VELOCITY, Required* - Specifies the language of the template code.
+* `context` - *Map of String → Object* - Data to be made available to the template during rendering.
+
+=== RenderTemplateOutput
+* `result` - *String* - Result of rendering the input template and context.

docs/docinfo.html

@@ -0,0 +1,27 @@
+<!--
+  ~ QQQ - Low-code Application Framework for Engineers.
+  ~ Copyright (C) 2021-2022.  Kingsrook, LLC
+  ~ 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+  ~ contact@kingsrook.com
+  ~ https://github.com/Kingsrook/
+  ~
+  ~ This program is free software: you can redistribute it and/or modify
+  ~ it under the terms of the GNU Affero General Public License as
+  ~ published by the Free Software Foundation, either version 3 of the
+  ~ License, or (at your option) any later version.
+  ~
+  ~ This program is distributed in the hope that it will be useful,
+  ~ but WITHOUT ANY WARRANTY; without even the implied warranty of
+  ~ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+  ~ GNU Affero General Public License for more details.
+  ~
+  ~ You should have received a copy of the GNU Affero General Public License
+  ~ along with this program.  If not, see <https://www.gnu.org/licenses/>.
+  -->
+
+<style>
+   pre
+   {
+      white-space: pre;
+   }
+</style>

docs/index.adoc

@@ -0,0 +1,17 @@
+= QQQ
+:doctype: book
+:toc: left
+:source-highlighter: coderay
+
+include::Introduction.adoc[leveloffset=+1]
+
+== Meta Data
+include::metaData/Tables.adoc[leveloffset=+1]
+''''
+include::metaData/Reports.adoc[leveloffset=+1]
+
+== Actions
+include::actions/QueryAction.adoc[leveloffset=+1]
+''''
+include::actions/RenderTemplateAction.adoc[leveloffset=+1]
+''''

docs/metaData/Fields.adoc

@@ -0,0 +1,24 @@
+== QQQ Fields
+include::../variables.adoc[]
+
+QQQ Fields define
+
+=== QFieldMetaData
+*QFieldMetaData Properties:*
+
+* `name` - *String, Required* - Unique name for the field within its container (table, process, etc).
+* `label` - *String* - User-facing label for the field, presented in User Interfaces.
+* `type` - *enum of QFieldType, Required* - Data type for values in the field.
+* `backendName` - *String* - Name of the field within its backend.
+** For example, in an RDBMS-backed table, a field's `name` may be written in camel case, but its `backendName` written with underscores.
+* `isRequired` - *boolean, default false* - Indicator that a value is required in this field.
+* `isEditable` - *boolean, default true* - Indicator that users may edit values in this field.
+* `displayFormat` - *String, default `%s`* - Java Format Specifier string, used to format values in the field for display in user interfaces.
+Used to set values in the `displayValues` map within a `QRecord`.
+** Recommended values for `displayFormat` come from the `DisplayFormat` interface, such as `DisplayFormat.CURRENCY`, `DisplayFormat.COMMAS`, or `DisplayFormat.DECIMAL2_COMMAS`.
+* `defaultValue` - Value to use for the field if no other value is given.  Type is based on the field's `type`.
+* `possibleValueSourceName` - *String* - Reference to a {link-pvs} to be used for this field.
+Values in this field should correspond to ids from the referenced Possible Value Source.
+* `maxLength` - *Integer* - Maximum length (number of characters) allowed for values in this field.
+Only applicable for fields with `type=STRING`.
+* `

docs/metaData/Reports.adoc

@@ -0,0 +1,173 @@
+== QQQ Reports
+include::../variables.adoc[]
+
+QQQ can generate reports based on {link-tables} defined within a QQQ Instance.
+Users can run reports, providing input values.
+Alternatively, application code can run reports as needed, supplying input values.
+
+=== QReportMetaData
+Reports are defined in a QQQ Instance with a `*QReportMetaData*` object.
+Reports are defined in terms of their sources of data (`QReportDataSource`), and their view(s) of that data (`QReportView`).
+
+*QReportMetaData Properties:*
+
+* `name` - *String, Required* - Unique name for the report within the QQQ Instance.
+* `label` - *String* - User-facing label for the report, presented in User Interfaces.
+Inferred from `name` if not set.
+* `processName` - *String* - Name of a {link-process} used to run the report in a User Interface.
+* `inputFields` - *List of {link-field}* - Optional list of fields used as input to the report.
+** The values in these fields can be used via the syntax `${input.NAME}`, where `NAME` is the `name` attribute of the `inputField`.
+** For example:
+
+[source,java]
+----
+// given this inputField:
+new QFieldMetaData("storeId", QFieldType.INTEGER)
+
+// its run-time value can be accessed, e.g., in a query filter under a data source:
+new QFilterCriteria("storeId", QCriteriaOperator.EQUALS, List.of("${input.storeId}"))
+
+// or in a report view's title or field formulas:
+.withTitleFields(List.of("${input.storeId}"))
+new QReportField().withName("storeId").withFormula("${input.storeId}")
+----
+
+* `dataSources` - *List of QReportDataSource, Required* - Definitions of the sources of data for the report.
+At least one is required.
+
+==== QReportDataSource
+Data sources for QQQ Reports can either reference {link-tables} within the QQQ Instance, or they can provide custom code in the form of a `CodeReference` to a  `Supplier`, for use cases such as a static data tab in an Excel report.
+
+*QReportDataSource Properties:*
+
+* `name` - *String, Required* - Unique name for the data source within its containing Report.
+* `sourceTable` - *String, Conditional* - Reference to a {link-table} in the QQQ Instance, which the data source queries data from.
+* `queryFilter` - *QQueryFilter* - If a `sourceTable` is defined, then the filter specified here is used to filter and sort the records queried from that table when generating the report.
+* `staticDataSupplier` - *QCodeReference, Conditional* - Reference to custom code which can be used to supply the data for the data source, as an alternative to querying a `sourceTable`.
+** Must be a `JAVA` code type
+** Must be a `REPORT_STATIC_DATA_SUPPLIER` code usage.
+** The referenced class must implement the interface: `Supplier<List<List<Serializable>>>`.
+
+==== QReportView
+Report Views control how the source data for a report is organized and presented to the user in the output report file.
+If a DataSource describes the rows for a report (e.g., what table provides what records), then a View may be thought of as describing the columns in the report.
+A single report can have multiple views, specifically, for the use-case where an Excel file is being generated, in which case each View creates a tab or sheet within the `xlsx` file.
+
+*QReportView Properties:*
+
+* `name` - *String, Required* - Unique name for the view within its containing Report.
+* `label` - *String* - Used as a sheet (tab) label in Excel formatted reports.
+* `type` - *enum of TABLE, SUMMARY, PIVOT. Required* - Defines the type of view being defined.
+** *TABLE* views are a simple listing of the records from the data source.
+** *SUMMARY* views are essentially pre-computed Pivot Tables.
+That is to say, the aggregation done by a Pivot Table in a spreadsheet file is done by QQQ while generating the report.
+In this way, a non-spreadsheet report (e.g., PDF or CSV) can have summarized data, as though it were a Pivot Table in a live spreadsheet.
+** *PIVOT* views produce actual Pivot Tables, and are only supported in Excel files _(and are not supported at the time of this writing)_.
+* `dataSourceName` - *String, Required* - Reference to a DataSource within the report, that is used to provide the rows for the view.
+* `varianceDataSourceName` - *String* - Optional reference to a second DataSource within the report, that is used in `*SUMMARY*` type views for computing variances.
+** For example, given a Data Source with a filter that selects all sales records for a given year, a Variance Data Source may have a filter that selects the previous year, for doing comparissons.
+* `pivotFields` - *List of String, Conditional* - For *SUMMARY* or *PIVOT* type views, specify the field(s) used as pivot rows.
+** For example, in a summary view of orders, you may "pivot" on the *customerId* field, to produce one row per-customer, with aggregate data for that customer.
+* `titleFormat` - *String* - Java Format String, used with `titleFields` (if given), to produce a title row, e.g., first row in the view (before any rows from the data source).
+* `titleFields` - *List of String, Conditional* - Used with `titleFormat`, to provide values for any format specifiers in the format string.
+Syntax to reference a field (e.g., from a report input field) is: `${input.NAME}`, where `NAME` is the `name` attribute of the inputField.
+** Example of using `titleFormat` and `titleFields`:
+
+[source,java]
+----
+// given these inputFields:
+new QFieldMetaData("startDate", QFieldType.DATE)
+new QFieldMetaData("endDate", QFieldType.DATE)
+
+// a view can have a title row like this:
+.withTitleFormat("Weekly Sales Report - %s - %s")
+.withTitleFields(List.of("${input.startDate}", "${input.endDate}"))
+----
+
+* `includeHeaderRow` - *boolean, default true* - Indication that first row of the view should be the column labels.
+** If true, then header row is put in the view.
+** If false, then no header row is put in the view.
+* `includeTotalRow` - *boolean, default false* - Indication that a totals row should be added to the view.
+All numeric columns are summed to produce values in the totals row.
+** If true, then totals row is put in the view.
+** If false, then no totals row is put in the view.
+* `includePivotSubTotals` - *boolean, default false* - For a *SUMMARY* or *PIVOT* type view, if there are more than 1 *pivotFields* being used, this field is an indication that each higher-level pivot should include sub-totals.
+** #TODO - provide example#
+* `columns` - *List of QReportField, required* - Definition of the columns to appear in the view.  See section on QReportField for details.
+* `orderByFields` - *List of QFilterOrderBy, optional* - For a *SUMMARY* or *PIVOT* type view, how to sort the rows.
+* `recordTransformStep` - *QCodeReference, subclass of `AbstractTransformStep`* - Custom code reference that can be used to transform records after they are queried from the data source, and before they are placed into the view.
+Can be used to transform or customize values, or to look up additional values to add to the report.
+** #TODO - provide example#
+* `viewCustomizer` - *QCodeReference, implementation of interface `Function<QReportView, QReportView>`* - Custom code reference that can be used to customize the report view, at runtime.
+Can be used, for example, to dynamically define the report's *columns*.
+** #TODO - provide example#
+
+===== QReportField
+Report Fields define the fields (AKA columns) of data that appear in a report view.
+These fields can either be direct references to fields from the report's data sources, or values computed using formula defined in the QReportField.
+
+*QReportField Properties:*
+
+* `name` - *String, required* - Unique identifier for the field within its ReportView.
+In general, will be a reference to a field from the ReportView's DataSource *unless a *formula* is given (for *SUMMARY* type views), the field is marked as *isVirtual*, or the field is marked as *showPossibleValueLabel*).
+* `label` - *String* - Optional text label to identify the field, for example, in a header row.
+If not given, may be derived from field, where possible.
+* `type` - *QFieldType*
+* `formula` - *String, conditional* - Required for *SUMMARY* type views.
+Defines the formula to be used for computing the value in this field.
+** For example:
+
+[source,java]
+----
+.withName("reportEndDate").withFormula("${input.endDate}")
+
+.withName("count").withFormula("${pivot.count.id}")
+
+.withName("percentOfTotal").withFormula("=DIVIDE(${pivot.count.id},${total.count.id})")
+
+.withName("sumCost").withFormula("${pivot.sum.cost}")
+
+.withName("sumCharge").withFormula("${pivot.sum.charge}")
+
+.withName("profit").withFormula("=MINUS(${pivot.sum.charge},${pivot.sum.cost})")
+
+.withName("totalCost").withFormula("=DIVIDE_SCALE(${pivot.sum.cost},${pivot.count.id},2)")
+
+.withName("revenuePer").withFormula("=DIVIDE_SCALE(${pivot.sum.charge},${pivot.count.id},2)")
+
+.withName("marginPer").withFormula("=MINUS(DIVIDE_SCALE(${pivot.sum.charge},${pivot.count.id},2),DIVIDE_SCALE(${pivot.sum.cost},${pivot.count.id},2))")
+
+.withName("thisWeekMargin").withFormula("=SCALE(DIVIDE(${thisRow.profit},${pivot.sum.charge}),2)")
+
+.withName("previousWeekProfit").withFormula("=MINUS(${variancePivot.sum.charge},${variancePivot.sum.cost})")
+
+.withName("previousWeekMargin").withFormula("=SCALE(DIVIDE(${thisRow.previousWeekProfit},${variancePivot.sum.charge}),2)")
+
+.withName("marginThisVsPrevious").withFormula("=SCALE(MINUS(${thisRow.margin},${thisRow.marginPrevious}),3)")
+
+.withName("exception").withFormula("""
+   =IF(LT(${thisRow.margin},0),Negative Margin,IF(LT(${thisRow.marginThisVsPrevious},0),Margin Decreased,""))""")
+----
+
+* `displayFormat` *String*
+* `isVirtual` *Boolean, default false* - (needs reviewed - may only be required for report views using a data source with a *staticDataSupplier*)
+* `showPossibleValueLabel` *Boolean, default false* - To show a translated value for a Possible Value field (e.g., a name or other value meaningful to a user, instead of a foreign key).
+* `sourceFieldName` *String* - Used for the scenario where a possibleValue field is included in a report both as the foreign key (raw, id value), and the translated "label" value.
+In that case, the field marked with *showPossibleValueLabel* = true should be given a different name, and should use *sourceFieldName* to indicate the field that has the id value.
+** For example:
+
+[source,java]
+----
+// this field would have the "raw" warehouseId values
+// e.g., integers - foreign keys to a warehouse table.  Generally useful for machines to know.
+new QReportField("warehouseId")
+   .withLabel("Warehouse Id"),
+
+// this field would have the translated values from the warehouse PossibleValueSource
+// for example, maybe the name field from the warehouse table.  A string, useful for humans to read.
+new QReportField("warehouseName")
+   .withSourceFieldName("warehouseId")
+   .withShowPossibleValueLabel(true)
+   .withLabel("Warehouse Name"),
+----
+

docs/metaData/Tables.adoc

@@ -0,0 +1,49 @@
+== QQQ Tables
+include::../variables.adoc[]
+
+The core type of object in a QQQ Instance is the Table.
+In the most common use-case, a QQQ Table may be the in-app representation of a Database table.
+That is, it is a collection of records (or rows) of data, each of which has a set of fields (or columns).
+
+QQQ also allows other types of data sources ({link-backends}) to be used as tables, such as File systems, API's, Java enums or objects, etc.
+All of these backend types present the same interfaces (both user-interfaces, and application programming interfaces), regardless of their backend type.
+
+=== QTableMetaData
+Tables are defined in a QQQ Instance in a `*QTableMetaData*` object.
+All tables must reference a {link-backend}, a list of fields that define the shape of records in the table, and additional data to describe how to work with the table within its backend.
+
+*QTableMetaData Properties:*
+
+* `name` - *String, Required* - Unique name for the table within the QQQ Instance.
+* `label` - *String* - User-facing label for the table, presented in User Interfaces.
+Inferred from `name` if not set.
+* `backendName` - *String, Required* - Name of a {link-backend} in which this table's data is managed.
+* `fields` - *Map of String → {link-field}, Required* - The columns of data that make up all records in this table.
+* `primaryKeyField` - *String, Conditional* - Name of a {link-field} that serves as the primary key (e.g., unique identifier) for records in this table.
+* `uniqueKeys` - *List of UniqueKey* - Definition of additional unique constraints (from an RDBMS point of view) from the table.
+e.g., sets of columns which must have unique values for each record in the table.
+* `backendDetails` - *QTableBackendDetails or subclass* - Additional data to configure the table within its {link-backend}.
+* `automationDetails` - *QTableAutomationDetails* - Configuration of automated jobs that run against records in the table, e.g., upon insert or update.
+* `customizers` - *Map of String → QCodeReference* - References to custom code that are injected into standard table actions, that allow applications to customize certain parts of how the table works.
+* `parentAppName` - *String* - Name of a {link-app} that this table exists within.
+* `icon` - *QIcon* - Icon associated with this table in certain user interfaces.
+* `recordLabelFormat` - *String* - Java Format String, used with `recordLabelFields` to produce a label shown for records from the table.
+* `recordLabelFields` - *List of String, Conditional* - Used with `recordLabelFormat` to provide values for any format specifiers in the format string.
+These strings must be field names within the table.
+** Example of using `recordLabelFormat` and `recordLabelFields`:
+
+[source,java]
+----
+// given these fields in the table:
+new QFieldMetaData("name", QFieldType.STRING)
+new QFieldMetaData("birthDate", QFieldType.DATE)
+
+// We can produce a record label such as "Darin Kelkhoff (1980-05-31)" via:
+.withRecordLabelFormat("%s (%s)")
+.withRecordLabelFields(List.of("name", "birthDate"))
+----
+* `sections` - *List of QFieldSection* - Mechanism to organize fields within user interfaces, into logical sections.
+If any sections are present in the table meta data, then all fields in the table must be listed in exactly 1 section.
+If no sections are defined, then instance enrichment will define default sections.
+* `associatedScripts` - *List of AssociatedScript* - Definition of user-defined scripts that can be associated with records within the table.
+* `enabledCapabilities` and `disabledCapabilities` - *Set of Capability enum values* - Overrides from the backend level, for capabilities that this table does or does not possess.

docs/metaData/Tables.html

@@ -0,0 +1,553 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+<meta charset="UTF-8">
+<meta http-equiv="X-UA-Compatible" content="IE=edge">
+<meta name="viewport" content="width=device-width, initial-scale=1.0">
+<meta name="generator" content="Asciidoctor 2.0.18">
+<title>QQQ Tables</title>
+<link rel="stylesheet" href="https://fonts.googleapis.com/css?family=Open+Sans:300,300italic,400,400italic,600,600italic%7CNoto+Serif:400,400italic,700,700italic%7CDroid+Sans+Mono:400,700">
+<style>
+/*! Asciidoctor default stylesheet | MIT License | https://asciidoctor.org */
+/* Uncomment the following line when using as a custom stylesheet */
+/* @import "https://fonts.googleapis.com/css?family=Open+Sans:300,300italic,400,400italic,600,600italic%7CNoto+Serif:400,400italic,700,700italic%7CDroid+Sans+Mono:400,700"; */
+html{font-family:sans-serif;-webkit-text-size-adjust:100%}
+a{background:none}
+a:focus{outline:thin dotted}
+a:active,a:hover{outline:0}
+h1{font-size:2em;margin:.67em 0}
+b,strong{font-weight:bold}
+abbr{font-size:.9em}
+abbr[title]{cursor:help;border-bottom:1px dotted #dddddf;text-decoration:none}
+dfn{font-style:italic}
+hr{height:0}
+mark{background:#ff0;color:#000}
+code,kbd,pre,samp{font-family:monospace;font-size:1em}
+pre{white-space:pre-wrap}
+q{quotes:"\201C" "\201D" "\2018" "\2019"}
+small{font-size:80%}
+sub,sup{font-size:75%;line-height:0;position:relative;vertical-align:baseline}
+sup{top:-.5em}
+sub{bottom:-.25em}
+img{border:0}
+svg:not(:root){overflow:hidden}
+figure{margin:0}
+audio,video{display:inline-block}
+audio:not([controls]){display:none;height:0}
+fieldset{border:1px solid silver;margin:0 2px;padding:.35em .625em .75em}
+legend{border:0;padding:0}
+button,input,select,textarea{font-family:inherit;font-size:100%;margin:0}
+button,input{line-height:normal}
+button,select{text-transform:none}
+button,html input[type=button],input[type=reset],input[type=submit]{-webkit-appearance:button;cursor:pointer}
+button[disabled],html input[disabled]{cursor:default}
+input[type=checkbox],input[type=radio]{padding:0}
+button::-moz-focus-inner,input::-moz-focus-inner{border:0;padding:0}
+textarea{overflow:auto;vertical-align:top}
+table{border-collapse:collapse;border-spacing:0}
+*,::before,::after{box-sizing:border-box}
+html,body{font-size:100%}
+body{background:#fff;color:rgba(0,0,0,.8);padding:0;margin:0;font-family:"Noto Serif","DejaVu Serif",serif;line-height:1;position:relative;cursor:auto;-moz-tab-size:4;-o-tab-size:4;tab-size:4;word-wrap:anywhere;-moz-osx-font-smoothing:grayscale;-webkit-font-smoothing:antialiased}
+a:hover{cursor:pointer}
+img,object,embed{max-width:100%;height:auto}
+object,embed{height:100%}
+img{-ms-interpolation-mode:bicubic}
+.left{float:left!important}
+.right{float:right!important}
+.text-left{text-align:left!important}
+.text-right{text-align:right!important}
+.text-center{text-align:center!important}
+.text-justify{text-align:justify!important}
+.hide{display:none}
+img,object,svg{display:inline-block;vertical-align:middle}
+textarea{height:auto;min-height:50px}
+select{width:100%}
+.subheader,.admonitionblock td.content>.title,.audioblock>.title,.exampleblock>.title,.imageblock>.title,.listingblock>.title,.literalblock>.title,.stemblock>.title,.openblock>.title,.paragraph>.title,.quoteblock>.title,table.tableblock>.title,.verseblock>.title,.videoblock>.title,.dlist>.title,.olist>.title,.ulist>.title,.qlist>.title,.hdlist>.title{line-height:1.45;color:#7a2518;font-weight:400;margin-top:0;margin-bottom:.25em}
+div,dl,dt,dd,ul,ol,li,h1,h2,h3,#toctitle,.sidebarblock>.content>.title,h4,h5,h6,pre,form,p,blockquote,th,td{margin:0;padding:0}
+a{color:#2156a5;text-decoration:underline;line-height:inherit}
+a:hover,a:focus{color:#1d4b8f}
+a img{border:0}
+p{line-height:1.6;margin-bottom:1.25em;text-rendering:optimizeLegibility}
+p aside{font-size:.875em;line-height:1.35;font-style:italic}
+h1,h2,h3,#toctitle,.sidebarblock>.content>.title,h4,h5,h6{font-family:"Open Sans","DejaVu Sans",sans-serif;font-weight:300;font-style:normal;color:#ba3925;text-rendering:optimizeLegibility;margin-top:1em;margin-bottom:.5em;line-height:1.0125em}
+h1 small,h2 small,h3 small,#toctitle small,.sidebarblock>.content>.title small,h4 small,h5 small,h6 small{font-size:60%;color:#e99b8f;line-height:0}
+h1{font-size:2.125em}
+h2{font-size:1.6875em}
+h3,#toctitle,.sidebarblock>.content>.title{font-size:1.375em}
+h4,h5{font-size:1.125em}
+h6{font-size:1em}
+hr{border:solid #dddddf;border-width:1px 0 0;clear:both;margin:1.25em 0 1.1875em}
+em,i{font-style:italic;line-height:inherit}
+strong,b{font-weight:bold;line-height:inherit}
+small{font-size:60%;line-height:inherit}
+code{font-family:"Droid Sans Mono","DejaVu Sans Mono",monospace;font-weight:400;color:rgba(0,0,0,.9)}
+ul,ol,dl{line-height:1.6;margin-bottom:1.25em;list-style-position:outside;font-family:inherit}
+ul,ol{margin-left:1.5em}
+ul li ul,ul li ol{margin-left:1.25em;margin-bottom:0}
+ul.circle{list-style-type:circle}
+ul.disc{list-style-type:disc}
+ul.square{list-style-type:square}
+ul.circle ul:not([class]),ul.disc ul:not([class]),ul.square ul:not([class]){list-style:inherit}
+ol li ul,ol li ol{margin-left:1.25em;margin-bottom:0}
+dl dt{margin-bottom:.3125em;font-weight:bold}
+dl dd{margin-bottom:1.25em}
+blockquote{margin:0 0 1.25em;padding:.5625em 1.25em 0 1.1875em;border-left:1px solid #ddd}
+blockquote,blockquote p{line-height:1.6;color:rgba(0,0,0,.85)}
+@media screen and (min-width:768px){h1,h2,h3,#toctitle,.sidebarblock>.content>.title,h4,h5,h6{line-height:1.2}
+h1{font-size:2.75em}
+h2{font-size:2.3125em}
+h3,#toctitle,.sidebarblock>.content>.title{font-size:1.6875em}
+h4{font-size:1.4375em}}
+table{background:#fff;margin-bottom:1.25em;border:1px solid #dedede;word-wrap:normal}
+table thead,table tfoot{background:#f7f8f7}
+table thead tr th,table thead tr td,table tfoot tr th,table tfoot tr td{padding:.5em .625em .625em;font-size:inherit;color:rgba(0,0,0,.8);text-align:left}
+table tr th,table tr td{padding:.5625em .625em;font-size:inherit;color:rgba(0,0,0,.8)}
+table tr.even,table tr.alt{background:#f8f8f7}
+table thead tr th,table tfoot tr th,table tbody tr td,table tr td,table tfoot tr td{line-height:1.6}
+h1,h2,h3,#toctitle,.sidebarblock>.content>.title,h4,h5,h6{line-height:1.2;word-spacing:-.05em}
+h1 strong,h2 strong,h3 strong,#toctitle strong,.sidebarblock>.content>.title strong,h4 strong,h5 strong,h6 strong{font-weight:400}
+.center{margin-left:auto;margin-right:auto}
+.stretch{width:100%}
+.clearfix::before,.clearfix::after,.float-group::before,.float-group::after{content:" ";display:table}
+.clearfix::after,.float-group::after{clear:both}
+:not(pre).nobreak{word-wrap:normal}
+:not(pre).nowrap{white-space:nowrap}
+:not(pre).pre-wrap{white-space:pre-wrap}
+:not(pre):not([class^=L])>code{font-size:.9375em;font-style:normal!important;letter-spacing:0;padding:.1em .5ex;word-spacing:-.15em;background:#f7f7f8;border-radius:4px;line-height:1.45;text-rendering:optimizeSpeed}
+pre{color:rgba(0,0,0,.9);font-family:"Droid Sans Mono","DejaVu Sans Mono",monospace;line-height:1.45;text-rendering:optimizeSpeed}
+pre code,pre pre{color:inherit;font-size:inherit;line-height:inherit}
+pre>code{display:block}
+pre.nowrap,pre.nowrap pre{white-space:pre;word-wrap:normal}
+em em{font-style:normal}
+strong strong{font-weight:400}
+.keyseq{color:rgba(51,51,51,.8)}
+kbd{font-family:"Droid Sans Mono","DejaVu Sans Mono",monospace;display:inline-block;color:rgba(0,0,0,.8);font-size:.65em;line-height:1.45;background:#f7f7f7;border:1px solid #ccc;border-radius:3px;box-shadow:0 1px 0 rgba(0,0,0,.2),inset 0 0 0 .1em #fff;margin:0 .15em;padding:.2em .5em;vertical-align:middle;position:relative;top:-.1em;white-space:nowrap}
+.keyseq kbd:first-child{margin-left:0}
+.keyseq kbd:last-child{margin-right:0}
+.menuseq,.menuref{color:#000}
+.menuseq b:not(.caret),.menuref{font-weight:inherit}
+.menuseq{word-spacing:-.02em}
+.menuseq b.caret{font-size:1.25em;line-height:.8}
+.menuseq i.caret{font-weight:bold;text-align:center;width:.45em}
+b.button::before,b.button::after{position:relative;top:-1px;font-weight:400}
+b.button::before{content:"[";padding:0 3px 0 2px}
+b.button::after{content:"]";padding:0 2px 0 3px}
+p a>code:hover{color:rgba(0,0,0,.9)}
+#header,#content,#footnotes,#footer{width:100%;margin:0 auto;max-width:62.5em;*zoom:1;position:relative;padding-left:.9375em;padding-right:.9375em}
+#header::before,#header::after,#content::before,#content::after,#footnotes::before,#footnotes::after,#footer::before,#footer::after{content:" ";display:table}
+#header::after,#content::after,#footnotes::after,#footer::after{clear:both}
+#content{margin-top:1.25em}
+#content::before{content:none}
+#header>h1:first-child{color:rgba(0,0,0,.85);margin-top:2.25rem;margin-bottom:0}
+#header>h1:first-child+#toc{margin-top:8px;border-top:1px solid #dddddf}
+#header>h1:only-child,body.toc2 #header>h1:nth-last-child(2){border-bottom:1px solid #dddddf;padding-bottom:8px}
+#header .details{border-bottom:1px solid #dddddf;line-height:1.45;padding-top:.25em;padding-bottom:.25em;padding-left:.25em;color:rgba(0,0,0,.6);display:flex;flex-flow:row wrap}
+#header .details span:first-child{margin-left:-.125em}
+#header .details span.email a{color:rgba(0,0,0,.85)}
+#header .details br{display:none}
+#header .details br+span::before{content:"\00a0\2013\00a0"}
+#header .details br+span.author::before{content:"\00a0\22c5\00a0";color:rgba(0,0,0,.85)}
+#header .details br+span#revremark::before{content:"\00a0|\00a0"}
+#header #revnumber{text-transform:capitalize}
+#header #revnumber::after{content:"\00a0"}
+#content>h1:first-child:not([class]){color:rgba(0,0,0,.85);border-bottom:1px solid #dddddf;padding-bottom:8px;margin-top:0;padding-top:1rem;margin-bottom:1.25rem}
+#toc{border-bottom:1px solid #e7e7e9;padding-bottom:.5em}
+#toc>ul{margin-left:.125em}
+#toc ul.sectlevel0>li>a{font-style:italic}
+#toc ul.sectlevel0 ul.sectlevel1{margin:.5em 0}
+#toc ul{font-family:"Open Sans","DejaVu Sans",sans-serif;list-style-type:none}
+#toc li{line-height:1.3334;margin-top:.3334em}
+#toc a{text-decoration:none}
+#toc a:active{text-decoration:underline}
+#toctitle{color:#7a2518;font-size:1.2em}
+@media screen and (min-width:768px){#toctitle{font-size:1.375em}
+body.toc2{padding-left:15em;padding-right:0}
+#toc.toc2{margin-top:0!important;background:#f8f8f7;position:fixed;width:15em;left:0;top:0;border-right:1px solid #e7e7e9;border-top-width:0!important;border-bottom-width:0!important;z-index:1000;padding:1.25em 1em;height:100%;overflow:auto}
+#toc.toc2 #toctitle{margin-top:0;margin-bottom:.8rem;font-size:1.2em}
+#toc.toc2>ul{font-size:.9em;margin-bottom:0}
+#toc.toc2 ul ul{margin-left:0;padding-left:1em}
+#toc.toc2 ul.sectlevel0 ul.sectlevel1{padding-left:0;margin-top:.5em;margin-bottom:.5em}
+body.toc2.toc-right{padding-left:0;padding-right:15em}
+body.toc2.toc-right #toc.toc2{border-right-width:0;border-left:1px solid #e7e7e9;left:auto;right:0}}
+@media screen and (min-width:1280px){body.toc2{padding-left:20em;padding-right:0}
+#toc.toc2{width:20em}
+#toc.toc2 #toctitle{font-size:1.375em}
+#toc.toc2>ul{font-size:.95em}
+#toc.toc2 ul ul{padding-left:1.25em}
+body.toc2.toc-right{padding-left:0;padding-right:20em}}
+#content #toc{border:1px solid #e0e0dc;margin-bottom:1.25em;padding:1.25em;background:#f8f8f7;border-radius:4px}
+#content #toc>:first-child{margin-top:0}
+#content #toc>:last-child{margin-bottom:0}
+#footer{max-width:none;background:rgba(0,0,0,.8);padding:1.25em}
+#footer-text{color:hsla(0,0%,100%,.8);line-height:1.44}
+#content{margin-bottom:.625em}
+.sect1{padding-bottom:.625em}
+@media screen and (min-width:768px){#content{margin-bottom:1.25em}
+.sect1{padding-bottom:1.25em}}
+.sect1:last-child{padding-bottom:0}
+.sect1+.sect1{border-top:1px solid #e7e7e9}
+#content h1>a.anchor,h2>a.anchor,h3>a.anchor,#toctitle>a.anchor,.sidebarblock>.content>.title>a.anchor,h4>a.anchor,h5>a.anchor,h6>a.anchor{position:absolute;z-index:1001;width:1.5ex;margin-left:-1.5ex;display:block;text-decoration:none!important;visibility:hidden;text-align:center;font-weight:400}
+#content h1>a.anchor::before,h2>a.anchor::before,h3>a.anchor::before,#toctitle>a.anchor::before,.sidebarblock>.content>.title>a.anchor::before,h4>a.anchor::before,h5>a.anchor::before,h6>a.anchor::before{content:"\00A7";font-size:.85em;display:block;padding-top:.1em}
+#content h1:hover>a.anchor,#content h1>a.anchor:hover,h2:hover>a.anchor,h2>a.anchor:hover,h3:hover>a.anchor,#toctitle:hover>a.anchor,.sidebarblock>.content>.title:hover>a.anchor,h3>a.anchor:hover,#toctitle>a.anchor:hover,.sidebarblock>.content>.title>a.anchor:hover,h4:hover>a.anchor,h4>a.anchor:hover,h5:hover>a.anchor,h5>a.anchor:hover,h6:hover>a.anchor,h6>a.anchor:hover{visibility:visible}
+#content h1>a.link,h2>a.link,h3>a.link,#toctitle>a.link,.sidebarblock>.content>.title>a.link,h4>a.link,h5>a.link,h6>a.link{color:#ba3925;text-decoration:none}
+#content h1>a.link:hover,h2>a.link:hover,h3>a.link:hover,#toctitle>a.link:hover,.sidebarblock>.content>.title>a.link:hover,h4>a.link:hover,h5>a.link:hover,h6>a.link:hover{color:#a53221}
+details,.audioblock,.imageblock,.literalblock,.listingblock,.stemblock,.videoblock{margin-bottom:1.25em}
+details{margin-left:1.25rem}
+details>summary{cursor:pointer;display:block;position:relative;line-height:1.6;margin-bottom:.625rem;outline:none;-webkit-tap-highlight-color:transparent}
+details>summary::-webkit-details-marker{display:none}
+details>summary::before{content:"";border:solid transparent;border-left:solid;border-width:.3em 0 .3em .5em;position:absolute;top:.5em;left:-1.25rem;transform:translateX(15%)}
+details[open]>summary::before{border:solid transparent;border-top:solid;border-width:.5em .3em 0;transform:translateY(15%)}
+details>summary::after{content:"";width:1.25rem;height:1em;position:absolute;top:.3em;left:-1.25rem}
+.admonitionblock td.content>.title,.audioblock>.title,.exampleblock>.title,.imageblock>.title,.listingblock>.title,.literalblock>.title,.stemblock>.title,.openblock>.title,.paragraph>.title,.quoteblock>.title,table.tableblock>.title,.verseblock>.title,.videoblock>.title,.dlist>.title,.olist>.title,.ulist>.title,.qlist>.title,.hdlist>.title{text-rendering:optimizeLegibility;text-align:left;font-family:"Noto Serif","DejaVu Serif",serif;font-size:1rem;font-style:italic}
+table.tableblock.fit-content>caption.title{white-space:nowrap;width:0}
+.paragraph.lead>p,#preamble>.sectionbody>[class=paragraph]:first-of-type p{font-size:1.21875em;line-height:1.6;color:rgba(0,0,0,.85)}
+.admonitionblock>table{border-collapse:separate;border:0;background:none;width:100%}
+.admonitionblock>table td.icon{text-align:center;width:80px}
+.admonitionblock>table td.icon img{max-width:none}
+.admonitionblock>table td.icon .title{font-weight:bold;font-family:"Open Sans","DejaVu Sans",sans-serif;text-transform:uppercase}
+.admonitionblock>table td.content{padding-left:1.125em;padding-right:1.25em;border-left:1px solid #dddddf;color:rgba(0,0,0,.6);word-wrap:anywhere}
+.admonitionblock>table td.content>:last-child>:last-child{margin-bottom:0}
+.exampleblock>.content{border:1px solid #e6e6e6;margin-bottom:1.25em;padding:1.25em;background:#fff;border-radius:4px}
+.exampleblock>.content>:first-child{margin-top:0}
+.exampleblock>.content>:last-child{margin-bottom:0}
+.sidebarblock{border:1px solid #dbdbd6;margin-bottom:1.25em;padding:1.25em;background:#f3f3f2;border-radius:4px}
+.sidebarblock>:first-child{margin-top:0}
+.sidebarblock>:last-child{margin-bottom:0}
+.sidebarblock>.content>.title{color:#7a2518;margin-top:0;text-align:center}
+.exampleblock>.content>:last-child>:last-child,.exampleblock>.content .olist>ol>li:last-child>:last-child,.exampleblock>.content .ulist>ul>li:last-child>:last-child,.exampleblock>.content .qlist>ol>li:last-child>:last-child,.sidebarblock>.content>:last-child>:last-child,.sidebarblock>.content .olist>ol>li:last-child>:last-child,.sidebarblock>.content .ulist>ul>li:last-child>:last-child,.sidebarblock>.content .qlist>ol>li:last-child>:last-child{margin-bottom:0}
+.literalblock pre,.listingblock>.content>pre{border-radius:4px;overflow-x:auto;padding:1em;font-size:.8125em}
+@media screen and (min-width:768px){.literalblock pre,.listingblock>.content>pre{font-size:.90625em}}
+@media screen and (min-width:1280px){.literalblock pre,.listingblock>.content>pre{font-size:1em}}
+.literalblock pre,.listingblock>.content>pre:not(.highlight),.listingblock>.content>pre[class=highlight],.listingblock>.content>pre[class^="highlight "]{background:#f7f7f8}
+.literalblock.output pre{color:#f7f7f8;background:rgba(0,0,0,.9)}
+.listingblock>.content{position:relative}
+.listingblock code[data-lang]::before{display:none;content:attr(data-lang);position:absolute;font-size:.75em;top:.425rem;right:.5rem;line-height:1;text-transform:uppercase;color:inherit;opacity:.5}
+.listingblock:hover code[data-lang]::before{display:block}
+.listingblock.terminal pre .command::before{content:attr(data-prompt);padding-right:.5em;color:inherit;opacity:.5}
+.listingblock.terminal pre .command:not([data-prompt])::before{content:"$"}
+.listingblock pre.highlightjs{padding:0}
+.listingblock pre.highlightjs>code{padding:1em;border-radius:4px}
+.listingblock pre.prettyprint{border-width:0}
+.prettyprint{background:#f7f7f8}
+pre.prettyprint .linenums{line-height:1.45;margin-left:2em}
+pre.prettyprint li{background:none;list-style-type:inherit;padding-left:0}
+pre.prettyprint li code[data-lang]::before{opacity:1}
+pre.prettyprint li:not(:first-child) code[data-lang]::before{display:none}
+table.linenotable{border-collapse:separate;border:0;margin-bottom:0;background:none}
+table.linenotable td[class]{color:inherit;vertical-align:top;padding:0;line-height:inherit;white-space:normal}
+table.linenotable td.code{padding-left:.75em}
+table.linenotable td.linenos,pre.pygments .linenos{border-right:1px solid;opacity:.35;padding-right:.5em;-webkit-user-select:none;-moz-user-select:none;-ms-user-select:none;user-select:none}
+pre.pygments span.linenos{display:inline-block;margin-right:.75em}
+.quoteblock{margin:0 1em 1.25em 1.5em;display:table}
+.quoteblock:not(.excerpt)>.title{margin-left:-1.5em;margin-bottom:.75em}
+.quoteblock blockquote,.quoteblock p{color:rgba(0,0,0,.85);font-size:1.15rem;line-height:1.75;word-spacing:.1em;letter-spacing:0;font-style:italic;text-align:justify}
+.quoteblock blockquote{margin:0;padding:0;border:0}
+.quoteblock blockquote::before{content:"\201c";float:left;font-size:2.75em;font-weight:bold;line-height:.6em;margin-left:-.6em;color:#7a2518;text-shadow:0 1px 2px rgba(0,0,0,.1)}
+.quoteblock blockquote>.paragraph:last-child p{margin-bottom:0}
+.quoteblock .attribution{margin-top:.75em;margin-right:.5ex;text-align:right}
+.verseblock{margin:0 1em 1.25em}
+.verseblock pre{font-family:"Open Sans","DejaVu Sans",sans-serif;font-size:1.15rem;color:rgba(0,0,0,.85);font-weight:300;text-rendering:optimizeLegibility}
+.verseblock pre strong{font-weight:400}
+.verseblock .attribution{margin-top:1.25rem;margin-left:.5ex}
+.quoteblock .attribution,.verseblock .attribution{font-size:.9375em;line-height:1.45;font-style:italic}
+.quoteblock .attribution br,.verseblock .attribution br{display:none}
+.quoteblock .attribution cite,.verseblock .attribution cite{display:block;letter-spacing:-.025em;color:rgba(0,0,0,.6)}
+.quoteblock.abstract blockquote::before,.quoteblock.excerpt blockquote::before,.quoteblock .quoteblock blockquote::before{display:none}
+.quoteblock.abstract blockquote,.quoteblock.abstract p,.quoteblock.excerpt blockquote,.quoteblock.excerpt p,.quoteblock .quoteblock blockquote,.quoteblock .quoteblock p{line-height:1.6;word-spacing:0}
+.quoteblock.abstract{margin:0 1em 1.25em;display:block}
+.quoteblock.abstract>.title{margin:0 0 .375em;font-size:1.15em;text-align:center}
+.quoteblock.excerpt>blockquote,.quoteblock .quoteblock{padding:0 0 .25em 1em;border-left:.25em solid #dddddf}
+.quoteblock.excerpt,.quoteblock .quoteblock{margin-left:0}
+.quoteblock.excerpt blockquote,.quoteblock.excerpt p,.quoteblock .quoteblock blockquote,.quoteblock .quoteblock p{color:inherit;font-size:1.0625rem}
+.quoteblock.excerpt .attribution,.quoteblock .quoteblock .attribution{color:inherit;font-size:.85rem;text-align:left;margin-right:0}
+p.tableblock:last-child{margin-bottom:0}
+td.tableblock>.content{margin-bottom:1.25em;word-wrap:anywhere}
+td.tableblock>.content>:last-child{margin-bottom:-1.25em}
+table.tableblock,th.tableblock,td.tableblock{border:0 solid #dedede}
+table.grid-all>*>tr>*{border-width:1px}
+table.grid-cols>*>tr>*{border-width:0 1px}
+table.grid-rows>*>tr>*{border-width:1px 0}
+table.frame-all{border-width:1px}
+table.frame-ends{border-width:1px 0}
+table.frame-sides{border-width:0 1px}
+table.frame-none>colgroup+*>:first-child>*,table.frame-sides>colgroup+*>:first-child>*{border-top-width:0}
+table.frame-none>:last-child>:last-child>*,table.frame-sides>:last-child>:last-child>*{border-bottom-width:0}
+table.frame-none>*>tr>:first-child,table.frame-ends>*>tr>:first-child{border-left-width:0}
+table.frame-none>*>tr>:last-child,table.frame-ends>*>tr>:last-child{border-right-width:0}
+table.stripes-all>*>tr,table.stripes-odd>*>tr:nth-of-type(odd),table.stripes-even>*>tr:nth-of-type(even),table.stripes-hover>*>tr:hover{background:#f8f8f7}
+th.halign-left,td.halign-left{text-align:left}
+th.halign-right,td.halign-right{text-align:right}
+th.halign-center,td.halign-center{text-align:center}
+th.valign-top,td.valign-top{vertical-align:top}
+th.valign-bottom,td.valign-bottom{vertical-align:bottom}
+th.valign-middle,td.valign-middle{vertical-align:middle}
+table thead th,table tfoot th{font-weight:bold}
+tbody tr th{background:#f7f8f7}
+tbody tr th,tbody tr th p,tfoot tr th,tfoot tr th p{color:rgba(0,0,0,.8);font-weight:bold}
+p.tableblock>code:only-child{background:none;padding:0}
+p.tableblock{font-size:1em}
+ol{margin-left:1.75em}
+ul li ol{margin-left:1.5em}
+dl dd{margin-left:1.125em}
+dl dd:last-child,dl dd:last-child>:last-child{margin-bottom:0}
+li p,ul dd,ol dd,.olist .olist,.ulist .ulist,.ulist .olist,.olist .ulist{margin-bottom:.625em}
+ul.checklist,ul.none,ol.none,ul.no-bullet,ol.no-bullet,ol.unnumbered,ul.unstyled,ol.unstyled{list-style-type:none}
+ul.no-bullet,ol.no-bullet,ol.unnumbered{margin-left:.625em}
+ul.unstyled,ol.unstyled{margin-left:0}
+li>p:empty:only-child::before{content:"";display:inline-block}
+ul.checklist>li>p:first-child{margin-left:-1em}
+ul.checklist>li>p:first-child>.fa-square-o:first-child,ul.checklist>li>p:first-child>.fa-check-square-o:first-child{width:1.25em;font-size:.8em;position:relative;bottom:.125em}
+ul.checklist>li>p:first-child>input[type=checkbox]:first-child{margin-right:.25em}
+ul.inline{display:flex;flex-flow:row wrap;list-style:none;margin:0 0 .625em -1.25em}
+ul.inline>li{margin-left:1.25em}
+.unstyled dl dt{font-weight:400;font-style:normal}
+ol.arabic{list-style-type:decimal}
+ol.decimal{list-style-type:decimal-leading-zero}
+ol.loweralpha{list-style-type:lower-alpha}
+ol.upperalpha{list-style-type:upper-alpha}
+ol.lowerroman{list-style-type:lower-roman}
+ol.upperroman{list-style-type:upper-roman}
+ol.lowergreek{list-style-type:lower-greek}
+.hdlist>table,.colist>table{border:0;background:none}
+.hdlist>table>tbody>tr,.colist>table>tbody>tr{background:none}
+td.hdlist1,td.hdlist2{vertical-align:top;padding:0 .625em}
+td.hdlist1{font-weight:bold;padding-bottom:1.25em}
+td.hdlist2{word-wrap:anywhere}
+.literalblock+.colist,.listingblock+.colist{margin-top:-.5em}
+.colist td:not([class]):first-child{padding:.4em .75em 0;line-height:1;vertical-align:top}
+.colist td:not([class]):first-child img{max-width:none}
+.colist td:not([class]):last-child{padding:.25em 0}
+.thumb,.th{line-height:0;display:inline-block;border:4px solid #fff;box-shadow:0 0 0 1px #ddd}
+.imageblock.left{margin:.25em .625em 1.25em 0}
+.imageblock.right{margin:.25em 0 1.25em .625em}
+.imageblock>.title{margin-bottom:0}
+.imageblock.thumb,.imageblock.th{border-width:6px}
+.imageblock.thumb>.title,.imageblock.th>.title{padding:0 .125em}
+.image.left,.image.right{margin-top:.25em;margin-bottom:.25em;display:inline-block;line-height:0}
+.image.left{margin-right:.625em}
+.image.right{margin-left:.625em}
+a.image{text-decoration:none;display:inline-block}
+a.image object{pointer-events:none}
+sup.footnote,sup.footnoteref{font-size:.875em;position:static;vertical-align:super}
+sup.footnote a,sup.footnoteref a{text-decoration:none}
+sup.footnote a:active,sup.footnoteref a:active{text-decoration:underline}
+#footnotes{padding-top:.75em;padding-bottom:.75em;margin-bottom:.625em}
+#footnotes hr{width:20%;min-width:6.25em;margin:-.25em 0 .75em;border-width:1px 0 0}
+#footnotes .footnote{padding:0 .375em 0 .225em;line-height:1.3334;font-size:.875em;margin-left:1.2em;margin-bottom:.2em}
+#footnotes .footnote a:first-of-type{font-weight:bold;text-decoration:none;margin-left:-1.05em}
+#footnotes .footnote:last-of-type{margin-bottom:0}
+#content #footnotes{margin-top:-.625em;margin-bottom:0;padding:.75em 0}
+div.unbreakable{page-break-inside:avoid}
+.big{font-size:larger}
+.small{font-size:smaller}
+.underline{text-decoration:underline}
+.overline{text-decoration:overline}
+.line-through{text-decoration:line-through}
+.aqua{color:#00bfbf}
+.aqua-background{background:#00fafa}
+.black{color:#000}
+.black-background{background:#000}
+.blue{color:#0000bf}
+.blue-background{background:#0000fa}
+.fuchsia{color:#bf00bf}
+.fuchsia-background{background:#fa00fa}
+.gray{color:#606060}
+.gray-background{background:#7d7d7d}
+.green{color:#006000}
+.green-background{background:#007d00}
+.lime{color:#00bf00}
+.lime-background{background:#00fa00}
+.maroon{color:#600000}
+.maroon-background{background:#7d0000}
+.navy{color:#000060}
+.navy-background{background:#00007d}
+.olive{color:#606000}
+.olive-background{background:#7d7d00}
+.purple{color:#600060}
+.purple-background{background:#7d007d}
+.red{color:#bf0000}
+.red-background{background:#fa0000}
+.silver{color:#909090}
+.silver-background{background:#bcbcbc}
+.teal{color:#006060}
+.teal-background{background:#007d7d}
+.white{color:#bfbfbf}
+.white-background{background:#fafafa}
+.yellow{color:#bfbf00}
+.yellow-background{background:#fafa00}
+span.icon>.fa{cursor:default}
+a span.icon>.fa{cursor:inherit}
+.admonitionblock td.icon [class^="fa icon-"]{font-size:2.5em;text-shadow:1px 1px 2px rgba(0,0,0,.5);cursor:default}
+.admonitionblock td.icon .icon-note::before{content:"\f05a";color:#19407c}
+.admonitionblock td.icon .icon-tip::before{content:"\f0eb";text-shadow:1px 1px 2px rgba(155,155,0,.8);color:#111}
+.admonitionblock td.icon .icon-warning::before{content:"\f071";color:#bf6900}
+.admonitionblock td.icon .icon-caution::before{content:"\f06d";color:#bf3400}
+.admonitionblock td.icon .icon-important::before{content:"\f06a";color:#bf0000}
+.conum[data-value]{display:inline-block;color:#fff!important;background:rgba(0,0,0,.8);border-radius:50%;text-align:center;font-size:.75em;width:1.67em;height:1.67em;line-height:1.67em;font-family:"Open Sans","DejaVu Sans",sans-serif;font-style:normal;font-weight:bold}
+.conum[data-value] *{color:#fff!important}
+.conum[data-value]+b{display:none}
+.conum[data-value]::after{content:attr(data-value)}
+pre .conum[data-value]{position:relative;top:-.125em}
+b.conum *{color:inherit!important}
+.conum:not([data-value]):empty{display:none}
+dt,th.tableblock,td.content,div.footnote{text-rendering:optimizeLegibility}
+h1,h2,p,td.content,span.alt,summary{letter-spacing:-.01em}
+p strong,td.content strong,div.footnote strong{letter-spacing:-.005em}
+p,blockquote,dt,td.content,span.alt,summary{font-size:1.0625rem}
+p{margin-bottom:1.25rem}
+.sidebarblock p,.sidebarblock dt,.sidebarblock td.content,p.tableblock{font-size:1em}
+.exampleblock>.content{background:#fffef7;border-color:#e0e0dc;box-shadow:0 1px 4px #e0e0dc}
+.print-only{display:none!important}
+@page{margin:1.25cm .75cm}
+@media print{*{box-shadow:none!important;text-shadow:none!important}
+html{font-size:80%}
+a{color:inherit!important;text-decoration:underline!important}
+a.bare,a[href^="#"],a[href^="mailto:"]{text-decoration:none!important}
+a[href^="http:"]:not(.bare)::after,a[href^="https:"]:not(.bare)::after{content:"(" attr(href) ")";display:inline-block;font-size:.875em;padding-left:.25em}
+abbr[title]{border-bottom:1px dotted}
+abbr[title]::after{content:" (" attr(title) ")"}
+pre,blockquote,tr,img,object,svg{page-break-inside:avoid}
+thead{display:table-header-group}
+svg{max-width:100%}
+p,blockquote,dt,td.content{font-size:1em;orphans:3;widows:3}
+h2,h3,#toctitle,.sidebarblock>.content>.title{page-break-after:avoid}
+#header,#content,#footnotes,#footer{max-width:none}
+#toc,.sidebarblock,.exampleblock>.content{background:none!important}
+#toc{border-bottom:1px solid #dddddf!important;padding-bottom:0!important}
+body.book #header{text-align:center}
+body.book #header>h1:first-child{border:0!important;margin:2.5em 0 1em}
+body.book #header .details{border:0!important;display:block;padding:0!important}
+body.book #header .details span:first-child{margin-left:0!important}
+body.book #header .details br{display:block}
+body.book #header .details br+span::before{content:none!important}
+body.book #toc{border:0!important;text-align:left!important;padding:0!important;margin:0!important}
+body.book #toc,body.book #preamble,body.book h1.sect0,body.book .sect1>h2{page-break-before:always}
+.listingblock code[data-lang]::before{display:block}
+#footer{padding:0 .9375em}
+.hide-on-print{display:none!important}
+.print-only{display:block!important}
+.hide-for-print{display:none!important}
+.show-for-print{display:inherit!important}}
+@media amzn-kf8,print{#header>h1:first-child{margin-top:1.25rem}
+.sect1{padding:0!important}
+.sect1+.sect1{border:0}
+#footer{background:none}
+#footer-text{color:rgba(0,0,0,.6);font-size:.9em}}
+@media amzn-kf8{#header,#content,#footnotes,#footer{padding:0}}
+</style>
+</head>
+<body class="article">
+<div id="header">
+</div>
+<div id="content">
+<div class="sect1">
+<h2 id="_qqq_tables">QQQ Tables</h2>
+<div class="sectionbody">
+<div class="paragraph">
+<p>The core type of object in a QQQ Instance is the Table.
+In the most common use-case, a QQQ Table may be the in-app representation of a Database table.
+That is, it is a collection of records (or rows) of data, each of which has a set of fields (or columns).</p>
+</div>
+<div class="paragraph">
+<p>QQQ also allows other types of data sources (<a href="Backends{relfilesuffix}">QQQ Backends</a>) to be used as tables, such as File systems, API&#8217;s, Java enums or objects, etc.
+All of these backend types present the same interfaces (both user-interfaces, and application programming interfaces), regardless of their backend type.</p>
+</div>
+<div class="sect2">
+<h3 id="_qtablemetadata">QTableMetaData</h3>
+<div class="paragraph">
+<p>Tables are defined in a QQQ Instance in a <code><strong>QTableMetaData</strong></code> object.
+All tables must reference a <a href="Backends{relfilesuffix}">QQQ Backend</a>, a list of fields that define the shape of records in the table, and additional data to describe how to work with the table within its backend.</p>
+</div>
+<div class="paragraph">
+<p><strong>QTableMetaData Properties:</strong></p>
+</div>
+<div class="ulist">
+<ul>
+<li>
+<p><code>name</code> - <strong>String, Required</strong> - Unique name for the table within the QQQ Instance.</p>
+</li>
+<li>
+<p><code>label</code> - <strong>String</strong> - User-facing label for the table, presented in User Interfaces.
+Inferred from <code>name</code> if not set.</p>
+</li>
+<li>
+<p><code>backendName</code> - <strong>String, Required</strong> - Name of a <a href="Backends{relfilesuffix}">QQQ Backend</a> in which this table&#8217;s data is managed.</p>
+</li>
+<li>
+<p><code>fields</code> - <strong>Map of String → <a href="Fields{relfilesuffix}">QQQ Field</a>, Required</strong> - The columns of data that make up all records in this table.</p>
+</li>
+<li>
+<p><code>primaryKeyField</code> - <strong>String, Conditional</strong> - Name of a <a href="Fields{relfilesuffix}">QQQ Field</a> that serves as the primary key (e.g., unique identifier) for records in this table.</p>
+</li>
+<li>
+<p><code>uniqueKeys</code> - <strong>List of UniqueKey</strong> - Definition of additional unique constraints (from an RDBMS point of view) from the table.
+e.g., sets of columns which must have unique values for each record in the table.</p>
+</li>
+<li>
+<p><code>backendDetails</code> - <strong>QTableBackendDetails or subclass</strong> - Additional data to configure the table within its <a href="Backends{relfilesuffix}">QQQ Backend</a>.</p>
+</li>
+<li>
+<p><code>automationDetails</code> - <strong>QTableAutomationDetails</strong> - Configuration of automated jobs that run against records in the table, e.g., upon insert or update.</p>
+</li>
+<li>
+<p><code>customizers</code> - <strong>Map of String → QCodeReference</strong> - References to custom code that are injected into standard table actions, that allow applications to customize certain parts of how the table works.</p>
+</li>
+<li>
+<p><code>parentAppName</code> - <strong>String</strong> - Name of a <a href="Apps{relfilesuffix}">QQQ App</a> that this table exists within.</p>
+</li>
+<li>
+<p><code>icon</code> - <strong>QIcon</strong> - Icon associated with this table in certain user interfaces.</p>
+</li>
+<li>
+<p><code>recordLabelFormat</code> - <strong>String</strong> - Java Format String, used with <code>recordLabelFields</code> to produce a label shown for records from the table.</p>
+</li>
+<li>
+<p><code>recordLabelFields</code> - <strong>List of String, Conditional</strong> - Used with <code>recordLabelFormat</code> to provide values for any format specifiers in the format string.
+These strings must be field names within the table.</p>
+<div class="ulist">
+<ul>
+<li>
+<p>Example of using <code>recordLabelFormat</code> and <code>recordLabelFields</code>:</p>
+</li>
+</ul>
+</div>
+</li>
+</ul>
+</div>
+<div class="listingblock">
+<div class="content">
+<pre class="CodeRay highlight"><code data-lang="java"><span style="color:#777">// given these fields in the table:</span>
+<span style="color:#080;font-weight:bold">new</span> QFieldMetaData(<span style="background-color:hsla(0,100%,50%,0.05)"><span style="color:#710">&quot;</span><span style="color:#D20">name</span><span style="color:#710">&quot;</span></span>, QFieldType.STRING)
+<span style="color:#080;font-weight:bold">new</span> QFieldMetaData(<span style="background-color:hsla(0,100%,50%,0.05)"><span style="color:#710">&quot;</span><span style="color:#D20">birthDate</span><span style="color:#710">&quot;</span></span>, QFieldType.DATE)
+
+<span style="color:#777">// We can produce a record label such as &quot;Darin Kelkhoff (1980-05-31)&quot; via:</span>
+.withRecordLabelFormat(<span style="background-color:hsla(0,100%,50%,0.05)"><span style="color:#710">&quot;</span><span style="color:#D20">%s (%s)</span><span style="color:#710">&quot;</span></span>)
+.withRecordLabelFields(<span style="color:#0a8;font-weight:bold">List</span>.of(<span style="background-color:hsla(0,100%,50%,0.05)"><span style="color:#710">&quot;</span><span style="color:#D20">name</span><span style="color:#710">&quot;</span></span>, <span style="background-color:hsla(0,100%,50%,0.05)"><span style="color:#710">&quot;</span><span style="color:#D20">birthDate</span><span style="color:#710">&quot;</span></span>))</code></pre>
+</div>
+</div>
+<div class="ulist">
+<ul>
+<li>
+<p><code>sections</code> - <strong>List of QFieldSection</strong> - Mechanism to organize fields within user interfaces, into logical sections.
+If any sections are present in the table meta data, then all fields in the table must be listed in exactly 1 section.
+If no sections are defined, then instance enrichment will define default sections.</p>
+</li>
+<li>
+<p><code>associatedScripts</code> - <strong>List of AssociatedScript</strong> - Definition of user-defined scripts that can be associated with records within the table.</p>
+</li>
+<li>
+<p><code>enabledCapabilities</code> and <code>disabledCapabilities</code> - <strong>Set of Capability enum values</strong> - Overrides from the backend level, for capabilities that this table does or does not possess.</p>
+</li>
+</ul>
+</div>
+</div>
+</div>
+</div>
+</div>
+<div id="footer">
+<div id="footer-text">
+Last updated 2022-11-21 09:02:56 -0600
+</div>
+</div>
+</body>
+</html>

docs/variables.adoc

@@ -0,0 +1,13 @@
+ifdef::env-name[:relfilesuffix: .adoc]
+:link-backend: link:Backends{relfilesuffix}[QQQ Backend]
+:link-backends: link:Backends{relfilesuffix}[QQQ Backends]
+:link-table: link:Tables{relfilesuffix}[QQQ Table]
+:link-tables: link:Tables{relfilesuffix}[QQQ Tables]
+:link-join: link:Joins{relfilesuffix}[QQQ Join]
+:link-joins: link:Joins{relfilesuffix}[QQQ Joins]
+:link-field: link:Fields{relfilesuffix}[QQQ Field]
+:link-fields: link:Fields{relfilesuffix}[QQQ Fields]
+:link-process: link:Processes{relfilesuffix}[QQQ Process]
+:link-processes: link:Processes{relfilesuffix}[QQQ Processes]
+:link-app: link:Apps{relfilesuffix}[QQQ App]
+:link-apps: link:Apps{relfilesuffix}[QQQ Apps]

pom.xml

@@ -44,7 +44,7 @@
    </modules>
 
    <properties>
-      <revision>0.13.0</revision>
+      <revision>0.20.0-SNAPSHOT</revision>
 
       <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
       <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
@@ -206,6 +206,63 @@
                <skipUpdateVersion>true</skipUpdateVersion>
             </configuration>
          </plugin>
+         <plugin>
+            <artifactId>exec-maven-plugin</artifactId>
+            <groupId>org.codehaus.mojo</groupId>
+            <version>3.0.0</version>
+            <executions>
+               <execution>
+                  <id>test-coverage-summary</id>
+                  <phase>verify</phase>
+                  <goals>
+                     <goal>exec</goal>
+                  </goals>
+                  <configuration>
+                     <executable>sh</executable>
+                     <arguments>
+                        <argument>-c</argument>
+                        <argument>
+                           <![CDATA[
+if [ ! -e target/site/jacoco/index.html ]; then
+   echo "No jacoco coverage report here.";
+   exit;
+fi
+
+echo
+echo "Jacoco coverage summary report:"
+echo "   See also target/site/jacoco/index.html"
+echo "   and https://www.jacoco.org/jacoco/trunk/doc/counters.html"
+echo "------------------------------------------------------------"
+
+which xpath > /dev/null 2>&1
+if [ "$?" == "0" ]; then
+   echo "Element\nInstructions Missed\nInstruction Coverage\nBranches Missed\nBranch Coverage\nComplexity Missed\nComplexity Hit\nLines Missed\nLines Hit\nMethods Missed\nMethods Hit\nClasses Missed\nClasses Hit\n" > /tmp/$$.headers
+   xpath -n -q -e '/html/body/table/tfoot/tr[1]/td/text()' target/site/jacoco/index.html > /tmp/$$.values
+   paste /tmp/$$.headers /tmp/$$.values | tail +2 | awk -v FS='\t' '{printf("%-20s %s\n",$1,$2)}'
+   rm /tmp/$$.headers /tmp/$$.values
+else
+   echo "xpath is not installed.  Jacoco coverage summary will not be produced here...";
+fi
+
+which xpath > /dev/null 2>&1
+if [ "$?" == "0" ]; then
+   echo "Untested classes, per Jacoco:"
+   echo "-----------------------------"
+   for i in target/site/jacoco/*/index.html; do
+      html2text -width 500 -nobs $i | sed '1,/^Total/d;' | grep -v Created | sed 's/ \+/ /g' | sed 's/ [[:digit:]]$//' | grep -v 0$ | cut -d' ' -f1;
+   done;
+   echo
+else
+   echo "html2text is not installed.  Untested classes from Jacoco will not be printed here...";
+fi
+
+                           ]]>
+                        </argument>
+                     </arguments>
+                  </configuration>
+               </execution>
+            </executions>
+         </plugin>
          <plugin>
             <groupId>org.jacoco</groupId>
             <artifactId>jacoco-maven-plugin</artifactId>
@@ -249,56 +306,14 @@
                </execution>
                <execution>
                   <id>post-unit-test</id>
-                  <phase>verify</phase>
+                  <!-- <phase>verify</phase> -->
+                  <phase>post-integration-test</phase>
                   <goals>
                      <goal>report</goal>
                   </goals>
                </execution>
             </executions>
          </plugin>
-         <plugin>
-            <artifactId>exec-maven-plugin</artifactId>
-            <groupId>org.codehaus.mojo</groupId>
-            <version>3.0.0</version>
-            <executions>
-               <execution>
-                  <id>test-coverage-summary</id>
-                  <phase>verify</phase>
-                  <goals>
-                     <goal>exec</goal>
-                  </goals>
-                  <configuration>
-                     <executable>sh</executable>
-                     <arguments>
-                        <argument>-c</argument>
-                        <argument>
-                           <![CDATA[
-if [ ! -e target/site/jacoco/index.html ]; then
-   echo "No jacoco coverage report here.";
-   exit;
-fi
-
-echo
-echo "Jacoco coverage summary report:"
-echo "   See also target/site/jacoco/index.html"
-echo "   and https://www.jacoco.org/jacoco/trunk/doc/counters.html"
-echo "------------------------------------------------------------"
-which xpath > /dev/null 2>&1
-if [ "$?" == "0" ]; then
-   echo "Element\nInstructions Missed\nInstruction Coverage\nBranches Missed\nBranch Coverage\nComplexity Missed\nComplexity Hit\nLines Missed\nLines Hit\nMethods Missed\nMethods Hit\nClasses Missed\nClasses Hit\n" > /tmp/$$.headers
-   xpath -n -q -e '/html/body/table/tfoot/tr[1]/td/text()' target/site/jacoco/index.html > /tmp/$$.values
-   paste /tmp/$$.headers /tmp/$$.values | tail +2 | awk -v FS='\t' '{printf("%-20s %s\n",$1,$2)}'
-   rm /tmp/$$.headers /tmp/$$.values
-else
-   echo "xpath is not installed.  Jacoco coverage summary will not be produced here..";
-fi
-                           ]]>
-                        </argument>
-                     </arguments>
-                  </configuration>
-               </execution>
-            </executions>
-         </plugin>
       </plugins>
    </build>
 

qqq-backend-core/pom.xml

@@ -56,6 +56,10 @@
          <groupId>software.amazon.awssdk</groupId>
          <artifactId>quicksight</artifactId>
       </dependency>
+      <dependency>
+         <groupId>software.amazon.awssdk</groupId>
+         <artifactId>apigateway</artifactId>
+      </dependency>
       <dependency>
          <groupId>com.amazonaws</groupId>
          <artifactId>aws-java-sdk-secretsmanager</artifactId>
@@ -80,7 +84,7 @@
       <dependency>
          <groupId>org.json</groupId>
          <artifactId>json</artifactId>
-         <version>20230227</version>
+         <version>20231013</version>
       </dependency>
       <dependency>
          <groupId>org.apache.commons</groupId>
@@ -100,8 +104,18 @@
       </dependency>
       <dependency>
          <groupId>com.auth0</groupId>
-         <artifactId>mvc-auth-commons</artifactId>
-         <version>1.9.2</version>
+         <artifactId>auth0</artifactId>
+         <version>2.1.0</version>
+      </dependency>
+      <dependency>
+         <groupId>com.auth0</groupId>
+         <artifactId>java-jwt</artifactId>
+         <version>4.4.0</version>
+      </dependency>
+      <dependency>
+         <groupId>com.auth0</groupId>
+         <artifactId>jwks-rsa</artifactId>
+         <version>0.22.0</version>
       </dependency>
       <dependency>
          <groupId>io.github.cdimascio</groupId>

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/AbstractQActionBiConsumer.java

@@ -23,7 +23,6 @@ package com.kingsrook.qqq.backend.core.actions;
 
 
 import java.util.concurrent.CompletableFuture;
-import java.util.concurrent.Executors;
 import java.util.concurrent.Future;
 import com.kingsrook.qqq.backend.core.context.CapturedContext;
 import com.kingsrook.qqq.backend.core.context.QContext;
@@ -54,7 +53,7 @@ public abstract class AbstractQActionBiConsumer<I extends AbstractActionInput, O
    {
       CapturedContext         capturedContext   = QContext.capture();
       CompletableFuture<Void> completableFuture = new CompletableFuture<>();
-      Executors.newCachedThreadPool().submit(() ->
+      ActionHelper.getExecutorService().submit(() ->
       {
          try
          {

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/AbstractQActionFunction.java

@@ -23,7 +23,6 @@ package com.kingsrook.qqq.backend.core.actions;
 
 
 import java.util.concurrent.CompletableFuture;
-import java.util.concurrent.Executors;
 import java.util.concurrent.Future;
 import com.kingsrook.qqq.backend.core.context.CapturedContext;
 import com.kingsrook.qqq.backend.core.context.QContext;
@@ -54,7 +53,7 @@ public abstract class AbstractQActionFunction<I extends AbstractActionInput, O e
    {
       CapturedContext      capturedContext   = QContext.capture();
       CompletableFuture<O> completableFuture = new CompletableFuture<>();
-      Executors.newCachedThreadPool().submit(() ->
+      ActionHelper.getExecutorService().submit(() ->
       {
          try
          {

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/ActionHelper.java

@@ -24,6 +24,10 @@ package com.kingsrook.qqq.backend.core.actions;
 
 import java.io.Serializable;
 import java.util.List;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.SynchronousQueue;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
 import java.util.function.Function;
 import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.exceptions.QAuthenticationException;
@@ -40,6 +44,20 @@ import com.kingsrook.qqq.backend.core.modules.authentication.QAuthenticationModu
  *******************************************************************************/
 public class ActionHelper
 {
+   /////////////////////////////////////////////////////////////////////////////
+   // we would probably use Executors.newCachedThreadPool() - but - it has no //
+   // maxPoolSize...  we think some limit is good, so that at a large number  //
+   // of attempted concurrent jobs we'll have new jobs block, rather than     //
+   // exhausting all server resources and locking up "everything"             //
+   // also, it seems like keeping a handful of core-threads around is very    //
+   // little actual waste, and better than ever wasting time starting a new   //
+   // one, which we know we'll often be doing.                                //
+   /////////////////////////////////////////////////////////////////////////////
+   private static Integer         CORE_THREADS    = 8;
+   private static Integer         MAX_THREADS     = 500;
+   private static ExecutorService executorService = new ThreadPoolExecutor(CORE_THREADS, MAX_THREADS, 60L, TimeUnit.SECONDS, new SynchronousQueue<>());
+
+
 
    /*******************************************************************************
     **
@@ -69,6 +87,17 @@ public class ActionHelper
 
 
 
+   /*******************************************************************************
+    ** access an executor service for sharing among the executeAsync methods of all
+    ** actions.
+    *******************************************************************************/
+   static ExecutorService getExecutorService()
+   {
+      return (executorService);
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/async/AsyncJobManager.java

@@ -28,16 +28,22 @@ import java.util.UUID;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.CompletionException;
 import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.SynchronousQueue;
+import java.util.concurrent.ThreadPoolExecutor;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.TimeoutException;
 import com.kingsrook.qqq.backend.core.context.CapturedContext;
 import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.exceptions.QUserFacingException;
 import com.kingsrook.qqq.backend.core.logging.QLogger;
 import com.kingsrook.qqq.backend.core.state.InMemoryStateProvider;
 import com.kingsrook.qqq.backend.core.state.StateProviderInterface;
 import com.kingsrook.qqq.backend.core.state.StateType;
 import com.kingsrook.qqq.backend.core.state.UUIDAndTypeStateKey;
+import com.kingsrook.qqq.backend.core.utils.StringUtils;
+import org.apache.logging.log4j.Level;
 import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
 
 
@@ -48,6 +54,22 @@ public class AsyncJobManager
 {
    private static final QLogger LOG = QLogger.getLogger(AsyncJobManager.class);
 
+   /////////////////////////////////////////////////////////////////////////////
+   // we would probably use Executors.newCachedThreadPool() - but - it has no //
+   // maxPoolSize...  we think some limit is good, so that at a large number  //
+   // of attempted concurrent jobs we'll have new jobs block, rather than     //
+   // exhausting all server resources and locking up "everything"             //
+   // also, it seems like keeping a handful of core-threads around is very    //
+   // little actual waste, and better than ever wasting time starting a new   //
+   // one, which we know we'll often be doing.                                //
+   /////////////////////////////////////////////////////////////////////////////
+   private static Integer         CORE_THREADS    = 8;
+   private static Integer         MAX_THREADS     = 500;
+   private static ExecutorService executorService = new ThreadPoolExecutor(CORE_THREADS, MAX_THREADS, 60L, TimeUnit.SECONDS, new SynchronousQueue<>());
+
+
+   private String forcedJobUUID = null;
+
 
 
    /*******************************************************************************
@@ -67,7 +89,8 @@ public class AsyncJobManager
     *******************************************************************************/
    public <T extends Serializable> T startJob(String jobName, long timeout, TimeUnit timeUnit, AsyncJob<T> asyncJob) throws JobGoingAsyncException, QException
    {
-      UUIDAndTypeStateKey uuidAndTypeStateKey = new UUIDAndTypeStateKey(UUID.randomUUID(), StateType.ASYNC_JOB_STATUS);
+      UUID                jobUUID             = StringUtils.hasContent(forcedJobUUID) ? UUID.fromString(forcedJobUUID) : UUID.randomUUID();
+      UUIDAndTypeStateKey uuidAndTypeStateKey = new UUIDAndTypeStateKey(jobUUID, StateType.ASYNC_JOB_STATUS);
       AsyncJobStatus      asyncJobStatus      = new AsyncJobStatus();
       asyncJobStatus.setState(AsyncJobState.RUNNING);
       getStateProvider().put(uuidAndTypeStateKey, asyncJobStatus);
@@ -79,7 +102,7 @@ public class AsyncJobManager
          {
             QContext.init(capturedContext);
             return (runAsyncJob(jobName, asyncJob, uuidAndTypeStateKey, asyncJobStatus));
-         });
+         }, executorService);
 
          if(timeout == 0)
          {
@@ -151,7 +174,11 @@ public class AsyncJobManager
          asyncJobStatus.setState(AsyncJobState.ERROR);
          asyncJobStatus.setCaughtException(e);
          getStateProvider().put(uuidAndTypeStateKey, asyncJobStatus);
-         LOG.warn("Job ended with an exception", e, logPair("jobId", uuidAndTypeStateKey.getUuid()));
+
+         //////////////////////////////////////////////////////
+         // if user facing, just log an info, warn otherwise //
+         //////////////////////////////////////////////////////
+         LOG.log((e instanceof QUserFacingException) ? Level.INFO : Level.WARN, "Job ended with an exception", e, logPair("jobId", uuidAndTypeStateKey.getUuid()));
          throw (new CompletionException(e));
       }
       finally
@@ -199,4 +226,35 @@ public class AsyncJobManager
       jobStatus.ifPresent(asyncJobStatus -> asyncJobStatus.setCancelRequested(true));
    }
 
+
+
+   /*******************************************************************************
+    ** Getter for forcedJobUUID
+    *******************************************************************************/
+   public String getForcedJobUUID()
+   {
+      return (this.forcedJobUUID);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for forcedJobUUID
+    *******************************************************************************/
+   public void setForcedJobUUID(String forcedJobUUID)
+   {
+      this.forcedJobUUID = forcedJobUUID;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for forcedJobUUID
+    *******************************************************************************/
+   public AsyncJobManager withForcedJobUUID(String forcedJobUUID)
+   {
+      this.forcedJobUUID = forcedJobUUID;
+      return (this);
+   }
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/async/AsyncRecordPipeLoop.java

@@ -25,6 +25,7 @@ package com.kingsrook.qqq.backend.core.actions.async;
 import java.io.Serializable;
 import java.util.Optional;
 import java.util.concurrent.TimeUnit;
+import com.kingsrook.qqq.backend.core.actions.reporting.BufferedRecordPipe;
 import com.kingsrook.qqq.backend.core.actions.reporting.RecordPipe;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.logging.QLogger;
@@ -142,6 +143,11 @@ public class AsyncRecordPipeLoop
          jobState = asyncJobStatus.getState();
       }
 
+      if(recordPipe instanceof BufferedRecordPipe bufferedRecordPipe)
+      {
+         bufferedRecordPipe.finalFlush();
+      }
+
       LOG.debug("Job [" + jobUUID + "][" + jobName + "] completed with status: " + asyncJobStatus);
 
       ///////////////////////////////////

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/audits/AuditAction.java

@@ -29,6 +29,7 @@ import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
+import java.util.Optional;
 import com.kingsrook.qqq.backend.core.actions.AbstractQActionFunction;
 import com.kingsrook.qqq.backend.core.actions.tables.GetAction;
 import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
@@ -44,10 +45,12 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertOutput;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
 import com.kingsrook.qqq.backend.core.model.metadata.security.RecordSecurityLock;
+import com.kingsrook.qqq.backend.core.model.metadata.security.RecordSecurityLockFilters;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 import com.kingsrook.qqq.backend.core.model.session.QUser;
 import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
 import com.kingsrook.qqq.backend.core.utils.Pair;
+import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
 
 
 /*******************************************************************************
@@ -76,6 +79,21 @@ public class AuditAction extends AbstractQActionFunction<AuditInput, AuditOutput
 
 
 
+   /*******************************************************************************
+    ** Execute to insert 1 audit, with a list of detail child records provided as just string messages
+    *******************************************************************************/
+   public static void executeWithStringDetails(String tableName, Integer recordId, Map<String, Serializable> securityKeyValues, String message, List<String> detailMessages)
+   {
+      List<QRecord> detailRecords = null;
+      if(CollectionUtils.nullSafeHasContents(detailMessages))
+      {
+         detailRecords = detailMessages.stream().map(m -> new QRecord().withValue("message", m)).toList();
+      }
+      execute(tableName, recordId, securityKeyValues, message, detailRecords);
+   }
+
+
+
    /*******************************************************************************
     ** Execute to insert 1 audit, with a list of detail child records
     *******************************************************************************/
@@ -92,12 +110,26 @@ public class AuditAction extends AbstractQActionFunction<AuditInput, AuditOutput
 
 
 
+   /*******************************************************************************
+    ** Simple overload that internally figures out primary key and security key values
+    **
+    ** Be aware - if the record doesn't have its security key values set (say it's a
+    ** partial record as part of an update), then those values won't be in the
+    ** security key map...  This should probably be considered a bug.
+    *******************************************************************************/
+   public static void appendToInput(AuditInput auditInput, QTableMetaData table, QRecord record, String auditMessage)
+   {
+      appendToInput(auditInput, table.getName(), record.getValueInteger(table.getPrimaryKeyField()), getRecordSecurityKeyValues(table, record, Optional.empty()), auditMessage);
+   }
+
+
+
    /*******************************************************************************
     ** Add 1 auditSingleInput to an AuditInput object - with no details (child records).
     *******************************************************************************/
-   public static AuditInput appendToInput(AuditInput auditInput, String tableName, Integer recordId, Map<String, Serializable> securityKeyValues, String message)
+   public static void appendToInput(AuditInput auditInput, String tableName, Integer recordId, Map<String, Serializable> securityKeyValues, String message)
    {
-      return (appendToInput(auditInput, tableName, recordId, securityKeyValues, message, null));
+      appendToInput(auditInput, tableName, recordId, securityKeyValues, message, null);
    }
 
 
@@ -123,6 +155,44 @@ public class AuditAction extends AbstractQActionFunction<AuditInput, AuditOutput
 
 
 
+   /*******************************************************************************
+    ** For a given record, from a given table, build a map of the record's security
+    ** key values.
+    **
+    ** If, in case, the record has null value(s), and the oldRecord is given (e.g.,
+    ** for the case of an update, where the record may not have all fields set, and
+    ** oldRecord should be known for doing field-diffs), then try to get the value(s)
+    ** from oldRecord.
+    **
+    ** Currently, will leave values null if they aren't found after that.
+    **
+    ** An alternative could be to re-fetch the record from its source if needed...
+    *******************************************************************************/
+   public static Map<String, Serializable> getRecordSecurityKeyValues(QTableMetaData table, QRecord record, Optional<QRecord> oldRecord)
+   {
+      Map<String, Serializable> securityKeyValues = new HashMap<>();
+      for(RecordSecurityLock recordSecurityLock : RecordSecurityLockFilters.filterForReadLocks(CollectionUtils.nonNullList(table.getRecordSecurityLocks())))
+      {
+         Serializable keyValue = record == null ? null : record.getValue(recordSecurityLock.getFieldName());
+
+         if(keyValue == null && oldRecord.isPresent())
+         {
+            LOG.debug("Table with a securityLock, but value not found in field", logPair("table", table.getName()), logPair("field", recordSecurityLock.getFieldName()));
+            keyValue = oldRecord.get().getValue(recordSecurityLock.getFieldName());
+         }
+
+         if(keyValue == null)
+         {
+            LOG.debug("Table with a securityLock, but value not found in field", logPair("table", table.getName()), logPair("field", recordSecurityLock.getFieldName()), logPair("oldRecordIsPresent", oldRecord.isPresent()));
+         }
+
+         securityKeyValues.put(recordSecurityLock.getSecurityKeyType(), keyValue);
+      }
+      return securityKeyValues;
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/
@@ -151,7 +221,7 @@ public class AuditAction extends AbstractQActionFunction<AuditInput, AuditOutput
                ///////////////////////////////////////////////////
                // validate security keys on the table are given //
                ///////////////////////////////////////////////////
-               for(RecordSecurityLock recordSecurityLock : CollectionUtils.nonNullList(table.getRecordSecurityLocks()))
+               for(RecordSecurityLock recordSecurityLock : RecordSecurityLockFilters.filterForReadLocks(CollectionUtils.nonNullList(table.getRecordSecurityLocks())))
                {
                   if(auditSingleInput.getSecurityKeyValues() == null || !auditSingleInput.getSecurityKeyValues().containsKey(recordSecurityLock.getSecurityKeyType()))
                   {
@@ -215,10 +285,13 @@ public class AuditAction extends AbstractQActionFunction<AuditInput, AuditOutput
                }
             }
 
-            insertInput = new InsertInput();
-            insertInput.setTableName("auditDetail");
-            insertInput.setRecords(auditDetailRecords);
-            new InsertAction().execute(insertInput);
+            if(!auditDetailRecords.isEmpty())
+            {
+               insertInput = new InsertInput();
+               insertInput.setTableName("auditDetail");
+               insertInput.setRecords(auditDetailRecords);
+               new InsertAction().execute(insertInput);
+            }
          }
          catch(Exception e)
          {

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/audits/DMLAuditAction.java

@@ -23,8 +23,10 @@ package com.kingsrook.qqq.backend.core.actions.audits;
 
 
 import java.io.Serializable;
+import java.math.BigDecimal;
 import java.time.Instant;
 import java.time.ZoneId;
+import java.time.temporal.ChronoUnit;
 import java.util.ArrayList;
 import java.util.Comparator;
 import java.util.HashMap;
@@ -52,12 +54,12 @@ import com.kingsrook.qqq.backend.core.model.metadata.audits.AuditLevel;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
 import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
-import com.kingsrook.qqq.backend.core.model.metadata.security.RecordSecurityLock;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 import com.kingsrook.qqq.backend.core.model.session.QSession;
 import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
 import com.kingsrook.qqq.backend.core.utils.StringUtils;
 import com.kingsrook.qqq.backend.core.utils.ValueUtils;
+import static com.kingsrook.qqq.backend.core.actions.audits.AuditAction.getRecordSecurityKeyValues;
 import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
 
 
@@ -69,6 +71,8 @@ public class DMLAuditAction extends AbstractQActionFunction<DMLAuditInput, DMLAu
 {
    private static final QLogger LOG = QLogger.getLogger(DMLAuditAction.class);
 
+   public static final String AUDIT_CONTEXT_FIELD_NAME = "auditContext";
+
 
 
    /*******************************************************************************
@@ -79,7 +83,6 @@ public class DMLAuditAction extends AbstractQActionFunction<DMLAuditInput, DMLAu
    {
       DMLAuditOutput           output           = new DMLAuditOutput();
       AbstractTableActionInput tableActionInput = input.getTableActionInput();
-      List<QRecord>            recordList       = input.getRecordList();
       List<QRecord>            oldRecordList    = input.getOldRecordList();
       QTableMetaData           table            = tableActionInput.getTable();
       long                     start            = System.currentTimeMillis();
@@ -87,6 +90,9 @@ public class DMLAuditAction extends AbstractQActionFunction<DMLAuditInput, DMLAu
 
       try
       {
+         List<QRecord> recordList = CollectionUtils.nonNullList(input.getRecordList()).stream()
+            .filter(r -> CollectionUtils.nullSafeIsEmpty(r.getErrors())).toList();
+
          AuditLevel auditLevel = getAuditLevel(tableActionInput);
          if(auditLevel == null || auditLevel.equals(AuditLevel.NONE) || CollectionUtils.nullSafeIsEmpty(recordList))
          {
@@ -96,29 +102,7 @@ public class DMLAuditAction extends AbstractQActionFunction<DMLAuditInput, DMLAu
             return (output);
          }
 
-         String                        contextSuffix = "";
-         Optional<AbstractActionInput> actionInput   = QContext.getFirstActionInStack();
-         if(actionInput.isPresent() && actionInput.get() instanceof RunProcessInput runProcessInput)
-         {
-            String           processName = runProcessInput.getProcessName();
-            QProcessMetaData process     = QContext.getQInstance().getProcess(processName);
-            if(process != null)
-            {
-               contextSuffix = " during process: " + process.getLabel();
-            }
-         }
-
-         QSession qSession   = QContext.getQSession();
-         String   apiVersion = qSession.getValue("apiVersion");
-         if(apiVersion != null)
-         {
-            String apiLabel = qSession.getValue("apiLabel");
-            if(!StringUtils.hasContent(apiLabel))
-            {
-               apiLabel = "API";
-            }
-            contextSuffix += (" via " + apiLabel + " Version: " + apiVersion);
-         }
+         String contextSuffix = getContentSuffix(input);
 
          AuditInput auditInput = new AuditInput();
          if(auditLevel.equals(AuditLevel.RECORD) || (auditLevel.equals(AuditLevel.FIELD) && !dmlType.supportsFields))
@@ -129,7 +113,7 @@ public class DMLAuditAction extends AbstractQActionFunction<DMLAuditInput, DMLAu
             ////////////////////////////////////////////////////////////////////////////////////////////////////////////
             for(QRecord record : recordList)
             {
-               AuditAction.appendToInput(auditInput, table.getName(), record.getValueInteger(table.getPrimaryKeyField()), getRecordSecurityKeyValues(table, record), "Record was " + dmlType.pastTenseVerb + contextSuffix);
+               AuditAction.appendToInput(auditInput, table.getName(), record.getValueInteger(table.getPrimaryKeyField()), getRecordSecurityKeyValues(table, record, Optional.empty()), "Record was " + dmlType.pastTenseVerb + contextSuffix);
             }
          }
          else if(auditLevel.equals(AuditLevel.FIELD))
@@ -139,7 +123,7 @@ public class DMLAuditAction extends AbstractQActionFunction<DMLAuditInput, DMLAu
             ///////////////////////////////////////////////////////////////////
             // do many audits, all with field level details, for FIELD level //
             ///////////////////////////////////////////////////////////////////
-            QPossibleValueTranslator qPossibleValueTranslator = new QPossibleValueTranslator(QContext.getQInstance(), qSession);
+            QPossibleValueTranslator qPossibleValueTranslator = new QPossibleValueTranslator(QContext.getQInstance(), QContext.getQSession());
             qPossibleValueTranslator.translatePossibleValuesInRecords(table, CollectionUtils.mergeLists(recordList, oldRecordList));
 
             //////////////////////////////////////////
@@ -161,67 +145,8 @@ public class DMLAuditAction extends AbstractQActionFunction<DMLAuditInput, DMLAu
                List<QRecord> details = new ArrayList<>();
                for(String fieldName : sortedFieldNames)
                {
-                  if(!record.getValues().containsKey(fieldName))
-                  {
-                     ////////////////////////////////////////////////////////////////////////////////////////////////
-                     // if the stored record doesn't have this field name, then don't audit anything about it      //
-                     // this is to deal with our Patch style updates not looking like every field was cleared out. //
-                     ////////////////////////////////////////////////////////////////////////////////////////////////
-                     continue;
-                  }
-
-                  if(fieldName.equals("modifyDate") || fieldName.equals("createDate") || fieldName.equals("automationStatus"))
-                  {
-                     continue;
-                  }
-
-                  QFieldMetaData field        = table.getField(fieldName);
-                  Serializable   value        = ValueUtils.getValueAsFieldType(field.getType(), record.getValue(fieldName));
-                  Serializable   oldValue     = oldRecord == null ? null : ValueUtils.getValueAsFieldType(field.getType(), oldRecord.getValue(fieldName));
-                  QRecord        detailRecord = null;
-
-                  if(oldRecord == null)
-                  {
-                     if(DMLType.INSERT.equals(dmlType) && value == null)
-                     {
-                        continue;
-                     }
-
-                     String formattedValue = getFormattedValueForAuditDetail(record, fieldName, field, value);
-                     detailRecord = new QRecord().withValue("message", "Set " + field.getLabel() + " to " + formattedValue);
-                     detailRecord.withValue("newValue", formattedValue);
-                  }
-                  else
-                  {
-                     if(!Objects.equals(oldValue, value))
-                     {
-                        String formattedValue    = getFormattedValueForAuditDetail(record, fieldName, field, value);
-                        String formattedOldValue = getFormattedValueForAuditDetail(oldRecord, fieldName, field, oldValue);
-
-                        if(oldValue == null)
-                        {
-                           detailRecord = new QRecord().withValue("message", "Set " + field.getLabel() + " to " + formatFormattedValueForDetailMessage(field, formattedValue));
-                           detailRecord.withValue("newValue", formattedValue);
-                        }
-                        else if(value == null)
-                        {
-                           detailRecord = new QRecord().withValue("message", "Removed " + formatFormattedValueForDetailMessage(field, formattedOldValue) + " from " + field.getLabel());
-                           detailRecord.withValue("oldValue", formattedOldValue);
-                        }
-                        else
-                        {
-                           detailRecord = new QRecord().withValue("message", "Changed " + field.getLabel() + " from " + formatFormattedValueForDetailMessage(field, formattedOldValue) + " to " + formatFormattedValueForDetailMessage(field, formattedValue));
-                           detailRecord.withValue("oldValue", formattedOldValue);
-                           detailRecord.withValue("newValue", formattedValue);
-                        }
-                     }
-                  }
-
-                  if(detailRecord != null)
-                  {
-                     detailRecord.withValue("fieldName", fieldName);
-                     details.add(detailRecord);
-                  }
+                  makeAuditDetailRecordForField(fieldName, table, dmlType, record, oldRecord)
+                     .ifPresent(details::add);
                }
 
                if(details.isEmpty() && DMLType.UPDATE.equals(dmlType))
@@ -231,7 +156,7 @@ public class DMLAuditAction extends AbstractQActionFunction<DMLAuditInput, DMLAu
                }
                else
                {
-                  AuditAction.appendToInput(auditInput, table.getName(), record.getValueInteger(table.getPrimaryKeyField()), getRecordSecurityKeyValues(table, record), "Record was " + dmlType.pastTenseVerb + contextSuffix, details);
+                  AuditAction.appendToInput(auditInput, table.getName(), record.getValueInteger(table.getPrimaryKeyField()), getRecordSecurityKeyValues(table, record, Optional.ofNullable(oldRecord)), "Record was " + dmlType.pastTenseVerb + contextSuffix, details);
                }
             }
          }
@@ -239,7 +164,7 @@ public class DMLAuditAction extends AbstractQActionFunction<DMLAuditInput, DMLAu
          // new AuditAction().executeAsync(auditInput); // todo async??? maybe get that from rules???
          new AuditAction().execute(auditInput);
          long end = System.currentTimeMillis();
-         LOG.debug("Audit performance", logPair("auditLevel", String.valueOf(auditLevel)), logPair("recordCount", recordList.size()), logPair("millis", (end - start)));
+         LOG.trace("Audit performance", logPair("auditLevel", String.valueOf(auditLevel)), logPair("recordCount", recordList.size()), logPair("millis", (end - start)));
       }
       catch(Exception e)
       {
@@ -251,6 +176,256 @@ public class DMLAuditAction extends AbstractQActionFunction<DMLAuditInput, DMLAu
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   static String getContentSuffix(DMLAuditInput input)
+   {
+      StringBuilder contextSuffix = new StringBuilder();
+
+      /////////////////////////////////////////////////////////////////////////////
+      // start with context from the input wrapper                               //
+      // note, these contexts get propagated down from Input/Update/Delete Input //
+      /////////////////////////////////////////////////////////////////////////////
+      if(StringUtils.hasContent(input.getAuditContext()))
+      {
+         contextSuffix.append(" ").append(input.getAuditContext());
+      }
+
+      /////////////////////////////////////////////////////////////////////////////////////
+      // note process label (and a possible context from the process's state) if present //
+      /////////////////////////////////////////////////////////////////////////////////////
+      Optional<AbstractActionInput> actionInput = QContext.getFirstActionInStack();
+      if(actionInput.isPresent() && actionInput.get() instanceof RunProcessInput runProcessInput)
+      {
+         String processAuditContext = ValueUtils.getValueAsString(runProcessInput.getValue(AUDIT_CONTEXT_FIELD_NAME));
+         if(StringUtils.hasContent(processAuditContext))
+         {
+            contextSuffix.append(" ").append(processAuditContext);
+         }
+
+         String           processName = runProcessInput.getProcessName();
+         QProcessMetaData process     = QContext.getQInstance().getProcess(processName);
+         if(process != null)
+         {
+            contextSuffix.append(" during process: ").append(process.getLabel());
+         }
+      }
+
+      ///////////////////////////////////////////////////
+      // use api label & version if present in session //
+      ///////////////////////////////////////////////////
+      QSession qSession   = QContext.getQSession();
+      String   apiVersion = qSession.getValue("apiVersion");
+      if(apiVersion != null)
+      {
+         String apiLabel = qSession.getValue("apiLabel");
+         if(!StringUtils.hasContent(apiLabel))
+         {
+            apiLabel = "API";
+         }
+         contextSuffix.append(" via ").append(apiLabel).append(" Version: ").append(apiVersion);
+      }
+      return (contextSuffix.toString());
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   static Optional<QRecord> makeAuditDetailRecordForField(String fieldName, QTableMetaData table, DMLType dmlType, QRecord record, QRecord oldRecord)
+   {
+      if(!record.getValues().containsKey(fieldName))
+      {
+         ////////////////////////////////////////////////////////////////////////////////////////////////
+         // if the stored record doesn't have this field name, then don't audit anything about it      //
+         // this is to deal with our Patch style updates not looking like every field was cleared out. //
+         ////////////////////////////////////////////////////////////////////////////////////////////////
+         return (Optional.empty());
+      }
+
+      if(fieldName.equals("modifyDate") || fieldName.equals("createDate") || fieldName.equals("automationStatus"))
+      {
+         return (Optional.empty());
+      }
+
+      QFieldMetaData field        = table.getField(fieldName);
+      Serializable   value        = ValueUtils.getValueAsFieldType(field.getType(), record.getValue(fieldName));
+      Serializable   oldValue     = oldRecord == null ? null : ValueUtils.getValueAsFieldType(field.getType(), oldRecord.getValue(fieldName));
+      QRecord        detailRecord = null;
+
+      if(oldRecord == null)
+      {
+         if(DMLType.INSERT.equals(dmlType) && value == null)
+         {
+            return (Optional.empty());
+         }
+
+         if(field.getType().equals(QFieldType.BLOB) || field.getType().needsMasked())
+         {
+            detailRecord = new QRecord().withValue("message", "Set " + field.getLabel());
+         }
+         else
+         {
+            String formattedValue = getFormattedValueForAuditDetail(record, fieldName, field, value);
+            detailRecord = new QRecord().withValue("message", "Set " + field.getLabel() + " to " + formattedValue);
+            detailRecord.withValue("newValue", formattedValue);
+         }
+      }
+      else
+      {
+         if(areValuesDifferentForAudit(field, value, oldValue))
+         {
+            if(field.getType().equals(QFieldType.BLOB) || field.getType().needsMasked())
+            {
+               if(oldValue == null)
+               {
+                  detailRecord = new QRecord().withValue("message", "Set " + field.getLabel());
+               }
+               else if(value == null)
+               {
+                  detailRecord = new QRecord().withValue("message", "Removed " + field.getLabel());
+               }
+               else
+               {
+                  detailRecord = new QRecord().withValue("message", "Changed " + field.getLabel());
+               }
+            }
+            else
+            {
+               String formattedValue    = getFormattedValueForAuditDetail(record, fieldName, field, value);
+               String formattedOldValue = getFormattedValueForAuditDetail(oldRecord, fieldName, field, oldValue);
+
+               if(oldValue == null)
+               {
+                  detailRecord = new QRecord().withValue("message", "Set " + field.getLabel() + " to " + formatFormattedValueForDetailMessage(field, formattedValue));
+                  detailRecord.withValue("newValue", formattedValue);
+               }
+               else if(value == null)
+               {
+                  detailRecord = new QRecord().withValue("message", "Removed " + formatFormattedValueForDetailMessage(field, formattedOldValue) + " from " + field.getLabel());
+                  detailRecord.withValue("oldValue", formattedOldValue);
+               }
+               else
+               {
+                  detailRecord = new QRecord().withValue("message", "Changed " + field.getLabel() + " from " + formatFormattedValueForDetailMessage(field, formattedOldValue) + " to " + formatFormattedValueForDetailMessage(field, formattedValue));
+                  detailRecord.withValue("oldValue", formattedOldValue);
+                  detailRecord.withValue("newValue", formattedValue);
+               }
+            }
+         }
+      }
+
+      if(detailRecord != null)
+      {
+         ////////////////////////////////////////////////////////////////////
+         // useful if doing dev in here - but overkill for any other time. //
+         ////////////////////////////////////////////////////////////////////
+         // LOG.debug("Returning with message: " + detailRecord.getValueString("message"));
+         detailRecord.withValue("fieldName", fieldName);
+         return (Optional.of(detailRecord));
+      }
+
+      return (Optional.empty());
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   static boolean areValuesDifferentForAudit(QFieldMetaData field, Serializable value, Serializable oldValue)
+   {
+      try
+      {
+         ///////////////////
+         // decimal rules //
+         ///////////////////
+         if(field.getType().equals(QFieldType.DECIMAL))
+         {
+            BigDecimal newBD = ValueUtils.getValueAsBigDecimal(value);
+            BigDecimal oldBD = ValueUtils.getValueAsBigDecimal(oldValue);
+
+            if(newBD == null && oldBD == null)
+            {
+               return (false);
+            }
+
+            if(newBD == null || oldBD == null)
+            {
+               return (true);
+            }
+
+            return (newBD.compareTo(oldBD) != 0);
+         }
+
+         ////////////////////
+         // dateTime rules //
+         ////////////////////
+         if(field.getType().equals(QFieldType.DATE_TIME))
+         {
+            Instant newI = ValueUtils.getValueAsInstant(value);
+            Instant oldI = ValueUtils.getValueAsInstant(oldValue);
+
+            if(newI == null && oldI == null)
+            {
+               return (false);
+            }
+
+            if(newI == null || oldI == null)
+            {
+               return (true);
+            }
+
+            ////////////////////////////////
+            // just compare to the second //
+            ////////////////////////////////
+            return (newI.truncatedTo(ChronoUnit.SECONDS).compareTo(oldI.truncatedTo(ChronoUnit.SECONDS)) != 0);
+         }
+
+         //////////////////
+         // string rules //
+         //////////////////
+         if(field.getType().isStringLike())
+         {
+            String newString = ValueUtils.getValueAsString(value);
+            String oldString = ValueUtils.getValueAsString(oldValue);
+
+            boolean newIsNullOrEmpty = !StringUtils.hasContent(newString);
+            boolean oldIsNullOrEmpty = !StringUtils.hasContent(oldString);
+
+            if(newIsNullOrEmpty && oldIsNullOrEmpty)
+            {
+               return (false);
+            }
+
+            if(newIsNullOrEmpty || oldIsNullOrEmpty)
+            {
+               return (true);
+            }
+
+            return (newString.compareTo(oldString) != 0);
+         }
+
+         /////////////////////////////////////
+         // default just use Objects.equals //
+         /////////////////////////////////////
+         return !Objects.equals(oldValue, value);
+      }
+      catch(Exception e)
+      {
+         LOG.debug("Error checking areValuesDifferentForAudit", e, logPair("fieldName", field.getName()), logPair("value", value), logPair("oldValue", oldValue));
+      }
+
+      ////////////////////////////////////
+      // default to something simple... //
+      ////////////////////////////////////
+      return !Objects.equals(oldValue, value);
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/
@@ -340,21 +515,6 @@ public class DMLAuditAction extends AbstractQActionFunction<DMLAuditInput, DMLAu
 
 
 
-   /*******************************************************************************
-    **
-    *******************************************************************************/
-   private static Map<String, Serializable> getRecordSecurityKeyValues(QTableMetaData table, QRecord record)
-   {
-      Map<String, Serializable> securityKeyValues = new HashMap<>();
-      for(RecordSecurityLock recordSecurityLock : CollectionUtils.nonNullList(table.getRecordSecurityLocks()))
-      {
-         securityKeyValues.put(recordSecurityLock.getSecurityKeyType(), record == null ? null : record.getValue(recordSecurityLock.getFieldName()));
-      }
-      return securityKeyValues;
-   }
-
-
-
    /*******************************************************************************
     **
     *******************************************************************************/
@@ -374,7 +534,7 @@ public class DMLAuditAction extends AbstractQActionFunction<DMLAuditInput, DMLAu
    /*******************************************************************************
     **
     *******************************************************************************/
-   private enum DMLType
+   enum DMLType
    {
       INSERT("Inserted", true),
       UPDATE("Edited", true),

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/automation/RecordAutomationStatusUpdater.java

@@ -22,9 +22,8 @@
 package com.kingsrook.qqq.backend.core.actions.automation;
 
 
-import java.util.ArrayList;
+import java.util.Collections;
 import java.util.List;
-import java.util.Objects;
 import com.kingsrook.qqq.backend.core.actions.tables.CountAction;
 import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction;
 import com.kingsrook.qqq.backend.core.context.QContext;
@@ -90,6 +89,10 @@ public class RecordAutomationStatusUpdater
          }
       }
 
+      ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      // Avoid setting records to PENDING_INSERT or PENDING_UPDATE even if they don't have any insert or update automations or triggers //
+      // such records should go straight to OK status.                                                                                  //
+      ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
       if(canWeSkipPendingAndGoToOkay(table, automationStatus))
       {
          automationStatus = AutomationStatus.OK;
@@ -121,9 +124,13 @@ public class RecordAutomationStatusUpdater
     ** being asked to set status to PENDING_INSERT (or PENDING_UPDATE), then just
     ** move the status straight to OK.
     *******************************************************************************/
-   private static boolean canWeSkipPendingAndGoToOkay(QTableMetaData table, AutomationStatus automationStatus)
+   static boolean canWeSkipPendingAndGoToOkay(QTableMetaData table, AutomationStatus automationStatus)
    {
-      List<TableAutomationAction> tableActions = Objects.requireNonNullElse(table.getAutomationDetails().getActions(), new ArrayList<>());
+      List<TableAutomationAction> tableActions = Collections.emptyList();
+      if(table.getAutomationDetails() != null && table.getAutomationDetails().getActions() != null)
+      {
+         tableActions = table.getAutomationDetails().getActions();
+      }
 
       if(automationStatus.equals(AutomationStatus.PENDING_INSERT_AUTOMATIONS))
       {
@@ -135,6 +142,12 @@ public class RecordAutomationStatusUpdater
          {
             return (false);
          }
+
+         ////////////////////////////////////////////////////////////////////////////////////////
+         // if we're going to pending-insert, and there are no insert automations or triggers, //
+         // then we may skip pending and go to okay.                                           //
+         ////////////////////////////////////////////////////////////////////////////////////////
+         return (true);
       }
       else if(automationStatus.equals(AutomationStatus.PENDING_UPDATE_AUTOMATIONS))
       {
@@ -146,9 +159,21 @@ public class RecordAutomationStatusUpdater
          {
             return (false);
          }
-      }
 
-      return (true);
+         ////////////////////////////////////////////////////////////////////////////////////////
+         // if we're going to pending-update, and there are no insert automations or triggers, //
+         // then we may skip pending and go to okay.                                           //
+         ////////////////////////////////////////////////////////////////////////////////////////
+         return (true);
+      }
+      else
+      {
+         ///////////////////////////////////////////////////////////////////////////////////////////////////////
+         // if we're going to any other automation status - then we may never "skip pending" and go to okay - //
+         // because we weren't asked to go to pending!                                                        //
+         ///////////////////////////////////////////////////////////////////////////////////////////////////////
+         return (false);
+      }
    }
 
 

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/automation/RunRecordScriptAutomationHandler.java

@@ -42,6 +42,7 @@ import com.kingsrook.qqq.backend.core.model.data.QRecord;
 import com.kingsrook.qqq.backend.core.model.metadata.code.AdHocScriptCodeReference;
 import com.kingsrook.qqq.backend.core.model.scripts.Script;
 import com.kingsrook.qqq.backend.core.model.scripts.ScriptRevision;
+import com.kingsrook.qqq.backend.core.model.scripts.ScriptsMetaDataProvider;
 import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
 import com.kingsrook.qqq.backend.core.utils.ValueUtils;
 import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
@@ -74,7 +75,7 @@ public class RunRecordScriptAutomationHandler extends RecordAutomationHandler
       QueryInput queryInput = new QueryInput();
       queryInput.setTableName(ScriptRevision.TABLE_NAME);
       queryInput.setFilter(new QQueryFilter(new QFilterCriteria("scriptId", QCriteriaOperator.EQUALS, scriptId)));
-      queryInput.withQueryJoin(new QueryJoin(Script.TABLE_NAME).withBaseTableOrAlias(ScriptRevision.TABLE_NAME).withJoinMetaData(QContext.getQInstance().getJoin("currentScriptRevision")));
+      queryInput.withQueryJoin(new QueryJoin(Script.TABLE_NAME).withBaseTableOrAlias(ScriptRevision.TABLE_NAME).withJoinMetaData(QContext.getQInstance().getJoin(ScriptsMetaDataProvider.CURRENT_SCRIPT_REVISION_JOIN_NAME)));
       QueryOutput queryOutput = new QueryAction().execute(queryInput);
       if(CollectionUtils.nullSafeIsEmpty(queryOutput.getRecords()))
       {

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/automation/polling/PollingAutomationPerTableRunner.java

@@ -39,12 +39,15 @@ import com.kingsrook.qqq.backend.core.actions.customizers.QCodeLoader;
 import com.kingsrook.qqq.backend.core.actions.processes.QProcessCallback;
 import com.kingsrook.qqq.backend.core.actions.processes.RunProcessAction;
 import com.kingsrook.qqq.backend.core.actions.reporting.RecordPipe;
+import com.kingsrook.qqq.backend.core.actions.tables.GetAction;
 import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
 import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.logging.QLogger;
 import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput;
 import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessOutput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetOutput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy;
@@ -61,11 +64,14 @@ import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.Automatio
 import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.QTableAutomationDetails;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.TableAutomationAction;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.TriggerEvent;
+import com.kingsrook.qqq.backend.core.model.savedfilters.SavedFilter;
 import com.kingsrook.qqq.backend.core.model.session.QSession;
 import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
+import com.kingsrook.qqq.backend.core.utils.JsonUtils;
 import com.kingsrook.qqq.backend.core.utils.StringUtils;
 import com.kingsrook.qqq.backend.core.utils.collections.MapBuilder;
 import org.apache.commons.lang.NotImplementedException;
+import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
 
 
 /*******************************************************************************
@@ -270,15 +276,39 @@ public class PollingAutomationPerTableRunner implements Runnable
          QueryOutput queryOutput = new QueryAction().execute(queryInput);
          for(QRecord record : queryOutput.getRecords())
          {
-            // todo - get filter if there is/was one
-            rs.add(new TableAutomationAction()
-               .withName("Script:" + record.getValue("scriptId"))
-               .withFilter(null)
-               .withTriggerEvent(triggerEvent)
-               .withPriority(record.getValueInteger("priority"))
-               .withCodeReference(new QCodeReference(RunRecordScriptAutomationHandler.class))
-               .withValues(MapBuilder.of("scriptId", record.getValue("scriptId")))
-            );
+            TableTrigger tableTrigger = new TableTrigger(record);
+
+            try
+            {
+               QQueryFilter filter   = null;
+               Integer      filterId = tableTrigger.getFilterId();
+               if(filterId != null)
+               {
+                  GetInput getInput = new GetInput();
+                  getInput.setTableName(SavedFilter.TABLE_NAME);
+                  getInput.setPrimaryKey(filterId);
+                  GetOutput getOutput = new GetAction().execute(getInput);
+                  if(getOutput.getRecord() != null)
+                  {
+                     SavedFilter savedFilter = new SavedFilter(getOutput.getRecord());
+                     filter = JsonUtils.toObject(savedFilter.getFilterJson(), QQueryFilter.class);
+                  }
+               }
+
+               rs.add(new TableAutomationAction()
+                  .withName("Script:" + tableTrigger.getScriptId())
+                  .withFilter(filter)
+                  .withTriggerEvent(triggerEvent)
+                  .withPriority(tableTrigger.getPriority())
+                  .withCodeReference(new QCodeReference(RunRecordScriptAutomationHandler.class))
+                  .withValues(MapBuilder.of("scriptId", tableTrigger.getScriptId()))
+                  .withIncludeRecordAssociations(true)
+               );
+            }
+            catch(Exception e)
+            {
+               LOG.error("Error setting up table trigger", e, logPair("tableTriggerId", tableTrigger.getId()));
+            }
          }
       }
 
@@ -312,22 +342,9 @@ public class PollingAutomationPerTableRunner implements Runnable
       boolean anyActionsFailed = false;
       for(TableAutomationAction action : actions)
       {
-         try
+         boolean hadError = applyActionToRecords(table, records, action);
+         if(hadError)
          {
-            //////////////////////////////////////////////////////////////////////////////////////////////////////////////
-            // note - this method - will re-query the objects, so we should have confidence that their data is fresh... //
-            //////////////////////////////////////////////////////////////////////////////////////////////////////////////
-            List<QRecord> matchingQRecords = getRecordsMatchingActionFilter(table, records, action);
-            LOG.debug("Of the {} records that were pending automations, {} of them match the filter on the action {}", records.size(), matchingQRecords.size(), action);
-            if(CollectionUtils.nullSafeHasContents(matchingQRecords))
-            {
-               LOG.debug("  Processing " + matchingQRecords.size() + " records in " + table + " for action " + action);
-               applyActionToMatchingRecords(table, matchingQRecords, action);
-            }
-         }
-         catch(Exception e)
-         {
-            LOG.warn("Caught exception processing records on " + table + " for action " + action, e);
             anyActionsFailed = true;
          }
       }
@@ -347,6 +364,37 @@ public class PollingAutomationPerTableRunner implements Runnable
 
 
 
+   /*******************************************************************************
+    ** Run one action over a list of records (if they match the action's filter).
+    **
+    ** @return hadError - true if an exception was caught; false if all OK.
+    *******************************************************************************/
+   protected boolean applyActionToRecords(QTableMetaData table, List<QRecord> records, TableAutomationAction action)
+   {
+      try
+      {
+         //////////////////////////////////////////////////////////////////////////////////////////////////////////////
+         // note - this method - will re-query the objects, so we should have confidence that their data is fresh... //
+         //////////////////////////////////////////////////////////////////////////////////////////////////////////////
+         List<QRecord> matchingQRecords = getRecordsMatchingActionFilter(table, records, action);
+         LOG.debug("Of the {} records that were pending automations, {} of them match the filter on the action {}", records.size(), matchingQRecords.size(), action);
+         if(CollectionUtils.nullSafeHasContents(matchingQRecords))
+         {
+            LOG.debug("  Processing " + matchingQRecords.size() + " records in " + table + " for action " + action);
+            applyActionToMatchingRecords(table, matchingQRecords, action);
+         }
+
+         return (false);
+      }
+      catch(Exception e)
+      {
+         LOG.warn("Caught exception processing records on " + table + " for action " + action, e);
+         return (true);
+      }
+   }
+
+
+
    /*******************************************************************************
     ** For a given action, and a list of records - return a new list, of the ones
     ** which match the action's filter (if there is one - if not, then all match).
@@ -364,26 +412,29 @@ public class PollingAutomationPerTableRunner implements Runnable
       QueryInput queryInput = new QueryInput();
       queryInput.setTableName(table.getName());
 
+      ///////////////////////////////////////////////////////////////////////////////////////
+      // set up a filter that is for the primary keys IN the list that we identified above //
+      ///////////////////////////////////////////////////////////////////////////////////////
       QQueryFilter filter = new QQueryFilter();
+      filter.addCriteria(new QFilterCriteria(table.getPrimaryKeyField(), QCriteriaOperator.IN, records.stream().map(r -> r.getValue(table.getPrimaryKeyField())).toList()));
 
-      /////////////////////////////////////////////////////////////////////////////////////////////////////
-      // copy filter criteria from the action's filter to a new filter that we'll run here.              //
-      // Critically - don't modify the filter object on the action!  as that object has a long lifespan. //
-      /////////////////////////////////////////////////////////////////////////////////////////////////////
       if(action.getFilter() != null)
       {
-         if(action.getFilter().getCriteria() != null)
-         {
-            action.getFilter().getCriteria().forEach(filter::addCriteria);
-         }
+         /////////////////////////////////////////////////////////////////////////////////////////////////////
+         // if the action defines a filter of its own, add that to the filter we'll run now as a sub-filter //
+         // not entirely clear if this needs to be a clone, but, it feels safe and cheap enough             //
+         /////////////////////////////////////////////////////////////////////////////////////////////////////
+         filter.addSubFilter(action.getFilter().clone());
+
+         ////////////////////////////////////////////////////////////////////////////////////////////////////////////
+         // we also want to set order-bys from the action into our filter (since they only apply at the top-level) //
+         ////////////////////////////////////////////////////////////////////////////////////////////////////////////
          if(action.getFilter().getOrderBys() != null)
          {
             action.getFilter().getOrderBys().forEach(filter::addOrderBy);
          }
       }
 
-      filter.addCriteria(new QFilterCriteria(table.getPrimaryKeyField(), QCriteriaOperator.IN, records.stream().map(r -> r.getValue(table.getPrimaryKeyField())).toList()));
-
       /////////////////////////////////////////////////////////////////////////////////////////////
       // always add order-by the primary key, to give more predictable/consistent results        //
       // todo - in future - if this becomes a source of slowness, make this a config to opt-out? //
@@ -392,6 +443,8 @@ public class PollingAutomationPerTableRunner implements Runnable
 
       queryInput.setFilter(filter);
 
+      queryInput.setIncludeAssociations(action.getIncludeRecordAssociations());
+
       return (new QueryAction().execute(queryInput).getRecords());
    }
 

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/AbstractPostDeleteCustomizer.java

@@ -0,0 +1,82 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.actions.customizers;
+
+
+import java.util.List;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+
+
+/*******************************************************************************
+ ** Abstract class that a table can specify an implementation of, to provide
+ ** custom actions after a delete takes place.
+ **
+ ** General implementation would be, to iterate over the records (ones which didn't
+ ** have a delete error), and look at their values:
+ ** - possibly adding Errors (`addError`) or Warnings (`addWarning`) to the records?
+ ** - possibly throwing an exception - though doing so won't stop the delete, and instead
+ **   will just set a warning on all of the deleted records...
+ ** - doing "whatever else" you may want to do.
+ ** - returning the list of records (can be the input list) that you want to go back
+ **   to the caller - this is how errors and warnings are propagated .
+ **
+ ** Note that the full deleteInput is available as a field in this class.
+ **
+ ** A future enhancement here may be to take (as fields in this class) the list of
+ ** records that the delete action marked in error - the user might want to do
+ ** something special with them (idk, try some other way to delete them?)
+ *******************************************************************************/
+public abstract class AbstractPostDeleteCustomizer
+{
+   protected DeleteInput deleteInput;
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public abstract List<QRecord> apply(List<QRecord> records) throws QException;
+
+
+
+   /*******************************************************************************
+    ** Getter for deleteInput
+    **
+    *******************************************************************************/
+   public DeleteInput getDeleteInput()
+   {
+      return deleteInput;
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for deleteInput
+    **
+    *******************************************************************************/
+   public void setDeleteInput(DeleteInput deleteInput)
+   {
+      this.deleteInput = deleteInput;
+   }
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/AbstractPostInsertCustomizer.java

@@ -23,12 +23,24 @@ package com.kingsrook.qqq.backend.core.actions.customizers;
 
 
 import java.util.List;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
 
 
 /*******************************************************************************
+ ** Abstract class that a table can specify an implementation of, to provide
+ ** custom actions after an insert takes place.
  **
+ ** General implementation would be, to iterate over the records (the outputs of
+ ** the insert action), and look at their values:
+ ** - possibly adding Errors (`addError`) or Warnings (`addWarning`) to the records
+ ** - possibly throwing an exception - though doing so won't stop the update, and instead
+ **   will just set a warning on all of the updated records...
+ ** - doing "whatever else" you may want to do.
+ ** - returning the list of records (can be the input list) that you want to go back to the caller.
+ **
+ ** Note that the full insertInput is available as a field in this class.
  *******************************************************************************/
 public abstract class AbstractPostInsertCustomizer
 {
@@ -39,7 +51,7 @@ public abstract class AbstractPostInsertCustomizer
    /*******************************************************************************
     **
     *******************************************************************************/
-   public abstract List<QRecord> apply(List<QRecord> records);
+   public abstract List<QRecord> apply(List<QRecord> records) throws QException;
 
 
 

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/AbstractPostUpdateCustomizer.java

@@ -0,0 +1,130 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.actions.customizers;
+
+
+import java.io.Serializable;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+
+
+/*******************************************************************************
+ ** Abstract class that a table can specify an implementation of, to provide
+ ** custom actions after an update takes place.
+ **
+ ** General implementation would be, to iterate over the records (the outputs of
+ ** the update action), and look at their values:
+ ** - possibly adding Errors (`addError`) or Warnings (`addWarning`) to the records?
+ ** - possibly throwing an exception - though doing so won't stop the update, and instead
+ **   will just set a warning on all of the updated records...
+ ** - doing "whatever else" you may want to do.
+ ** - returning the list of records (can be the input list) that you want to go back to the caller.
+ **
+ ** Note that the full updateInput is available as a field in this class, and the
+ ** "old records" (e.g., with values freshly fetched from the backend) will be
+ ** available (if the backend supports it) - both as a list (`getOldRecordList`)
+ ** and as a memoized (by this class) map of primaryKey to record (`getOldRecordMap`).
+ *******************************************************************************/
+public abstract class AbstractPostUpdateCustomizer
+{
+   protected UpdateInput   updateInput;
+   protected List<QRecord> oldRecordList;
+
+   private Map<Serializable, QRecord> oldRecordMap = null;
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public abstract List<QRecord> apply(List<QRecord> records) throws QException;
+
+
+
+   /*******************************************************************************
+    ** Getter for updateInput
+    **
+    *******************************************************************************/
+   public UpdateInput getUpdateInput()
+   {
+      return updateInput;
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for updateInput
+    **
+    *******************************************************************************/
+   public void setUpdateInput(UpdateInput updateInput)
+   {
+      this.updateInput = updateInput;
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public void setOldRecordList(List<QRecord> oldRecordList)
+   {
+      this.oldRecordList = oldRecordList;
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public List<QRecord> getOldRecordList()
+   {
+      return oldRecordList;
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   protected Map<Serializable, QRecord> getOldRecordMap()
+   {
+      if(oldRecordMap == null)
+      {
+         oldRecordMap = new HashMap<>();
+
+         if(oldRecordList != null && updateInput != null)
+         {
+            for(QRecord qRecord : oldRecordList)
+            {
+               oldRecordMap.put(qRecord.getValue(updateInput.getTable().getPrimaryKeyField()), qRecord);
+            }
+         }
+      }
+
+      return (oldRecordMap);
+   }
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/AbstractPreDeleteCustomizer.java

@@ -0,0 +1,119 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.actions.customizers;
+
+
+import java.util.List;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+
+
+/*******************************************************************************
+ ** Abstract class that a table can specify an implementation of, to provide
+ ** custom actions before a delete takes place.
+ **
+ ** It's important for implementations to be aware of the isPreview field, which
+ ** is set to true when the code is running to give users advice, e.g., on a review
+ ** screen - vs. being false when the action is ACTUALLY happening.  So, if you're doing
+ ** things like storing data, you don't want to do that if isPreview is true!!
+ **
+ ** General implementation would be, to iterate over the records (which the DeleteAction
+ ** would look up based on the inputs to the delete action), and look at their values:
+ ** - possibly adding Errors (`addError`) or Warnings (`addWarning`) to the records
+ ** - possibly throwing an exception - if you really don't want the delete operation to continue.
+ ** - doing "whatever else" you may want to do.
+ ** - returning the list of records (can be the input list) - this is how errors
+ **   and warnings are propagated to the DeleteAction.  Note that any records with
+ **   an error will NOT proceed to the backend's delete interface - but those with
+ **   warnings will.
+ **
+ ** Note that the full deleteInput is available as a field in this class.
+ **
+ *******************************************************************************/
+public abstract class AbstractPreDeleteCustomizer
+{
+   protected DeleteInput deleteInput;
+
+   protected boolean isPreview = false;
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public abstract List<QRecord> apply(List<QRecord> records) throws QException;
+
+
+
+   /*******************************************************************************
+    ** Getter for deleteInput
+    **
+    *******************************************************************************/
+   public DeleteInput getDeleteInput()
+   {
+      return deleteInput;
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for deleteInput
+    **
+    *******************************************************************************/
+   public void setDeleteInput(DeleteInput deleteInput)
+   {
+      this.deleteInput = deleteInput;
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for isPreview
+    *******************************************************************************/
+   public boolean getIsPreview()
+   {
+      return (this.isPreview);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for isPreview
+    *******************************************************************************/
+   public void setIsPreview(boolean isPreview)
+   {
+      this.isPreview = isPreview;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for isPreview
+    *******************************************************************************/
+   public AbstractPreDeleteCustomizer withIsPreview(boolean isPreview)
+   {
+      this.isPreview = isPreview;
+      return (this);
+   }
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/AbstractPreInsertCustomizer.java

@@ -0,0 +1,141 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.actions.customizers;
+
+
+import java.util.List;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+
+
+/*******************************************************************************
+ ** Abstract class that a table can specify an implementation of, to provide
+ ** custom actions before an insert takes place.
+ **
+ ** It's important for implementations to be aware of the isPreview field, which
+ ** is set to true when the code is running to give users advice, e.g., on a review
+ ** screen - vs. being false when the action is ACTUALLY happening.  So, if you're doing
+ ** things like storing data, you don't want to do that if isPreview is true!!
+ **
+ ** General implementation would be, to iterate over the records (the inputs to
+ ** the insert action), and look at their values:
+ ** - possibly adding Errors (`addError`) or Warnings (`addWarning`) to the records
+ ** - possibly manipulating values (`setValue`)
+ ** - possibly throwing an exception - if you really don't want the insert operation to continue.
+ ** - doing "whatever else" you may want to do.
+ ** - returning the list of records (can be the input list) that you want to go on to the backend implementation class.
+ **
+ ** Note that the full insertInput is available as a field in this class.
+ *******************************************************************************/
+public abstract class AbstractPreInsertCustomizer
+{
+   protected InsertInput insertInput;
+
+   protected boolean isPreview = false;
+
+
+
+   /////////////////////////////////////////////////////////////////////////////////
+   // allow the customizer to specify when it should be executed as part of the   //
+   // insert action.  default (per method in this class) is AFTER_ALL_VALIDATIONS //
+   /////////////////////////////////////////////////////////////////////////////////
+   public enum WhenToRun
+   {
+      BEFORE_ALL_VALIDATIONS,
+      BEFORE_UNIQUE_KEY_CHECKS,
+      BEFORE_REQUIRED_FIELD_CHECKS,
+      BEFORE_SECURITY_CHECKS,
+      AFTER_ALL_VALIDATIONS
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public abstract List<QRecord> apply(List<QRecord> records) throws QException;
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public WhenToRun getWhenToRun()
+   {
+      return (WhenToRun.AFTER_ALL_VALIDATIONS);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for insertInput
+    **
+    *******************************************************************************/
+   public InsertInput getInsertInput()
+   {
+      return insertInput;
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for insertInput
+    **
+    *******************************************************************************/
+   public void setInsertInput(InsertInput insertInput)
+   {
+      this.insertInput = insertInput;
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for isPreview
+    *******************************************************************************/
+   public boolean getIsPreview()
+   {
+      return (this.isPreview);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for isPreview
+    *******************************************************************************/
+   public void setIsPreview(boolean isPreview)
+   {
+      this.isPreview = isPreview;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for isPreview
+    *******************************************************************************/
+   public AbstractPreInsertCustomizer withIsPreview(boolean isPreview)
+   {
+      this.isPreview = isPreview;
+      return (this);
+   }
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/AbstractPreUpdateCustomizer.java

@@ -0,0 +1,163 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.actions.customizers;
+
+
+import java.io.Serializable;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+
+
+/*******************************************************************************
+ ** Abstract class that a table can specify an implementation of, to provide
+ ** custom actions before an update takes place.
+ **
+ ** It's important for implementations to be aware of the isPreview field, which
+ ** is set to true when the code is running to give users advice, e.g., on a review
+ ** screen - vs. being false when the action is ACTUALLY happening.  So, if you're doing
+ ** things like storing data, you don't want to do that if isPreview is true!!
+ **
+ ** General implementation would be, to iterate over the records (the inputs to
+ ** the update action), and look at their values:
+ ** - possibly adding Errors (`addError`) or Warnings (`addWarning`) to the records
+ ** - possibly manipulating values (`setValue`)
+ ** - possibly throwing an exception - if you really don't want the update operation to continue.
+ ** - doing "whatever else" you may want to do.
+ ** - returning the list of records (can be the input list) that you want to go on to the backend implementation class.
+ **
+ ** Note that the full updateInput is available as a field in this class, and the
+ ** "old records" (e.g., with values freshly fetched from the backend) will be
+ ** available (if the backend supports it) - both as a list (`getOldRecordList`)
+ ** and as a memoized (by this class) map of primaryKey to record (`getOldRecordMap`).
+ *******************************************************************************/
+public abstract class AbstractPreUpdateCustomizer
+{
+   protected UpdateInput   updateInput;
+   protected List<QRecord> oldRecordList;
+   protected boolean       isPreview = false;
+
+   private Map<Serializable, QRecord> oldRecordMap = null;
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public abstract List<QRecord> apply(List<QRecord> records) throws QException;
+
+
+
+   /*******************************************************************************
+    ** Getter for updateInput
+    **
+    *******************************************************************************/
+   public UpdateInput getUpdateInput()
+   {
+      return updateInput;
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for updateInput
+    **
+    *******************************************************************************/
+   public void setUpdateInput(UpdateInput updateInput)
+   {
+      this.updateInput = updateInput;
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public void setOldRecordList(List<QRecord> oldRecordList)
+   {
+      this.oldRecordList = oldRecordList;
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public List<QRecord> getOldRecordList()
+   {
+      return oldRecordList;
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   protected Map<Serializable, QRecord> getOldRecordMap()
+   {
+      if(oldRecordMap == null)
+      {
+         oldRecordMap = new HashMap<>();
+         for(QRecord qRecord : oldRecordList)
+         {
+            oldRecordMap.put(qRecord.getValue(updateInput.getTable().getPrimaryKeyField()), qRecord);
+         }
+      }
+
+      return (oldRecordMap);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for isPreview
+    *******************************************************************************/
+   public boolean getIsPreview()
+   {
+      return (this.isPreview);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for isPreview
+    *******************************************************************************/
+   public void setIsPreview(boolean isPreview)
+   {
+      this.isPreview = isPreview;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for isPreview
+    *******************************************************************************/
+   public AbstractPreUpdateCustomizer withIsPreview(boolean isPreview)
+   {
+      this.isPreview = isPreview;
+      return (this);
+   }
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/ChildInserterPostInsertCustomizer.java

@@ -29,28 +29,30 @@ import java.util.List;
 import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
 import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.exceptions.QRuntimeException;
 import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertOutput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+import com.kingsrook.qqq.backend.core.model.statusmessages.QStatusMessage;
+import com.kingsrook.qqq.backend.core.model.statusmessages.QWarningMessage;
+import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
 
 
 /*******************************************************************************
  ** Standard/re-usable post-insert customizer, for the use case where, when we
  ** do an insert into table "parent", we want a record automatically inserted into
- ** table "child", and there's a foreign key in "parent", pointed at "child"
- ** e.g., named: "parent.childId".
+ ** table "child".  Optionally (based on RelationshipType), there can be a foreign
+ ** key in "parent", pointed at "child".  e.g., named: "parent.childId".
  **
- ** A similar use-case would have the foreign key in the child table - in which case,
- ** we could add a "Type" enum, plus abstract method to get our "Type", then logic
- ** to switch behavior based on type.  See existing type enum, but w/ only 1 case :)
  *******************************************************************************/
 public abstract class ChildInserterPostInsertCustomizer extends AbstractPostInsertCustomizer
 {
    public enum RelationshipType
    {
-      PARENT_POINTS_AT_CHILD
+      PARENT_POINTS_AT_CHILD,
+      CHILD_POINTS_AT_PARENT
    }
 
 
@@ -65,10 +67,17 @@ public abstract class ChildInserterPostInsertCustomizer extends AbstractPostInse
     *******************************************************************************/
    public abstract String getChildTableName();
 
+
+
    /*******************************************************************************
     **
     *******************************************************************************/
-   public abstract String getForeignKeyFieldName();
+   public String getForeignKeyFieldName()
+   {
+      return (null);
+   }
+
+
 
    /*******************************************************************************
     **
@@ -85,7 +94,7 @@ public abstract class ChildInserterPostInsertCustomizer extends AbstractPostInse
    {
       try
       {
-         List<QRecord>  rs               = new ArrayList<>();
+         List<QRecord>  rs               = records;
          List<QRecord>  childrenToInsert = new ArrayList<>();
          QTableMetaData table            = getInsertInput().getTable();
          QTableMetaData childTable       = getInsertInput().getInstance().getTable(getChildTableName());
@@ -94,12 +103,37 @@ public abstract class ChildInserterPostInsertCustomizer extends AbstractPostInse
          // iterate over the inserted records, building a list child records to insert //
          // for ones missing a value in the foreign key field.                         //
          ////////////////////////////////////////////////////////////////////////////////
-         for(QRecord record : records)
+         switch(getRelationshipType())
          {
-            if(record.getValue(getForeignKeyFieldName()) == null)
+            case PARENT_POINTS_AT_CHILD ->
             {
-               childrenToInsert.add(buildChildForRecord(record));
+               String foreignKeyFieldName = getForeignKeyFieldName();
+               try
+               {
+                  table.getField(foreignKeyFieldName);
+               }
+               catch(Exception e)
+               {
+                  throw new QRuntimeException("For RelationshipType.PARENT_POINTS_AT_CHILD, a valid foreignKeyFieldName in the parent table must be given.  "
+                     + "[" + foreignKeyFieldName + "] is not a valid field name in table [" + table.getName() + "]");
+               }
+
+               for(QRecord record : records)
+               {
+                  if(record.getValue(foreignKeyFieldName) == null)
+                  {
+                     childrenToInsert.add(buildChildForRecord(record));
+                  }
+               }
             }
+            case CHILD_POINTS_AT_PARENT ->
+            {
+               for(QRecord record : records)
+               {
+                  childrenToInsert.add(buildChildForRecord(record));
+               }
+            }
+            default -> throw new IllegalStateException("Unexpected value: " + getRelationshipType());
          }
 
          ///////////////////////////////////////////////////////////////////////////////////
@@ -120,38 +154,76 @@ public abstract class ChildInserterPostInsertCustomizer extends AbstractPostInse
          InsertOutput      insertOutput           = new InsertAction().execute(insertInput);
          Iterator<QRecord> insertedRecordIterator = insertOutput.getRecords().iterator();
 
+         /////////////////////////////////////////////////////////////////////////////////
+         // check for any errors when inserting the children, if any errors were found, //
+         // then set a warning in the parent with the details of the problem            //
+         /////////////////////////////////////////////////////////////////////////////////
+
          //////////////////////////////////////////////////////////////////////////////////////////////////////
+         // for the PARENT_POINTS_AT_CHILD relationship type:
          // iterate over the original list of records again - for any that need a child (e.g., are missing   //
          // foreign key), set their foreign key to a newly inserted child's key, and add them to be updated. //
          //////////////////////////////////////////////////////////////////////////////////////////////////////
-         List<QRecord> recordsToUpdate = new ArrayList<>();
-         for(QRecord record : records)
+         switch(getRelationshipType())
          {
-            Serializable primaryKey = record.getValue(table.getPrimaryKeyField());
-            if(record.getValue(getForeignKeyFieldName()) == null)
+            case PARENT_POINTS_AT_CHILD ->
             {
-               Serializable foreignKey = insertedRecordIterator.next().getValue(childTable.getPrimaryKeyField());
-               recordsToUpdate.add(new QRecord().withValue(table.getPrimaryKeyField(), primaryKey).withValue(getForeignKeyFieldName(), foreignKey));
-               record.setValue(getForeignKeyFieldName(), foreignKey);
-               rs.add(record);
+               rs = new ArrayList<>();
+               List<QRecord> recordsToUpdate = new ArrayList<>();
+               for(QRecord record : records)
+               {
+                  Serializable primaryKey = record.getValue(table.getPrimaryKeyField());
+                  if(record.getValue(getForeignKeyFieldName()) == null)
+                  {
+                     ///////////////////////////////////////////////////////////////////////////////////////////////////
+                     // get the corresponding child record, if it has any errors, set that as a warning in the parent //
+                     ///////////////////////////////////////////////////////////////////////////////////////////////////
+                     QRecord childRecord = insertedRecordIterator.next();
+                     if(CollectionUtils.nullSafeHasContents(childRecord.getErrors()))
+                     {
+                        for(QStatusMessage error : childRecord.getErrors())
+                        {
+                           record.addWarning(new QWarningMessage("Error creating child " + childTable.getLabel() + " (" + error.toString() + ")"));
+                        }
+                        rs.add(record);
+                        continue;
+                     }
+
+                     Serializable foreignKey = childRecord.getValue(childTable.getPrimaryKeyField());
+                     recordsToUpdate.add(new QRecord().withValue(table.getPrimaryKeyField(), primaryKey).withValue(getForeignKeyFieldName(), foreignKey));
+                     record.setValue(getForeignKeyFieldName(), foreignKey);
+                     rs.add(record);
+                  }
+                  else
+                  {
+                     rs.add(record);
+                  }
+               }
+
+               ////////////////////////////////////////////////////////////////////////////
+               // update the originally inserted records to reference their new children //
+               ////////////////////////////////////////////////////////////////////////////
+               UpdateInput updateInput = new UpdateInput();
+               updateInput.setTableName(getInsertInput().getTableName());
+               updateInput.setRecords(recordsToUpdate);
+               updateInput.setTransaction(this.insertInput.getTransaction());
+               new UpdateAction().execute(updateInput);
             }
-            else
+            case CHILD_POINTS_AT_PARENT ->
             {
-               rs.add(record);
+               ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+               // todo - some version of looking at the inserted children to confirm that they were inserted, and updating the parents with warnings if they weren't //
+               ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
             }
+            default -> throw new IllegalStateException("Unexpected value: " + getRelationshipType());
          }
 
-         ////////////////////////////////////////////////////////////////////////////
-         // update the originally inserted records to reference their new children //
-         ////////////////////////////////////////////////////////////////////////////
-         UpdateInput updateInput = new UpdateInput();
-         updateInput.setTableName(getInsertInput().getTableName());
-         updateInput.setRecords(recordsToUpdate);
-         updateInput.setTransaction(this.insertInput.getTransaction());
-         new UpdateAction().execute(updateInput);
-
          return (rs);
       }
+      catch(RuntimeException re)
+      {
+         throw (re);
+      }
       catch(Exception e)
       {
          throw new RuntimeException("Error inserting new child records for new parent records", e);

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/RecordCustomizerUtilityInterface.java

@@ -0,0 +1,149 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2023.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.actions.customizers;
+
+
+import java.io.Serializable;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Objects;
+import com.kingsrook.qqq.backend.core.context.QContext;
+import com.kingsrook.qqq.backend.core.logging.QLogger;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+import com.kingsrook.qqq.backend.core.model.statusmessages.BadInputStatusMessage;
+import com.kingsrook.qqq.backend.core.utils.StringUtils;
+import com.kingsrook.qqq.backend.core.utils.ValueUtils;
+import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
+
+
+/*******************************************************************************
+ ** Interface with utility methods that pre insert/update/delete customizers
+ ** may want to use.
+ *******************************************************************************/
+public interface RecordCustomizerUtilityInterface
+{
+   QLogger LOG = QLogger.getLogger(RecordCustomizerUtilityInterface.class);
+
+
+
+   /*******************************************************************************
+    ** Container for an old value and a new value.
+    *******************************************************************************/
+   @SuppressWarnings("checkstyle:MethodName")
+   record Change(Serializable oldValue, Serializable newValue)
+   {
+   }
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   default Map<String, Change> getChanges(String tableName, QRecord oldRecord, QRecord newRecord)
+   {
+      Map<String, Change> rs = new HashMap<>();
+
+      QTableMetaData table = QContext.getQInstance().getTable(tableName);
+      for(Map.Entry<String, Serializable> entry : newRecord.getValues().entrySet())
+      {
+         String       fieldName = entry.getKey();
+         Serializable newValue  = entry.getValue();
+         Serializable oldValue  = oldRecord.getValue(fieldName);
+
+         try
+         {
+            QFieldMetaData field         = table.getField(fieldName);
+            Serializable   newTypedValue = ValueUtils.getValueAsFieldType(field.getType(), newValue);
+            Serializable   oldTypedValue = ValueUtils.getValueAsFieldType(field.getType(), oldValue);
+
+            if(!Objects.equals(oldTypedValue, newTypedValue))
+            {
+               rs.put(fieldName, new Change(oldTypedValue, newTypedValue));
+            }
+         }
+         catch(Exception e)
+         {
+            LOG.info("Error getting a value as field's type", e, logPair("fieldName", fieldName), logPair("oldValue", oldValue), logPair("newValue", newValue));
+         }
+      }
+
+      return (rs);
+   }
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   default void errorIfNoValue(Serializable value, QRecord record, String errorMessage)
+   {
+      errorIf(!StringUtils.hasContent(ValueUtils.getValueAsString(value)), record, errorMessage);
+   }
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   default void errorIfEditedValue(QRecord oldRecord, QRecord newRecord, String fieldName, String errorMessage)
+   {
+      if(newRecord.getValues().containsKey(fieldName))
+      {
+         errorIf(isChangedValue(oldRecord.getValue(fieldName), newRecord.getValue(fieldName)), newRecord, errorMessage);
+      }
+   }
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   default boolean isChangedValue(Serializable oldValue, Serializable newValue)
+   {
+      //////////////////////////////////////////////
+      // todo - probably ... some type "coercion" //
+      //////////////////////////////////////////////
+      return (!Objects.equals(oldValue, newValue));
+   }
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   default void errorIfAnyValue(Serializable value, QRecord record, String errorMessage)
+   {
+      if(StringUtils.hasContent(ValueUtils.getValueAsString(value)))
+      {
+         record.addError(new BadInputStatusMessage(errorMessage));
+      }
+   }
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   default void errorIf(boolean condition, QRecord record, String errorMessage)
+   {
+      if(condition)
+      {
+         record.addError(new BadInputStatusMessage(errorMessage));
+      }
+   }
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/TableCustomizers.java

@@ -26,33 +26,30 @@ package com.kingsrook.qqq.backend.core.actions.customizers;
  ** Enum definition of possible table customizers - "roles" for custom code that
  ** can be applied to tables.
  **
- ** Works with TableCustomizer (singular version of this name) objects, during
- ** instance validation, to provide validation of the referenced code (and to
- ** make such validation from sub-backend-modules possible in the future).
- **
- ** The idea of the 3rd argument here is to provide a way that we can enforce
- ** the type-parameters for the custom code.  E.g., if it's a Function - how
- ** can we check at run-time that the type-params are correct?  We couldn't find
- ** how to do this "reflectively", so we can instead try to run the custom code,
- ** passing it objects of the type that this customizer expects, and a validation
- ** error will raise upon ClassCastException...  This maybe could improve!
  *******************************************************************************/
 public enum TableCustomizers
 {
-   POST_QUERY_RECORD(new TableCustomizer("postQueryRecord", AbstractPostQueryCustomizer.class)),
-   POST_INSERT_RECORD(new TableCustomizer("postInsertRecord", AbstractPostInsertCustomizer.class));
+   POST_QUERY_RECORD("postQueryRecord", AbstractPostQueryCustomizer.class),
+   PRE_INSERT_RECORD("preInsertRecord", AbstractPreInsertCustomizer.class),
+   POST_INSERT_RECORD("postInsertRecord", AbstractPostInsertCustomizer.class),
+   PRE_UPDATE_RECORD("preUpdateRecord", AbstractPreUpdateCustomizer.class),
+   POST_UPDATE_RECORD("postUpdateRecord", AbstractPostUpdateCustomizer.class),
+   PRE_DELETE_RECORD("preDeleteRecord", AbstractPreDeleteCustomizer.class),
+   POST_DELETE_RECORD("postDeleteRecord", AbstractPostDeleteCustomizer.class);
 
 
-   private final TableCustomizer tableCustomizer;
+   private final String   role;
+   private final Class<?> expectedType;
 
 
 
    /*******************************************************************************
     **
     *******************************************************************************/
-   TableCustomizers(TableCustomizer tableCustomizer)
+   TableCustomizers(String role, Class<?> expectedType)
    {
-      this.tableCustomizer = tableCustomizer;
+      this.role = role;
+      this.expectedType = expectedType;
    }
 
 
@@ -65,7 +62,7 @@ public enum TableCustomizers
    {
       for(TableCustomizers value : values())
       {
-         if(value.tableCustomizer.getRole().equals(name))
+         if(value.role.equals(name))
          {
             return (value);
          }
@@ -76,24 +73,23 @@ public enum TableCustomizers
 
 
 
-   /*******************************************************************************
-    ** Getter for tableCustomizer
-    **
-    *******************************************************************************/
-   public TableCustomizer getTableCustomizer()
-   {
-      return tableCustomizer;
-   }
-
-
-
    /*******************************************************************************
     ** get the role from the tableCustomizer
     **
     *******************************************************************************/
    public String getRole()
    {
-      return (tableCustomizer.getRole());
+      return (role);
    }
 
+
+
+   /*******************************************************************************
+    ** Getter for expectedType
+    **
+    *******************************************************************************/
+   public Class<?> getExpectedType()
+   {
+      return expectedType;
+   }
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/dashboard/AbstractHTMLWidgetRenderer.java

@@ -216,7 +216,7 @@ public abstract class AbstractHTMLWidgetRenderer extends AbstractWidgetRenderer
    /*******************************************************************************
     **
     *******************************************************************************/
-   public static String linkTableFilter(RenderWidgetInput input, String tableName, QQueryFilter filter) throws QException
+   public static String linkTableFilter(String tableName, QQueryFilter filter) throws QException
    {
       String tablePath = QContext.getQInstance().getTablePath(tableName);
       if(tablePath == null)
@@ -232,9 +232,9 @@ public abstract class AbstractHTMLWidgetRenderer extends AbstractWidgetRenderer
    /*******************************************************************************
     **
     *******************************************************************************/
-   public static String aHrefTableFilterNoOfRecords(RenderWidgetInput input, String tableName, QQueryFilter filter, Integer noOfRecords, String singularLabel, String pluralLabel) throws QException
+   public static String aHrefTableFilterNoOfRecords(String tableName, QQueryFilter filter, Integer noOfRecords, String singularLabel, String pluralLabel) throws QException
    {
-      return (aHrefTableFilterNoOfRecords(input, tableName, filter, noOfRecords, singularLabel, pluralLabel, false));
+      return (aHrefTableFilterNoOfRecords(tableName, filter, noOfRecords, singularLabel, pluralLabel, false));
    }
 
 
@@ -242,7 +242,7 @@ public abstract class AbstractHTMLWidgetRenderer extends AbstractWidgetRenderer
    /*******************************************************************************
     **
     *******************************************************************************/
-   public static String aHrefTableFilterNoOfRecords(RenderWidgetInput input, String tableName, QQueryFilter filter, Integer noOfRecords, String singularLabel, String pluralLabel, boolean onlyLinkCount) throws QException
+   public static String aHrefTableFilterNoOfRecords(String tableName, QQueryFilter filter, Integer noOfRecords, String singularLabel, String pluralLabel, boolean onlyLinkCount) throws QException
    {
       String plural      = StringUtils.plural(noOfRecords, singularLabel, pluralLabel);
       String countString = QValueFormatter.formatValue(DisplayFormat.COMMAS, noOfRecords);
@@ -253,7 +253,7 @@ public abstract class AbstractHTMLWidgetRenderer extends AbstractWidgetRenderer
          return (countString + displayText);
       }
 
-      String href = linkTableFilter(input, tableName, filter);
+      String href = linkTableFilter(tableName, filter);
       if(onlyLinkCount)
       {
          return ("<a href=\"" + href + "\">" + countString + "</a>" + displayText);
@@ -269,7 +269,7 @@ public abstract class AbstractHTMLWidgetRenderer extends AbstractWidgetRenderer
    /*******************************************************************************
     **
     *******************************************************************************/
-   public static String aHrefViewRecord(RenderWidgetInput input, String tableName, Serializable id, String linkText) throws QException
+   public static String aHrefViewRecord(String tableName, Serializable id, String linkText) throws QException
    {
       String tablePath = QContext.getQInstance().getTablePath(tableName);
       if(tablePath == null)
@@ -277,7 +277,7 @@ public abstract class AbstractHTMLWidgetRenderer extends AbstractWidgetRenderer
          return (linkText);
       }
 
-      return ("<a href=\"" + linkRecordView(input, tableName, id) + "\">" + linkText + "</a>");
+      return ("<a href=\"" + linkRecordView(tableName, id) + "\">" + linkText + "</a>");
    }
 
 
@@ -296,7 +296,7 @@ public abstract class AbstractHTMLWidgetRenderer extends AbstractWidgetRenderer
    /*******************************************************************************
     **
     *******************************************************************************/
-   public static String linkRecordView(AbstractActionInput input, String tableName, Serializable recordId) throws QException
+   public static String linkRecordView(String tableName, Serializable recordId) throws QException
    {
       String tablePath = QContext.getQInstance().getTablePath(tableName);
       if(tablePath == null)

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/dashboard/widgets/AbstractWidgetRenderer.java

@@ -46,6 +46,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleVal
 import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSource;
 import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
 import com.kingsrook.qqq.backend.core.utils.StringUtils;
+import com.kingsrook.qqq.backend.core.utils.collections.MapBuilder;
 
 
 /*******************************************************************************
@@ -123,7 +124,7 @@ public abstract class AbstractWidgetRenderer
          output.getResults().removeIf(pvs -> !exists.add(pvs.getLabel()));
          for(QPossibleValue<?> possibleValue : output.getResults())
          {
-            dropdownOptionList.add(Map.of(
+            dropdownOptionList.add(MapBuilder.of(
                "id", String.valueOf(possibleValue.getId()),
                "label", possibleValue.getLabel()
             ));

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/dashboard/widgets/ChildRecordListRenderer.java

@@ -30,10 +30,13 @@ import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
+import com.kingsrook.qqq.backend.core.actions.tables.CountAction;
 import com.kingsrook.qqq.backend.core.actions.tables.GetAction;
 import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
+import com.kingsrook.qqq.backend.core.actions.values.QValueFormatter;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.exceptions.QNotFoundException;
+import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetOutput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
@@ -72,7 +75,7 @@ public class ChildRecordListRenderer extends AbstractWidgetRenderer
       return (new Builder(new QWidgetMetaData()
          .withName(join.getName())
          .withIsCard(true)
-         .withCodeReference(new QCodeReference(ChildRecordListRenderer.class, null))
+         .withCodeReference(new QCodeReference(ChildRecordListRenderer.class))
          .withType(WidgetType.CHILD_RECORD_LIST.getType())
          .withDefaultValue("joinName", join.getName())));
    }
@@ -158,16 +161,22 @@ public class ChildRecordListRenderer extends AbstractWidgetRenderer
    @Override
    public RenderWidgetOutput render(RenderWidgetInput input) throws QException
    {
-      String        widgetLabel = input.getQueryParams().get("widgetLabel");
-      String        joinName    = input.getQueryParams().get("joinName");
-      QJoinMetaData join        = input.getInstance().getJoin(joinName);
-      String        id          = input.getQueryParams().get("id");
+      String         widgetLabel = input.getQueryParams().get("widgetLabel");
+      String         joinName    = input.getQueryParams().get("joinName");
+      QJoinMetaData  join        = input.getInstance().getJoin(joinName);
+      String         id          = input.getQueryParams().get("id");
+      QTableMetaData leftTable   = input.getInstance().getTable(join.getLeftTable());
+      QTableMetaData rightTable  = input.getInstance().getTable(join.getRightTable());
 
       Integer maxRows = null;
       if(StringUtils.hasContent(input.getQueryParams().get("maxRows")))
       {
          maxRows = ValueUtils.getValueAsInteger(input.getQueryParams().get("maxRows"));
       }
+      else if(input.getWidgetMetaData().getDefaultValues().containsKey("maxRows"))
+      {
+         maxRows = ValueUtils.getValueAsInteger(input.getWidgetMetaData().getDefaultValues().containsKey("maxRows"));
+      }
 
       ////////////////////////////////////////////////////////
       // fetch the record that we're getting children for.  //
@@ -181,8 +190,7 @@ public class ChildRecordListRenderer extends AbstractWidgetRenderer
 
       if(record == null)
       {
-         QTableMetaData table = input.getInstance().getTable(join.getLeftTable());
-         throw (new QNotFoundException("Could not find " + (table == null ? "" : table.getLabel()) + " with primary key " + id));
+         throw (new QNotFoundException("Could not find " + (leftTable == null ? "" : leftTable.getLabel()) + " with primary key " + id));
       }
 
       ////////////////////////////////////////////////////////////////////
@@ -194,20 +202,34 @@ public class ChildRecordListRenderer extends AbstractWidgetRenderer
          filter.addCriteria(new QFilterCriteria(joinOn.getRightField(), QCriteriaOperator.EQUALS, List.of(record.getValue(joinOn.getLeftField()))));
       }
       filter.setOrderBys(join.getOrderBys());
+      filter.setLimit(maxRows);
 
       QueryInput queryInput = new QueryInput();
       queryInput.setTableName(join.getRightTable());
       queryInput.setShouldTranslatePossibleValues(true);
       queryInput.setShouldGenerateDisplayValues(true);
       queryInput.setFilter(filter);
-      queryInput.setLimit(maxRows);
       QueryOutput queryOutput = new QueryAction().execute(queryInput);
 
-      QTableMetaData table       = input.getInstance().getTable(join.getRightTable());
-      String         tablePath   = input.getInstance().getTablePath(table.getName());
-      String         viewAllLink = tablePath == null ? null : (tablePath + "?filter=" + URLEncoder.encode(JsonUtils.toJson(filter), Charset.defaultCharset()));
+      QValueFormatter.setBlobValuesToDownloadUrls(rightTable, queryOutput.getRecords());
 
-      ChildRecordListData widgetData = new ChildRecordListData(widgetLabel, queryOutput, table, tablePath, viewAllLink);
+      int totalRows = queryOutput.getRecords().size();
+      if(maxRows != null && (queryOutput.getRecords().size() == maxRows))
+      {
+         /////////////////////////////////////////////////////////////////////////////////////
+         // if the input said to only do some max, and the # of results we got is that max, //
+         // then do a count query, for displaying 1-n of <count>                            //
+         /////////////////////////////////////////////////////////////////////////////////////
+         CountInput countInput = new CountInput();
+         countInput.setTableName(join.getRightTable());
+         countInput.setFilter(filter);
+         totalRows = new CountAction().execute(countInput).getCount();
+      }
+
+      String tablePath   = input.getInstance().getTablePath(rightTable.getName());
+      String viewAllLink = tablePath == null ? null : (tablePath + "?filter=" + URLEncoder.encode(JsonUtils.toJson(filter), Charset.defaultCharset()));
+
+      ChildRecordListData widgetData = new ChildRecordListData(widgetLabel, queryOutput, rightTable, tablePath, viewAllLink, totalRows);
 
       if(BooleanUtils.isTrue(ValueUtils.getValueAsBoolean(input.getQueryParams().get("canAddChildRecord"))))
       {

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/dashboard/widgets/DateTimeGroupBy.java

@@ -99,7 +99,17 @@ public enum DateTimeGroupBy
    public String getSqlExpression()
    {
       ZoneId sessionOrInstanceZoneId = ValueUtils.getSessionOrInstanceZoneId();
-      String targetTimezone          = sessionOrInstanceZoneId.toString();
+      return (getSqlExpression(sessionOrInstanceZoneId));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public String getSqlExpression(ZoneId targetZoneId)
+   {
+      String targetTimezone = targetZoneId.toString();
 
       if("Z".equals(targetTimezone) || !StringUtils.hasContent(targetTimezone))
       {
@@ -158,7 +168,18 @@ public enum DateTimeGroupBy
     *******************************************************************************/
    public String makeSelectedString(Instant time)
    {
-      ZonedDateTime zoned = time.atZone(ValueUtils.getSessionOrInstanceZoneId());
+      return (makeSelectedString(time, ValueUtils.getSessionOrInstanceZoneId()));
+   }
+
+
+
+   /*******************************************************************************
+    ** Make an Instant into a string that will match what came out of the database's
+    ** DATE_FORMAT() function
+    *******************************************************************************/
+   public String makeSelectedString(Instant time, ZoneId zoneId)
+   {
+      ZonedDateTime zoned = time.atZone(zoneId);
 
       if(this == WEEK)
       {
@@ -182,7 +203,17 @@ public enum DateTimeGroupBy
     *******************************************************************************/
    public String makeHumanString(Instant instant)
    {
-      ZonedDateTime zoned = instant.atZone(ValueUtils.getSessionOrInstanceZoneId());
+      return (makeHumanString(instant, ValueUtils.getSessionOrInstanceZoneId()));
+   }
+
+
+
+   /*******************************************************************************
+    ** Make a string to show to a user
+    *******************************************************************************/
+   public String makeHumanString(Instant instant, ZoneId zoneId)
+   {
+      ZonedDateTime zoned = instant.atZone(zoneId);
       if(this.equals(WEEK))
       {
          DateTimeFormatter dateTimeFormatter = DateTimeFormatter.ofPattern("M'/'d");
@@ -215,25 +246,35 @@ public enum DateTimeGroupBy
    /*******************************************************************************
     **
     *******************************************************************************/
-   @SuppressWarnings("checkstyle:indentation")
    public Instant roundDown(Instant instant)
    {
-      ZonedDateTime zoned = instant.atZone(ValueUtils.getSessionOrInstanceZoneId());
+      return roundDown(instant, ValueUtils.getSessionOrInstanceZoneId());
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @SuppressWarnings("checkstyle:indentation")
+   public Instant roundDown(Instant instant, ZoneId zoneId)
+   {
+      ZonedDateTime zoned = instant.atZone(zoneId);
       return switch(this)
+      {
+         case YEAR -> zoned.with(TemporalAdjusters.firstDayOfYear()).truncatedTo(ChronoUnit.DAYS).toInstant();
+         case MONTH -> zoned.with(TemporalAdjusters.firstDayOfMonth()).truncatedTo(ChronoUnit.DAYS).toInstant();
+         case WEEK ->
          {
-            case YEAR -> zoned.with(TemporalAdjusters.firstDayOfYear()).truncatedTo(ChronoUnit.DAYS).toInstant();
-            case MONTH -> zoned.with(TemporalAdjusters.firstDayOfMonth()).truncatedTo(ChronoUnit.DAYS).toInstant();
-            case WEEK ->
+            while(zoned.get(ChronoField.DAY_OF_WEEK) != DayOfWeek.SUNDAY.getValue())
             {
-               while(zoned.get(ChronoField.DAY_OF_WEEK) != DayOfWeek.SUNDAY.getValue())
-               {
-                  zoned = zoned.minusDays(1);
-               }
-               yield (zoned.truncatedTo(ChronoUnit.DAYS).toInstant());
+               zoned = zoned.minusDays(1);
             }
-            case DAY -> zoned.truncatedTo(ChronoUnit.DAYS).toInstant();
-            case HOUR -> zoned.truncatedTo(ChronoUnit.HOURS).toInstant();
-         };
+            yield (zoned.truncatedTo(ChronoUnit.DAYS).toInstant());
+         }
+         case DAY -> zoned.truncatedTo(ChronoUnit.DAYS).toInstant();
+         case HOUR -> zoned.truncatedTo(ChronoUnit.HOURS).toInstant();
+      };
    }
 
 
@@ -243,7 +284,17 @@ public enum DateTimeGroupBy
     *******************************************************************************/
    public Instant increment(Instant instant)
    {
-      ZonedDateTime zoned = instant.atZone(ValueUtils.getSessionOrInstanceZoneId());
+      return (increment(instant, ValueUtils.getSessionOrInstanceZoneId()));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public Instant increment(Instant instant, ZoneId zoneId)
+   {
+      ZonedDateTime zoned = instant.atZone(zoneId);
       return (zoned.plus(noOfChronoUnitsToAdd, chronoUnitToAdd).toInstant());
    }
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/dashboard/widgets/NoCodeWidgetVelocityUtils.java

@@ -64,14 +64,24 @@ public class NoCodeWidgetVelocityUtils
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public String icon(String iconName, String color)
+   {
+      return String.format("""
+         <span class="material-icons-round notranslate MuiIcon-root MuiIcon-fontSizeInherit" style="color: %s; position: relative; top: 6px;" aria-hidden="true">%s</span>
+         """, color, iconName);
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/
    public String helpIcon()
    {
-      return ("""
-         <span class="material-icons-round notranslate MuiIcon-root MuiIcon-fontSizeInherit" style="color: blue; position: relative; top: 6px;" aria-hidden="true">help_outline</span>
-         """);
+      return (icon("help_outline", "blue"));
    }
 
 
@@ -81,9 +91,7 @@ public class NoCodeWidgetVelocityUtils
     *******************************************************************************/
    public String errorIcon()
    {
-      return ("""
-         <span class="material-icons-round notranslate MuiIcon-root MuiIcon-fontSizeInherit" style="color: red; position: relative; top: 6px;" aria-hidden="true">error_outline</span>
-         """);
+      return (icon("error_outline", "red"));
    }
 
 
@@ -93,9 +101,7 @@ public class NoCodeWidgetVelocityUtils
     *******************************************************************************/
    public String warningIcon()
    {
-      return ("""
-         <span class="material-icons-round notranslate MuiIcon-root MuiIcon-fontSizeInherit" style="color: orange; position: relative; top: 6px;" aria-hidden="true">warning</span>
-         """);
+      return (icon("warning", "orange"));
    }
 
 
@@ -105,9 +111,7 @@ public class NoCodeWidgetVelocityUtils
     *******************************************************************************/
    public String checkIcon()
    {
-      return ("""
-         <span class="material-icons-round notranslate MuiIcon-root MuiIcon-fontSizeInherit" style="color: green; position: relative; top: 6px;" aria-hidden="true">check</span>
-         """);
+      return (icon("check", "green"));
    }
 
 
@@ -117,9 +121,7 @@ public class NoCodeWidgetVelocityUtils
     *******************************************************************************/
    public String pendingIcon()
    {
-      return ("""
-         <span class="material-icons-round notranslate MuiIcon-root MuiIcon-fontSizeInherit" style="color: #0062ff; position: relative; top: 6px;" aria-hidden="true">pending</span>
-         """);
+      return (icon("pending", "#0062ff"));
    }
 
 
@@ -288,7 +290,7 @@ public class NoCodeWidgetVelocityUtils
          WidgetCount  widgetCount = (WidgetCount) context.get(countVariableName + ".source");
          Integer      count       = ValueUtils.getValueAsInteger(context.get(countVariableName));
          QQueryFilter filter      = widgetCount.getEffectiveFilter(input);
-         return (AbstractHTMLWidgetRenderer.aHrefTableFilterNoOfRecords(null, widgetCount.getTableName(), filter, count, singular, plural));
+         return (AbstractHTMLWidgetRenderer.aHrefTableFilterNoOfRecords(widgetCount.getTableName(), filter, count, singular, plural));
       }
       catch(Exception e)
       {

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/dashboard/widgets/ParentWidgetRenderer.java

@@ -60,6 +60,8 @@ public class ParentWidgetRenderer extends AbstractWidgetRenderer
             widgetData.setChildWidgetNameList(metaData.getChildWidgetNameList());
          }
 
+         widgetData.setLayoutType(metaData.getLayoutType());
+
          return (new RenderWidgetOutput(widgetData));
       }
       catch(Exception e)

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/interfaces/AggregateInterface.java

@@ -31,7 +31,7 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateOu
  ** Interface for the Aggregate action.
  **
  *******************************************************************************/
-public interface AggregateInterface
+public interface AggregateInterface extends BaseQueryInterface
 {
    /*******************************************************************************
     **

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/interfaces/BaseQueryInterface.java

@@ -0,0 +1,81 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2023.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.actions.interfaces;
+
+
+import java.time.Instant;
+import com.kingsrook.qqq.backend.core.model.querystats.QueryStat;
+
+
+/*******************************************************************************
+ ** Base class for "query" (e.g., read-operations) action interfaces (query, count, aggregate).
+ ** Initially just here for the QueryStat methods - if we expand those to apply
+ ** to insert/update/delete, well, then rename this maybe to BaseActionInterface?
+ *******************************************************************************/
+public interface BaseQueryInterface
+{
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   default void setQueryStat(QueryStat queryStat)
+   {
+      //////////
+      // noop //
+      //////////
+   }
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   default QueryStat getQueryStat()
+   {
+      return (null);
+   }
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   default void setQueryStatFirstResultTime()
+   {
+      QueryStat queryStat = getQueryStat();
+      if(queryStat != null)
+      {
+         if(queryStat.getFirstResultTimestamp() == null)
+         {
+            queryStat.setFirstResultTimestamp(Instant.now());
+         }
+      }
+   }
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   default void cancelAction()
+   {
+      //////////////////////////////////////////////
+      // initially at least, a noop in base class //
+      //////////////////////////////////////////////
+   }
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/interfaces/CountInterface.java

@@ -31,7 +31,7 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountOutput;
  ** Interface for the Count action.
  **
  *******************************************************************************/
-public interface CountInterface
+public interface CountInterface extends BaseQueryInterface
 {
    /*******************************************************************************
     **

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/interfaces/DeleteInterface.java

@@ -50,4 +50,13 @@ public interface DeleteInterface
       return (false);
    }
 
+   /*******************************************************************************
+    ** Specify whether this particular module's delete action can & should fetch
+    ** records before deleting them, e.g., for audits or "not-found-checks"
+    *******************************************************************************/
+   default boolean supportsPreFetchQuery()
+   {
+      return (true);
+   }
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/interfaces/GetInterface.java

@@ -28,6 +28,7 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetOutput;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.UniqueKey;
+import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
 
 
 /*******************************************************************************
@@ -55,7 +56,7 @@ public interface GetInterface
       {
          QTableMetaData table      = getInput.getTable();
          boolean        foundMatch = false;
-         for(UniqueKey uniqueKey : table.getUniqueKeys())
+         for(UniqueKey uniqueKey : CollectionUtils.nonNullList(table.getUniqueKeys()))
          {
             if(new HashSet<>(uniqueKey.getFieldNames()).equals(getInput.getUniqueKey().keySet()))
             {

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/interfaces/QueryInterface.java

@@ -31,10 +31,11 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput;
  ** Interface for the Query action.
  **
  *******************************************************************************/
-public interface QueryInterface
+public interface QueryInterface extends BaseQueryInterface
 {
    /*******************************************************************************
     **
     *******************************************************************************/
    QueryOutput execute(QueryInput queryInput) throws QException;
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/interfaces/UpdateInterface.java

@@ -37,4 +37,14 @@ public interface UpdateInterface
     **
     *******************************************************************************/
    UpdateOutput execute(UpdateInput updateInput) throws QException;
+
+   /*******************************************************************************
+    ** Specify whether this particular module's update action can & should fetch
+    ** records before updating them, e.g., for audits or "not-found-checks"
+    *******************************************************************************/
+   default boolean supportsPreFetchQuery()
+   {
+      return (true);
+   }
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/metadata/JoinGraph.java

@@ -0,0 +1,336 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2023.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.actions.metadata;
+
+
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Objects;
+import java.util.Set;
+import java.util.TreeSet;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.joins.QJoinMetaData;
+import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
+import com.kingsrook.qqq.backend.core.utils.StringUtils;
+
+
+/*******************************************************************************
+ ** Object to represent the graph of joins in a QQQ Instance.  e.g., all of the
+ ** connections among tables through joins.
+ *******************************************************************************/
+public class JoinGraph
+{
+
+   private Set<Edge> edges = new HashSet<>();
+
+
+
+   /*******************************************************************************
+    ** Graph edge (no graph nodes needed in here)
+    *******************************************************************************/
+   private record Edge(String joinName, String leftTable, String rightTable)
+   {
+   }
+
+
+
+   /*******************************************************************************
+    ** In this class, we are treating joins as non-directional graph edges - so -
+    ** use this class to "normalize" what may otherwise be duplicated joins in the
+    ** qInstance (e.g., A -> B and B -> A -- in the instance, those are valid, but
+    ** in our graph here, we want to consider those the same).
+    *******************************************************************************/
+   private static class NormalizedJoin
+   {
+      private String tableA;
+      private String tableB;
+      private String joinFieldA;
+      private String joinFieldB;
+
+
+
+      /*******************************************************************************
+       **
+       *******************************************************************************/
+      public NormalizedJoin(QJoinMetaData joinMetaData)
+      {
+         boolean needFlip     = false;
+         int     tableCompare = joinMetaData.getLeftTable().compareTo(joinMetaData.getRightTable());
+         if(tableCompare < 0)
+         {
+            needFlip = true;
+         }
+         else if(tableCompare == 0)
+         {
+            int fieldCompare = joinMetaData.getJoinOns().get(0).getLeftField().compareTo(joinMetaData.getJoinOns().get(0).getRightField());
+            if(fieldCompare < 0)
+            {
+               needFlip = true;
+            }
+         }
+
+         if(needFlip)
+         {
+            joinMetaData = joinMetaData.flip();
+         }
+
+         tableA = joinMetaData.getLeftTable();
+         tableB = joinMetaData.getRightTable();
+         joinFieldA = joinMetaData.getJoinOns().get(0).getLeftField();
+         joinFieldB = joinMetaData.getJoinOns().get(0).getRightField();
+      }
+
+
+
+      /*******************************************************************************
+       **
+       *******************************************************************************/
+      @Override
+      public boolean equals(Object o)
+      {
+         if(this == o)
+         {
+            return true;
+         }
+         if(o == null || getClass() != o.getClass())
+         {
+            return false;
+         }
+         NormalizedJoin that = (NormalizedJoin) o;
+         return Objects.equals(tableA, that.tableA) && Objects.equals(tableB, that.tableB) && Objects.equals(joinFieldA, that.joinFieldA) && Objects.equals(joinFieldB, that.joinFieldB);
+      }
+
+
+
+      /*******************************************************************************
+       **
+       *******************************************************************************/
+      @Override
+      public int hashCode()
+      {
+         return Objects.hash(tableA, tableB, joinFieldA, joinFieldB);
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** Constructor
+    **
+    *******************************************************************************/
+   public JoinGraph(QInstance qInstance)
+   {
+      Set<NormalizedJoin> usedJoins = new HashSet<>();
+      for(QJoinMetaData join : CollectionUtils.nonNullMap(qInstance.getJoins()).values())
+      {
+         NormalizedJoin normalizedJoin = new NormalizedJoin(join);
+         if(usedJoins.contains(normalizedJoin))
+         {
+            continue;
+         }
+
+         usedJoins.add(normalizedJoin);
+         edges.add(new Edge(join.getName(), join.getLeftTable(), join.getRightTable()));
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public record JoinConnection(String joinTable, String viaJoinName) implements Comparable<JoinConnection>
+   {
+
+      /*******************************************************************************
+       **
+       *******************************************************************************/
+      @Override
+      public int compareTo(JoinConnection that)
+      {
+         Comparator<JoinConnection> comparator = Comparator.comparing((JoinConnection jc) -> jc.joinTable())
+            .thenComparing((JoinConnection jc) -> jc.viaJoinName());
+         return (comparator.compare(this, that));
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public record JoinConnectionList(List<JoinConnection> list) implements Comparable<JoinConnectionList>
+   {
+
+      /*******************************************************************************
+       **
+       *******************************************************************************/
+      public JoinConnectionList copy()
+      {
+         return new JoinConnectionList(new ArrayList<>(list));
+      }
+
+
+
+      /*******************************************************************************
+       **
+       *******************************************************************************/
+      public int compareTo(JoinConnectionList that)
+      {
+         if(this.equals(that))
+         {
+            return (0);
+         }
+
+         for(int i = 0; i < Math.min(this.list.size(), that.list.size()); i++)
+         {
+            int comp = this.list.get(i).compareTo(that.list.get(i));
+            if(comp != 0)
+            {
+               return (comp);
+            }
+         }
+
+         return (this.list.size() - that.list.size());
+      }
+
+
+
+      /*******************************************************************************
+       **
+       *******************************************************************************/
+      public boolean matchesJoinPath(List<String> joinPath)
+      {
+         if(list.size() != joinPath.size())
+         {
+            return (false);
+         }
+
+         for(int i = 0; i < list.size(); i++)
+         {
+            if(!list.get(i).viaJoinName().equals(joinPath.get(i)))
+            {
+               return (false);
+            }
+         }
+
+         return (true);
+      }
+
+
+
+      /*******************************************************************************
+       **
+       *******************************************************************************/
+      public String getJoinNamesAsString()
+      {
+         return (StringUtils.join(", ", list().stream().map(jc -> jc.viaJoinName()).toList()));
+      }
+
+
+
+      /*******************************************************************************
+       **
+       *******************************************************************************/
+      public List<String> getJoinNamesAsList()
+      {
+         return (list().stream().map(jc -> jc.viaJoinName()).toList());
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public Set<JoinConnectionList> getJoinConnections(String tableName)
+   {
+      Set<JoinConnectionList> rs = new TreeSet<>();
+      doGetJoinConnections(rs, tableName, new ArrayList<>(), new JoinConnectionList(new ArrayList<>()));
+      return (rs);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private void doGetJoinConnections(Set<JoinConnectionList> joinConnections, String tableName, List<String> path, JoinConnectionList connectionList)
+   {
+      for(Edge edge : edges)
+      {
+         if(edge.leftTable.equals(tableName) || edge.rightTable.equals(tableName))
+         {
+            if(path.contains(edge.joinName))
+            {
+               continue;
+            }
+
+            List<String> newPath = new ArrayList<>(path);
+            newPath.add(edge.joinName);
+            if(!joinConnectionsContain(joinConnections, newPath))
+            {
+               String otherTableName = null;
+               if(!edge.leftTable.equals(tableName))
+               {
+                  otherTableName = edge.leftTable;
+               }
+               else if(!edge.rightTable.equals(tableName))
+               {
+                  otherTableName = edge.rightTable;
+               }
+
+               if(otherTableName != null)
+               {
+
+                  JoinConnectionList newConnectionList = connectionList.copy();
+                  JoinConnection     joinConnection    = new JoinConnection(otherTableName, edge.joinName);
+                  newConnectionList.list.add(joinConnection);
+                  joinConnections.add(newConnectionList);
+                  doGetJoinConnections(joinConnections, otherTableName, new ArrayList<>(newPath), newConnectionList);
+               }
+            }
+         }
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private boolean joinConnectionsContain(Set<JoinConnectionList> joinPaths, List<String> newPath)
+   {
+      for(JoinConnectionList joinConnections : joinPaths)
+      {
+         List<String> joinConnectionJoins = joinConnections.list.stream().map(jc -> jc.viaJoinName).toList();
+         if(joinConnectionJoins.equals(newPath))
+         {
+            return (true);
+         }
+      }
+      return (false);
+   }
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/metadata/MetaDataAction.java

@@ -83,11 +83,14 @@ public class MetaDataAction
          }
 
          QBackendMetaData backendForTable = metaDataInput.getInstance().getBackendForTable(tableName);
-         tables.put(tableName, new QFrontendTableMetaData(metaDataInput, backendForTable, table, false));
+         tables.put(tableName, new QFrontendTableMetaData(metaDataInput, backendForTable, table, false, false));
          treeNodes.put(tableName, new AppTreeNode(table));
       }
       metaDataOutput.setTables(tables);
 
+      // addJoinsToTables(tables);
+      // addJoinedTablesToTables(tables);
+
       ////////////////////////////////////////
       // map processes to frontend metadata //
       ////////////////////////////////////////

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/metadata/TableMetaDataAction.java

@@ -54,7 +54,7 @@ public class TableMetaDataAction
          throw (new QNotFoundException("Table [" + tableMetaDataInput.getTableName() + "] was not found."));
       }
       QBackendMetaData backendForTable = tableMetaDataInput.getInstance().getBackendForTable(table.getName());
-      tableMetaDataOutput.setTable(new QFrontendTableMetaData(tableMetaDataInput, backendForTable, table, true));
+      tableMetaDataOutput.setTable(new QFrontendTableMetaData(tableMetaDataInput, backendForTable, table, true, true));
 
       // todo post-customization - can do whatever w/ the result if you want
 

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/processes/RunProcessAction.java

@@ -35,6 +35,7 @@ import com.kingsrook.qqq.backend.core.actions.dashboard.widgets.NoCodeWidgetRend
 import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
 import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
 import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction;
+import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.logging.QLogger;
 import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessState;
@@ -50,6 +51,7 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.processes.NoCodeWidgetFrontendComponentMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData;
@@ -57,6 +59,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendComponen
 import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendStepMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.processes.QStepMetaData;
+import com.kingsrook.qqq.backend.core.model.session.QSession;
 import com.kingsrook.qqq.backend.core.processes.implementations.basepull.BasepullConfiguration;
 import com.kingsrook.qqq.backend.core.state.InMemoryStateProvider;
 import com.kingsrook.qqq.backend.core.state.StateProviderInterface;
@@ -483,6 +486,35 @@ public class RunProcessAction
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   protected String determineBasepullKeyValue(QProcessMetaData process, BasepullConfiguration basepullConfiguration) throws QException
+   {
+      String basepullKeyValue = (basepullConfiguration.getKeyValue() != null) ? basepullConfiguration.getKeyValue() : process.getName();
+
+      //////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      // if backend specifies that it uses variants, look for that data in the session and append to our basepull key //
+      //////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      if(process.getSchedule() != null && process.getSchedule().getVariantBackend() != null)
+      {
+         QSession         session         = QContext.getQSession();
+         QBackendMetaData backendMetaData = QContext.getQInstance().getBackend(process.getSchedule().getVariantBackend());
+         if(session.getBackendVariants() == null || !session.getBackendVariants().containsKey(backendMetaData.getVariantOptionsTableTypeValue()))
+         {
+            LOG.info("Could not find Backend Variant information for Backend '" + backendMetaData.getName() + "'");
+         }
+         else
+         {
+            basepullKeyValue += "-" + session.getBackendVariants().get(backendMetaData.getVariantOptionsTableTypeValue());
+         }
+      }
+
+      return (basepullKeyValue);
+   }
+
+
+
    /*******************************************************************************
     ** Insert or update the last runtime value for this basepull into the backend.
     *******************************************************************************/
@@ -491,7 +523,7 @@ public class RunProcessAction
       String basepullTableName            = basepullConfiguration.getTableName();
       String basepullKeyFieldName         = basepullConfiguration.getKeyField();
       String basepullLastRunTimeFieldName = basepullConfiguration.getLastRunTimeFieldName();
-      String basepullKeyValue             = (basepullConfiguration.getKeyValue() != null) ? basepullConfiguration.getKeyValue() : process.getName();
+      String basepullKeyValue             = determineBasepullKeyValue(process, basepullConfiguration);
 
       ///////////////////////////////////////
       // get the stored basepull timestamp //
@@ -571,7 +603,7 @@ public class RunProcessAction
       String  basepullKeyFieldName                 = basepullConfiguration.getKeyField();
       String  basepullLastRunTimeFieldName         = basepullConfiguration.getLastRunTimeFieldName();
       Integer basepullHoursBackForInitialTimestamp = basepullConfiguration.getHoursBackForInitialTimestamp();
-      String  basepullKeyValue                     = (basepullConfiguration.getKeyValue() != null) ? basepullConfiguration.getKeyValue() : process.getName();
+      String  basepullKeyValue                     = determineBasepullKeyValue(process, basepullConfiguration);
 
       ///////////////////////////////////////
       // get the stored basepull timestamp //

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/BufferedRecordPipe.java

@@ -24,6 +24,7 @@ package com.kingsrook.qqq.backend.core.actions.reporting;
 
 import java.util.ArrayList;
 import java.util.List;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
 
 
@@ -63,7 +64,7 @@ public class BufferedRecordPipe extends RecordPipe
     **
     *******************************************************************************/
    @Override
-   public void addRecord(QRecord record)
+   public void addRecord(QRecord record) throws QException
    {
       buffer.add(record);
       if(buffer.size() >= bufferSize)
@@ -78,7 +79,7 @@ public class BufferedRecordPipe extends RecordPipe
    /*******************************************************************************
     **
     *******************************************************************************/
-   public void finalFlush()
+   public void finalFlush() throws QException
    {
       if(!buffer.isEmpty())
       {

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/ExportAction.java

@@ -23,8 +23,12 @@ package com.kingsrook.qqq.backend.core.actions.reporting;
 
 
 import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
 import java.util.List;
+import java.util.Map;
 import java.util.Optional;
+import java.util.Set;
 import java.util.concurrent.TimeUnit;
 import com.kingsrook.qqq.backend.core.actions.ActionHelper;
 import com.kingsrook.qqq.backend.core.actions.async.AsyncJobManager;
@@ -32,6 +36,7 @@ import com.kingsrook.qqq.backend.core.actions.async.AsyncJobState;
 import com.kingsrook.qqq.backend.core.actions.async.AsyncJobStatus;
 import com.kingsrook.qqq.backend.core.actions.interfaces.CountInterface;
 import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
+import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.exceptions.QReportingException;
 import com.kingsrook.qqq.backend.core.exceptions.QUserFacingException;
@@ -41,10 +46,13 @@ import com.kingsrook.qqq.backend.core.model.actions.reporting.ExportOutput;
 import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportFormat;
 import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountOutput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryJoin;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.ExposedJoin;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleDispatcher;
 import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleInterface;
@@ -95,15 +103,25 @@ public class ExportAction
       ///////////////////////////////////
       if(CollectionUtils.nullSafeHasContents(exportInput.getFieldNames()))
       {
-         QTableMetaData table         = exportInput.getTable();
-         List<String>   badFieldNames = new ArrayList<>();
+         QTableMetaData              table        = exportInput.getTable();
+         Map<String, QTableMetaData> joinTableMap = getJoinTableMap(table);
+
+         List<String> badFieldNames = new ArrayList<>();
          for(String fieldName : exportInput.getFieldNames())
          {
             try
             {
-               table.getField(fieldName);
+               if(fieldName.contains("."))
+               {
+                  String[] parts = fieldName.split("\\.", 2);
+                  joinTableMap.get(parts[0]).getField(parts[1]);
+               }
+               else
+               {
+                  table.getField(fieldName);
+               }
             }
-            catch(IllegalArgumentException iae)
+            catch(Exception e)
             {
                badFieldNames.add(fieldName);
             }
@@ -128,6 +146,21 @@ public class ExportAction
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private static Map<String, QTableMetaData> getJoinTableMap(QTableMetaData table)
+   {
+      Map<String, QTableMetaData> joinTableMap = new HashMap<>();
+      for(ExposedJoin exposedJoin : CollectionUtils.nonNullList(table.getExposedJoins()))
+      {
+         joinTableMap.put(exposedJoin.getJoinTable(), QContext.getQInstance().getTable(exposedJoin.getJoinTable()));
+      }
+      return joinTableMap;
+   }
+
+
+
    /*******************************************************************************
     ** Run the report.
     *******************************************************************************/
@@ -151,7 +184,53 @@ public class ExportAction
       QueryInput  queryInput  = new QueryInput();
       queryInput.setTableName(exportInput.getTableName());
       queryInput.setFilter(exportInput.getQueryFilter());
-      queryInput.setLimit(exportInput.getLimit());
+
+      List<QueryJoin> queryJoins     = new ArrayList<>();
+      Set<String>     addedJoinNames = new HashSet<>();
+      if(CollectionUtils.nullSafeHasContents(exportInput.getFieldNames()))
+      {
+         for(String fieldName : exportInput.getFieldNames())
+         {
+            if(fieldName.contains("."))
+            {
+               String[] parts         = fieldName.split("\\.", 2);
+               String   joinTableName = parts[0];
+               if(!addedJoinNames.contains(joinTableName))
+               {
+                  QueryJoin queryJoin = new QueryJoin(joinTableName).withType(QueryJoin.Type.LEFT).withSelect(true);
+                  queryJoins.add(queryJoin);
+
+                  /////////////////////////////////////////////////////////////////////////////////////////////
+                  // in at least some cases, we need to let the queryJoin know what join-meta-data to use... //
+                  // This code basically mirrors what QFMD is doing right now, so it's better -              //
+                  // but shouldn't all of this just be in JoinsContext?  it does some of this...             //
+                  /////////////////////////////////////////////////////////////////////////////////////////////
+                  QTableMetaData        table               = exportInput.getTable();
+                  Optional<ExposedJoin> exposedJoinOptional = CollectionUtils.nonNullList(table.getExposedJoins()).stream().filter(ej -> ej.getJoinTable().equals(joinTableName)).findFirst();
+                  if(exposedJoinOptional.isEmpty())
+                  {
+                     throw (new QException("Could not find exposed join between base table " + table.getName() + " and requested join table " + joinTableName));
+                  }
+                  ExposedJoin exposedJoin = exposedJoinOptional.get();
+
+                  if(exposedJoin.getJoinPath().size() == 1)
+                  {
+                     queryJoin.setJoinMetaData(QContext.getQInstance().getJoin(exposedJoin.getJoinPath().get(exposedJoin.getJoinPath().size() - 1)));
+                  }
+
+                  addedJoinNames.add(joinTableName);
+               }
+            }
+         }
+      }
+
+      queryInput.setQueryJoins(queryJoins);
+
+      if(queryInput.getFilter() == null)
+      {
+         queryInput.setFilter(new QQueryFilter());
+      }
+      queryInput.getFilter().setLimit(exportInput.getLimit());
       queryInput.setShouldTranslatePossibleValues(true);
 
       /////////////////////////////////////////////////////////////////
@@ -298,24 +377,51 @@ public class ExportAction
     *******************************************************************************/
    private List<QFieldMetaData> getFields(ExportInput exportInput)
    {
+      QTableMetaData              table        = exportInput.getTable();
+      Map<String, QTableMetaData> joinTableMap = getJoinTableMap(table);
+
       List<QFieldMetaData> fieldList;
-      QTableMetaData       table = exportInput.getTable();
       if(exportInput.getFieldNames() != null)
       {
-         fieldList = exportInput.getFieldNames().stream().map(table::getField).toList();
+         fieldList = new ArrayList<>();
+         for(String fieldName : exportInput.getFieldNames())
+         {
+            if(fieldName.contains("."))
+            {
+               String[]       parts     = fieldName.split("\\.", 2);
+               QTableMetaData joinTable = joinTableMap.get(parts[0]);
+               QFieldMetaData field     = joinTable.getField(parts[1]).clone();
+               field.setName(fieldName);
+               field.setLabel(joinTable.getLabel() + ": " + field.getLabel());
+               fieldList.add(field);
+            }
+            else
+            {
+               fieldList.add(table.getField(fieldName));
+            }
+         }
       }
       else
       {
          fieldList = new ArrayList<>(table.getFields().values());
       }
 
-      //////////////////////////////////////////
-      // add fields for possible value labels //
-      //////////////////////////////////////////
       List<QFieldMetaData> returnList = new ArrayList<>();
       for(QFieldMetaData field : fieldList)
       {
+         /////////////////////////////////////////////////////////////////////////////////////////
+         // skip heavy fields.  they aren't fetched, and we generally think we don't want them. //
+         /////////////////////////////////////////////////////////////////////////////////////////
+         if(field.getIsHeavy())
+         {
+            continue;
+         }
+
          returnList.add(field);
+
+         //////////////////////////////////////////
+         // add fields for possible value labels //
+         //////////////////////////////////////////
          if(StringUtils.hasContent(field.getPossibleValueSourceName()))
          {
             returnList.add(new QFieldMetaData(field.getName() + ":possibleValueLabel", QFieldType.STRING).withLabel(field.getLabel() + " Name"));

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/RecordPipe.java

@@ -26,10 +26,11 @@ import java.util.ArrayList;
 import java.util.List;
 import java.util.concurrent.ArrayBlockingQueue;
 import java.util.concurrent.TimeUnit;
-import java.util.function.Consumer;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.logging.QLogger;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
 import com.kingsrook.qqq.backend.core.utils.SleepUtils;
+import com.kingsrook.qqq.backend.core.utils.lambdas.UnsafeConsumer;
 
 
 /*******************************************************************************
@@ -47,7 +48,7 @@ public class RecordPipe
 
    private boolean isTerminated = false;
 
-   private Consumer<List<QRecord>> postRecordActions = null;
+   private UnsafeConsumer<List<QRecord>, QException> postRecordActions = null;
 
    /////////////////////////////////////
    // See usage below for explanation //
@@ -93,7 +94,7 @@ public class RecordPipe
    /*******************************************************************************
     ** Add a record to the pipe.  Will block if the pipe is full.  Will noop if pipe is terminated.
     *******************************************************************************/
-   public void addRecord(QRecord record)
+   public void addRecord(QRecord record) throws QException
    {
       if(isTerminated)
       {
@@ -109,7 +110,7 @@ public class RecordPipe
          // (which we'll create as a field in this class, to avoid always re-constructing) //
          ////////////////////////////////////////////////////////////////////////////////////
          singleRecordListForPostRecordActions.add(record);
-         postRecordActions.accept(singleRecordListForPostRecordActions);
+         postRecordActions.run(singleRecordListForPostRecordActions);
          record = singleRecordListForPostRecordActions.remove(0);
       }
 
@@ -152,11 +153,11 @@ public class RecordPipe
    /*******************************************************************************
     ** Add a list of records to the pipe.  Will block if the pipe is full.  Will noop if pipe is terminated.
     *******************************************************************************/
-   public void addRecords(List<QRecord> records)
+   public void addRecords(List<QRecord> records) throws QException
    {
       if(postRecordActions != null)
       {
-         postRecordActions.accept(records);
+         postRecordActions.run(records);
       }
 
       //////////////////////////////////////////////////////////////////////////////////////////////////
@@ -207,7 +208,7 @@ public class RecordPipe
    /*******************************************************************************
     **
     *******************************************************************************/
-   public void setPostRecordActions(Consumer<List<QRecord>> postRecordActions)
+   public void setPostRecordActions(UnsafeConsumer<List<QRecord>, QException> postRecordActions)
    {
       this.postRecordActions = postRecordActions;
    }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/RecordPipeBufferedWrapper.java

@@ -0,0 +1,79 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.actions.reporting;
+
+
+import java.util.List;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+
+
+/*******************************************************************************
+ ** Subclass of BufferedRecordPipe, which ultimately sends records down to an
+ ** original RecordPipe.
+ **
+ ** Meant to be used where: someone passed in a RecordPipe (so they have a reference
+ ** to it, and they are waiting to read from it), but the producer knows that
+ ** it will be better to buffer the records, so they want to use a buffered pipe
+ ** (but they still need the records to end up in the original pipe - thus -
+ ** it gets wrapped by an object of this class).
+ *******************************************************************************/
+public class RecordPipeBufferedWrapper extends BufferedRecordPipe
+{
+   private RecordPipe wrappedPipe;
+
+
+
+   /*******************************************************************************
+    ** Constructor - uses default buffer size
+    **
+    *******************************************************************************/
+   public RecordPipeBufferedWrapper(RecordPipe wrappedPipe)
+   {
+      this.wrappedPipe = wrappedPipe;
+   }
+
+
+
+   /*******************************************************************************
+    ** Constructor - customize buffer size.
+    **
+    *******************************************************************************/
+   public RecordPipeBufferedWrapper(Integer bufferSize, RecordPipe wrappedPipe)
+   {
+      super(bufferSize);
+      this.wrappedPipe = wrappedPipe;
+   }
+
+
+
+   /*******************************************************************************
+    ** when it's time to actually add records into the pipe, actually add them
+    ** into the wrapped pipe!
+    *******************************************************************************/
+   @Override
+   public void addRecords(List<QRecord> records) throws QException
+   {
+      wrappedPipe.addRecords(records);
+   }
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/scripts/AssociatedScriptContextPrimerInterface.java

@@ -0,0 +1,41 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.actions.scripts;
+
+
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.actions.scripts.ExecuteCodeInput;
+import com.kingsrook.qqq.backend.core.model.scripts.ScriptRevision;
+
+
+/*******************************************************************************
+ **
+ *******************************************************************************/
+public interface AssociatedScriptContextPrimerInterface
+{
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   void primeContext(ExecuteCodeInput executeCodeInput, ScriptRevision scriptRevision) throws QException;
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/scripts/ExecuteCodeAction.java

@@ -23,15 +23,36 @@ package com.kingsrook.qqq.backend.core.actions.scripts;
 
 
 import java.io.Serializable;
+import java.util.ArrayList;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
+import java.util.Objects;
 import com.kingsrook.qqq.backend.core.actions.scripts.logging.Log4jCodeExecutionLogger;
 import com.kingsrook.qqq.backend.core.actions.scripts.logging.QCodeExecutionLoggerInterface;
+import com.kingsrook.qqq.backend.core.actions.scripts.logging.ScriptExecutionLoggerInterface;
+import com.kingsrook.qqq.backend.core.actions.scripts.logging.StoreScriptLogAndScriptLogLineExecutionLogger;
+import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
+import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.exceptions.QCodeException;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.logging.QLogger;
+import com.kingsrook.qqq.backend.core.model.actions.scripts.AbstractRunScriptInput;
 import com.kingsrook.qqq.backend.core.model.actions.scripts.ExecuteCodeInput;
 import com.kingsrook.qqq.backend.core.model.actions.scripts.ExecuteCodeOutput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
 import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
+import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeType;
+import com.kingsrook.qqq.backend.core.model.scripts.ScriptRevision;
+import com.kingsrook.qqq.backend.core.model.scripts.ScriptRevisionFile;
+import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
+import com.kingsrook.qqq.backend.core.utils.ObjectUtils;
+import com.kingsrook.qqq.backend.core.utils.StringUtils;
 
 
 /*******************************************************************************
@@ -49,6 +70,9 @@ import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
  *******************************************************************************/
 public class ExecuteCodeAction
 {
+   private static final QLogger LOG = QLogger.getLogger(ExecuteCodeAction.class);
+
+
 
    /*******************************************************************************
     **
@@ -68,10 +92,10 @@ public class ExecuteCodeAction
       try
       {
          String languageExecutor = switch(codeReference.getCodeType())
-            {
-               case JAVA -> "com.kingsrook.qqq.backend.core.actions.scripts.QJavaExecutor";
-               case JAVA_SCRIPT -> "com.kingsrook.qqq.languages.javascript.QJavaScriptExecutor";
-            };
+         {
+            case JAVA -> "com.kingsrook.qqq.backend.core.actions.scripts.QJavaExecutor";
+            case JAVA_SCRIPT -> "com.kingsrook.qqq.languages.javascript.QJavaScriptExecutor";
+         };
 
          @SuppressWarnings("unchecked")
          Class<? extends QCodeExecutor> executorClass = (Class<? extends QCodeExecutor>) Class.forName(languageExecutor);
@@ -90,6 +114,22 @@ public class ExecuteCodeAction
             context.putAll(input.getInput());
          }
 
+         //////////////////////////////////////////
+         // safely always set the deploymentMode //
+         //////////////////////////////////////////
+         context.put("deploymentMode", ObjectUtils.tryAndRequireNonNullElse(() -> QContext.getQInstance().getDeploymentMode(), null));
+
+         /////////////////////////////////////////////////////////////////////////////////
+         // set the qCodeExecutor into any context objects which are QCodeExecutorAware //
+         /////////////////////////////////////////////////////////////////////////////////
+         for(Serializable value : context.values())
+         {
+            if(value instanceof QCodeExecutorAware qCodeExecutorAware)
+            {
+               qCodeExecutorAware.setQCodeExecutor(qCodeExecutor);
+            }
+         }
+
          Serializable codeOutput = qCodeExecutor.execute(codeReference, context, executionLogger);
          output.setOutput(codeOutput);
          executionLogger.acceptExecutionEnd(codeOutput);
@@ -108,6 +148,156 @@ public class ExecuteCodeAction
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public static ExecuteCodeInput setupExecuteCodeInput(AbstractRunScriptInput<?> input, ScriptRevision scriptRevision) throws QException
+   {
+      return setupExecuteCodeInput(input, scriptRevision, null);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public static ExecuteCodeInput setupExecuteCodeInput(AbstractRunScriptInput<?> input, ScriptRevision scriptRevision, String fileName) throws QException
+   {
+      ExecuteCodeInput executeCodeInput = new ExecuteCodeInput();
+      executeCodeInput.setInput(new HashMap<>(Objects.requireNonNullElseGet(input.getInputValues(), HashMap::new)));
+      executeCodeInput.setContext(new HashMap<>());
+
+      Map<String, Serializable> context = executeCodeInput.getContext();
+      if(input.getOutputObject() != null)
+      {
+         context.put("output", input.getOutputObject());
+      }
+
+      if(input.getScriptUtils() != null)
+      {
+         context.put("scriptUtils", input.getScriptUtils());
+      }
+
+      if(CollectionUtils.nullSafeIsEmpty(scriptRevision.getFiles()))
+      {
+         QueryInput queryInput = new QueryInput();
+         queryInput.setTableName(ScriptRevisionFile.TABLE_NAME);
+         queryInput.setFilter(new QQueryFilter(new QFilterCriteria("scriptRevisionId", QCriteriaOperator.EQUALS, scriptRevision.getId())));
+         QueryOutput queryOutput = new QueryAction().execute(queryInput);
+
+         scriptRevision.setFiles(new ArrayList<>());
+         for(QRecord record : queryOutput.getRecords())
+         {
+            scriptRevision.getFiles().add(new ScriptRevisionFile(record));
+         }
+      }
+
+      List<ScriptRevisionFile> files = scriptRevision.getFiles();
+      if(files == null || files.isEmpty())
+      {
+         throw (new QException("Script Revision " + scriptRevision.getId() + " had more than 1 associated ScriptRevisionFile (and the name to use was not specified)."));
+      }
+      else
+      {
+         String contents = null;
+         if(fileName == null || files.size() == 1)
+         {
+            contents = files.get(0).getContents();
+         }
+         else
+         {
+            for(ScriptRevisionFile file : files)
+            {
+               if(file.getFileName().equals(fileName))
+               {
+                  contents = file.getContents();
+               }
+            }
+            if(contents == null)
+            {
+               throw (new QException("Could not find file named " + fileName + " for Script Revision " + scriptRevision.getId()));
+            }
+         }
+
+         executeCodeInput.setCodeReference(new QCodeReference().withInlineCode(contents).withCodeType(QCodeType.JAVA_SCRIPT)); // todo - code type as attribute of script!!
+      }
+
+      ExecuteCodeAction.addApiUtilityToContext(context, scriptRevision);
+      context.put("qqq", new QqqScriptUtils());
+      ExecuteCodeAction.setExecutionLoggerInExecuteCodeInput(input, scriptRevision, executeCodeInput);
+
+      return (executeCodeInput);
+   }
+
+
+
+   /*******************************************************************************
+    ** Try to (dynamically) load the ApiScriptUtils object from the api middleware
+    ** module -- in case the runtime doesn't have that module deployed (e.g, not in
+    ** the project pom).
+    *******************************************************************************/
+   public static void addApiUtilityToContext(Map<String, Serializable> context, ScriptRevision scriptRevision)
+   {
+      addApiUtilityToContext(context, scriptRevision.getApiName(), scriptRevision.getApiVersion());
+   }
+
+
+
+   /*******************************************************************************
+    ** Try to (dynamically) load the ApiScriptUtils object from the api middleware
+    ** module -- in case the runtime doesn't have that module deployed (e.g, not in
+    ** the project pom).
+    *******************************************************************************/
+   public static void addApiUtilityToContext(Map<String, Serializable> context, String apiName, String apiVersion)
+   {
+      if(!StringUtils.hasContent(apiName) || !StringUtils.hasContent(apiVersion))
+      {
+         return;
+      }
+
+      try
+      {
+         Class<?> apiScriptUtilsClass  = Class.forName("com.kingsrook.qqq.api.utils.ApiScriptUtils");
+         Object   apiScriptUtilsObject = apiScriptUtilsClass.getConstructor(String.class, String.class).newInstance(apiName, apiVersion);
+         context.put("api", (Serializable) apiScriptUtilsObject);
+      }
+      catch(ClassNotFoundException e)
+      {
+         /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+         // this is the only exception we're kinda expecting here - so catch for it specifically, and just log.trace - others, warn //
+         /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+         LOG.trace("Couldn't load ApiScriptUtils class - qqq-middleware-api not on the classpath?");
+      }
+      catch(Exception e)
+      {
+         LOG.warn("Error adding api utility to script context", e);
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private static void setExecutionLoggerInExecuteCodeInput(AbstractRunScriptInput<?> input, ScriptRevision scriptRevision, ExecuteCodeInput executeCodeInput)
+   {
+      /////////////////////////////////////////////////////////////////////////////////////////////////
+      // let caller supply a logger, or by default use StoreScriptLogAndScriptLogLineExecutionLogger //
+      /////////////////////////////////////////////////////////////////////////////////////////////////
+      QCodeExecutionLoggerInterface executionLogger = Objects.requireNonNullElseGet(input.getLogger(), () -> new StoreScriptLogAndScriptLogLineExecutionLogger(scriptRevision.getScriptId(), scriptRevision.getId()));
+      executeCodeInput.setExecutionLogger(executionLogger);
+      if(executionLogger instanceof ScriptExecutionLoggerInterface scriptExecutionLoggerInterface)
+      {
+         ////////////////////////////////////////////////////////////////////////////////////////////////////
+         // if logger is aware of scripts (as opposed to a generic CodeExecution logger), give it the ids. //
+         ////////////////////////////////////////////////////////////////////////////////////////////////////
+         scriptExecutionLoggerInterface.setScriptId(scriptRevision.getScriptId());
+         scriptExecutionLoggerInterface.setScriptRevisionId(scriptRevision.getId());
+      }
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/scripts/QCodeExecutor.java

@@ -41,4 +41,24 @@ public interface QCodeExecutor
     *******************************************************************************/
    Serializable execute(QCodeReference codeReference, Map<String, Serializable> inputContext, QCodeExecutionLoggerInterface executionLogger) throws QCodeException;
 
+   /*******************************************************************************
+    ** Process an object from the script's language/runtime into a (more) native java object.
+    ** e.g., a Nashorn ScriptObjectMirror will end up as a "primitive", or a List or Map of such
+    **
+    *******************************************************************************/
+   default Object convertObjectToJava(Object object) throws QCodeException
+   {
+      return (object);
+   }
+
+   /*******************************************************************************
+    ** Convert a native java object into one for the script's language/runtime.
+    ** e.g., a java Instant to a Nashorn Date
+    **
+    *******************************************************************************/
+   default Object convertJavaObject(Object object, Object requestedTypeHint) throws QCodeException
+   {
+      return (object);
+   }
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/scripts/QCodeExecutorAware.java

@@ -0,0 +1,36 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2023.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.actions.scripts;
+
+
+/*******************************************************************************
+ ** Interface for classes that can accept a QCodeExecutor object via a setter.
+ *******************************************************************************/
+public interface QCodeExecutorAware
+{
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   void setQCodeExecutor(QCodeExecutor qCodeExecutor);
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/scripts/QqqScriptUtils.java

@@ -47,8 +47,17 @@ import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 /*******************************************************************************
  ** Object made available to scripts for access to qqq api (e.g., query, insert,
  ** etc, plus object constructors).
+ **
+ ** Before scripts knew about the API, this class made sense and was used.
+ ** But, when scripts gained knowledge of the API, then it felt like this class could
+ ** be deleted... but, what about, a QQQ deployment without the API module...
+ ** In that case, we might still want this class... think about it.
+ **
+ ** And/Or - it turns out - sometimes using QQQ directly is "better" (?) than using
+ ** an api - so - this object may be available for other use cases (e.g., getting
+ ** a record's backendDetails (e.g., for full json from a source backend api)).
  *******************************************************************************/
-public class ScriptApi implements Serializable
+public class QqqScriptUtils implements Serializable
 {
 
    /*******************************************************************************

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/scripts/RecordScriptTestInterface.java

@@ -0,0 +1,159 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2023.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.actions.scripts;
+
+
+import java.io.Serializable;
+import java.util.Collections;
+import java.util.List;
+import com.kingsrook.qqq.backend.core.actions.scripts.logging.BuildScriptLogAndScriptLogLineExecutionLogger;
+import com.kingsrook.qqq.backend.core.actions.tables.GetAction;
+import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
+import com.kingsrook.qqq.backend.core.context.QContext;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.actions.scripts.ExecuteCodeInput;
+import com.kingsrook.qqq.backend.core.model.actions.scripts.RunAdHocRecordScriptInput;
+import com.kingsrook.qqq.backend.core.model.actions.scripts.RunAdHocRecordScriptOutput;
+import com.kingsrook.qqq.backend.core.model.actions.scripts.TestScriptInput;
+import com.kingsrook.qqq.backend.core.model.actions.scripts.TestScriptOutput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.metadata.code.AdHocScriptCodeReference;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+import com.kingsrook.qqq.backend.core.model.scripts.Script;
+import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
+import com.kingsrook.qqq.backend.core.utils.StringUtils;
+import com.kingsrook.qqq.backend.core.utils.ValueUtils;
+
+
+/*******************************************************************************
+ **
+ *******************************************************************************/
+public class RecordScriptTestInterface implements TestScriptActionInterface
+{
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public void setupTestScriptInput(TestScriptInput testScriptInput, ExecuteCodeInput executeCodeInput) throws QException
+   {
+
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public void execute(TestScriptInput input, TestScriptOutput output) throws QException
+   {
+      try
+      {
+         Serializable scriptId = input.getInputValues().get("scriptId");
+         QRecord      script   = new GetAction().executeForRecord(new GetInput(Script.TABLE_NAME).withPrimaryKey(scriptId));
+
+         //////////////////////////////////////////////
+         // look up the records being tested against //
+         //////////////////////////////////////////////
+         String         tableName = script.getValueString("tableName");
+         QTableMetaData table     = QContext.getQInstance().getTable(tableName);
+         if(table == null)
+         {
+            throw (new QException("Could not find table [" + tableName + "] for script"));
+         }
+
+         String recordPrimaryKeyList = ValueUtils.getValueAsString(input.getInputValues().get("recordPrimaryKeyList"));
+         if(!StringUtils.hasContent(recordPrimaryKeyList))
+         {
+            throw (new QException("Record primary key list was not given."));
+         }
+
+         QueryOutput queryOutput = new QueryAction().execute(new QueryInput(tableName)
+            .withFilter(new QQueryFilter(new QFilterCriteria(table.getPrimaryKeyField(), QCriteriaOperator.IN, recordPrimaryKeyList.split(","))))
+            .withIncludeAssociations(true));
+         if(CollectionUtils.nullSafeIsEmpty(queryOutput.getRecords()))
+         {
+            throw (new QException("No records were found by the given primary keys."));
+         }
+
+         /////////////////////////////
+         // set up & run the action //
+         /////////////////////////////
+         RunAdHocRecordScriptInput runAdHocRecordScriptInput = new RunAdHocRecordScriptInput();
+         runAdHocRecordScriptInput.setRecordList(queryOutput.getRecords());
+
+         BuildScriptLogAndScriptLogLineExecutionLogger executionLogger = new BuildScriptLogAndScriptLogLineExecutionLogger(null, null);
+         runAdHocRecordScriptInput.setLogger(executionLogger);
+
+         runAdHocRecordScriptInput.setTableName(tableName);
+         runAdHocRecordScriptInput.setCodeReference((AdHocScriptCodeReference) input.getCodeReference());
+         RunAdHocRecordScriptOutput runAdHocRecordScriptOutput = new RunAdHocRecordScriptOutput();
+         new RunAdHocRecordScriptAction().run(runAdHocRecordScriptInput, runAdHocRecordScriptOutput);
+
+         /////////////////////////////////
+         // send outputs back to caller //
+         /////////////////////////////////
+         output.setScriptLog(executionLogger.getScriptLog());
+         output.setScriptLogLines(executionLogger.getScriptLogLines());
+         if(runAdHocRecordScriptOutput.getException().isPresent())
+         {
+            output.setException(runAdHocRecordScriptOutput.getException().get());
+         }
+      }
+      catch(QException e)
+      {
+         output.setException(e);
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public List<QFieldMetaData> getTestInputFields()
+   {
+      return (List.of(new QFieldMetaData("recordPrimaryKeyList", QFieldType.STRING).withLabel("Record Primary Key List")));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public List<QFieldMetaData> getTestOutputFields()
+   {
+      return (Collections.emptyList());
+   }
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/scripts/RunAdHocRecordScriptAction.java

@@ -22,15 +22,14 @@
 package com.kingsrook.qqq.backend.core.actions.scripts;
 
 
+import java.io.Serializable;
+import java.lang.reflect.Method;
 import java.util.ArrayList;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
-import java.util.Objects;
 import java.util.Optional;
 import com.kingsrook.qqq.backend.core.actions.ActionHelper;
-import com.kingsrook.qqq.backend.core.actions.scripts.logging.QCodeExecutionLoggerInterface;
-import com.kingsrook.qqq.backend.core.actions.scripts.logging.ScriptExecutionLoggerInterface;
-import com.kingsrook.qqq.backend.core.actions.scripts.logging.StoreScriptLogAndScriptLogLineExecutionLogger;
 import com.kingsrook.qqq.backend.core.actions.tables.GetAction;
 import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
 import com.kingsrook.qqq.backend.core.context.QContext;
@@ -49,11 +48,10 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryJoin;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput;
 import com.kingsrook.qqq.backend.core.model.metadata.code.AdHocScriptCodeReference;
-import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
-import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeType;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 import com.kingsrook.qqq.backend.core.model.scripts.Script;
 import com.kingsrook.qqq.backend.core.model.scripts.ScriptRevision;
+import com.kingsrook.qqq.backend.core.model.scripts.ScriptsMetaDataProvider;
 import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
 
 
@@ -96,6 +94,7 @@ public class RunAdHocRecordScriptAction
             QueryInput queryInput = new QueryInput();
             queryInput.setTableName(input.getTableName());
             queryInput.setFilter(new QQueryFilter(new QFilterCriteria(table.getPrimaryKeyField(), QCriteriaOperator.IN, input.getRecordPrimaryKeyList())));
+            queryInput.setIncludeAssociations(true);
             QueryOutput queryOutput = new QueryAction().execute(queryInput);
             input.setRecordList(queryOutput.getRecords());
          }
@@ -112,43 +111,14 @@ public class RunAdHocRecordScriptAction
          /////////////
          // run it! //
          /////////////
-         ExecuteCodeInput executeCodeInput = new ExecuteCodeInput();
-         executeCodeInput.setInput(new HashMap<>(Objects.requireNonNullElseGet(input.getInputValues(), HashMap::new)));
-         executeCodeInput.getInput().put("records", new ArrayList<>(input.getRecordList()));
-         executeCodeInput.setContext(new HashMap<>());
-         if(input.getOutputObject() != null)
-         {
-            executeCodeInput.getContext().put("output", input.getOutputObject());
-         }
-
-         if(input.getScriptUtils() != null)
-         {
-            executeCodeInput.getContext().put("scriptUtils", input.getScriptUtils());
-         }
-
-         executeCodeInput.getContext().put("api", new ScriptApi());
-
-         executeCodeInput.setCodeReference(new QCodeReference().withInlineCode(scriptRevision.getContents()).withCodeType(QCodeType.JAVA_SCRIPT)); // todo - code type as attribute of script!!
-
-         /////////////////////////////////////////////////////////////////////////////////////////////////
-         // let caller supply a logger, or by default use StoreScriptLogAndScriptLogLineExecutionLogger //
-         /////////////////////////////////////////////////////////////////////////////////////////////////
-         QCodeExecutionLoggerInterface executionLogger = Objects.requireNonNullElseGet(input.getLogger(), () -> new StoreScriptLogAndScriptLogLineExecutionLogger(scriptRevision.getScriptId(), scriptRevision.getId()));
-         executeCodeInput.setExecutionLogger(executionLogger);
-         if(executionLogger instanceof ScriptExecutionLoggerInterface scriptExecutionLoggerInterface)
-         {
-            ////////////////////////////////////////////////////////////////////////////////////////////////////
-            // if logger is aware of scripts (as opposed to a generic CodeExecution logger), give it the ids. //
-            ////////////////////////////////////////////////////////////////////////////////////////////////////
-            scriptExecutionLoggerInterface.setScriptId(scriptRevision.getScriptId());
-            scriptExecutionLoggerInterface.setScriptRevisionId(scriptRevision.getId());
-         }
+         ExecuteCodeInput executeCodeInput = ExecuteCodeAction.setupExecuteCodeInput(input, scriptRevision);
+         executeCodeInput.getInput().put("records", getRecordsForScript(input, scriptRevision));
 
          ExecuteCodeOutput executeCodeOutput = new ExecuteCodeOutput();
          new ExecuteCodeAction().run(executeCodeInput, executeCodeOutput);
 
          output.setOutput(executeCodeOutput.getOutput());
-         output.setLogger(executionLogger);
+         output.setLogger(executeCodeInput.getExecutionLogger());
       }
       catch(Exception e)
       {
@@ -158,6 +128,37 @@ public class RunAdHocRecordScriptAction
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private static ArrayList<? extends Serializable> getRecordsForScript(RunAdHocRecordScriptInput input, ScriptRevision scriptRevision)
+   {
+      try
+      {
+         Class<?> apiScriptUtilsClass        = Class.forName("com.kingsrook.qqq.api.utils.ApiScriptUtils");
+         Method   qRecordListToApiRecordList = apiScriptUtilsClass.getMethod("qRecordListToApiRecordList", List.class, String.class, String.class, String.class);
+         Object   apiRecordList              = qRecordListToApiRecordList.invoke(null, input.getRecordList(), input.getTableName(), scriptRevision.getApiName(), scriptRevision.getApiVersion());
+
+         // noinspection unchecked
+         return (ArrayList<? extends Serializable>) apiRecordList;
+      }
+      catch(ClassNotFoundException e)
+      {
+         /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+         // this is the only exception we're kinda expecting here - so catch for it specifically, and just log.trace - others, warn //
+         /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+         LOG.trace("Couldn't load ApiScriptUtils class - qqq-middleware-api not on the classpath?");
+      }
+      catch(Exception e)
+      {
+         LOG.warn("Error converting QRecord list to api record list", e);
+      }
+
+      return (new ArrayList<>(input.getRecordList()));
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/
@@ -197,7 +198,7 @@ public class RunAdHocRecordScriptAction
             QueryInput queryInput = new QueryInput();
             queryInput.setTableName(ScriptRevision.TABLE_NAME);
             queryInput.setFilter(new QQueryFilter(new QFilterCriteria("script.id", QCriteriaOperator.EQUALS, codeReference.getScriptId())));
-            queryInput.withQueryJoin(new QueryJoin(Script.TABLE_NAME).withBaseTableOrAlias(ScriptRevision.TABLE_NAME).withJoinMetaData(QContext.getQInstance().getJoin("currentScriptRevision")));
+            queryInput.withQueryJoin(new QueryJoin(Script.TABLE_NAME).withBaseTableOrAlias(ScriptRevision.TABLE_NAME).withJoinMetaData(QContext.getQInstance().getJoin(ScriptsMetaDataProvider.CURRENT_SCRIPT_REVISION_JOIN_NAME)));
             QueryOutput queryOutput = new QueryAction().execute(queryInput);
 
             if(CollectionUtils.nullSafeHasContents(queryOutput.getRecords()))

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/scripts/RunAssociatedScriptAction.java

@@ -25,11 +25,7 @@ package com.kingsrook.qqq.backend.core.actions.scripts;
 import java.io.Serializable;
 import java.util.HashMap;
 import java.util.Map;
-import java.util.Objects;
 import com.kingsrook.qqq.backend.core.actions.ActionHelper;
-import com.kingsrook.qqq.backend.core.actions.scripts.logging.QCodeExecutionLoggerInterface;
-import com.kingsrook.qqq.backend.core.actions.scripts.logging.ScriptExecutionLoggerInterface;
-import com.kingsrook.qqq.backend.core.actions.scripts.logging.StoreScriptLogAndScriptLogLineExecutionLogger;
 import com.kingsrook.qqq.backend.core.actions.tables.GetAction;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.exceptions.QNotFoundException;
@@ -40,8 +36,6 @@ import com.kingsrook.qqq.backend.core.model.actions.scripts.RunAssociatedScriptO
 import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetOutput;
 import com.kingsrook.qqq.backend.core.model.metadata.code.AssociatedScriptCodeReference;
-import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
-import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeType;
 import com.kingsrook.qqq.backend.core.model.scripts.Script;
 import com.kingsrook.qqq.backend.core.model.scripts.ScriptRevision;
 
@@ -54,6 +48,7 @@ public class RunAssociatedScriptAction
    private Map<AssociatedScriptCodeReference, ScriptRevision> scriptRevisionCache = new HashMap<>();
 
 
+
    /*******************************************************************************
     **
     *******************************************************************************/
@@ -61,41 +56,19 @@ public class RunAssociatedScriptAction
    {
       ActionHelper.validateSession(input);
 
-      ScriptRevision scriptRevision = getScriptRevision(input);
+      ScriptRevision   scriptRevision   = getScriptRevision(input);
+      ExecuteCodeInput executeCodeInput = ExecuteCodeAction.setupExecuteCodeInput(input, scriptRevision);
 
-      ExecuteCodeInput executeCodeInput = new ExecuteCodeInput();
-      executeCodeInput.setInput(new HashMap<>(input.getInputValues()));
-      executeCodeInput.setContext(new HashMap<>());
-      if(input.getOutputObject() != null)
+      if(input.getAssociatedScriptContextPrimerInterface() != null)
       {
-         executeCodeInput.getContext().put("output", input.getOutputObject());
-      }
-
-      if(input.getScriptUtils() != null)
-      {
-         executeCodeInput.getContext().put("scriptUtils", input.getScriptUtils());
-      }
-
-      executeCodeInput.setCodeReference(new QCodeReference().withInlineCode(scriptRevision.getContents()).withCodeType(QCodeType.JAVA_SCRIPT)); // todo - code type as attribute of script!!
-
-      /////////////////////////////////////////////////////////////////////////////////////////////////
-      // let caller supply a logger, or by default use StoreScriptLogAndScriptLogLineExecutionLogger //
-      /////////////////////////////////////////////////////////////////////////////////////////////////
-      QCodeExecutionLoggerInterface executionLogger = Objects.requireNonNullElseGet(input.getLogger(), () -> new StoreScriptLogAndScriptLogLineExecutionLogger(scriptRevision.getScriptId(), scriptRevision.getId()));
-      executeCodeInput.setExecutionLogger(executionLogger);
-      if(executionLogger instanceof ScriptExecutionLoggerInterface scriptExecutionLoggerInterface)
-      {
-         ////////////////////////////////////////////////////////////////////////////////////////////////////
-         // if logger is aware of scripts (as opposed to a generic CodeExecution logger), give it the ids. //
-         ////////////////////////////////////////////////////////////////////////////////////////////////////
-         scriptExecutionLoggerInterface.setScriptId(scriptRevision.getScriptId());
-         scriptExecutionLoggerInterface.setScriptRevisionId(scriptRevision.getId());
+         input.getAssociatedScriptContextPrimerInterface().primeContext(executeCodeInput, scriptRevision);
       }
 
       ExecuteCodeOutput executeCodeOutput = new ExecuteCodeOutput();
       new ExecuteCodeAction().run(executeCodeInput, executeCodeOutput);
 
       output.setOutput(executeCodeOutput.getOutput());
+      output.setScriptRevisionId(scriptRevision.getId());
    }
 
 
@@ -138,6 +111,7 @@ public class RunAssociatedScriptAction
       GetInput getInput = new GetInput();
       getInput.setTableName("scriptRevision");
       getInput.setPrimaryKey(scriptRevisionId);
+      getInput.setIncludeAssociations(true);
       GetOutput getOutput = new GetAction().execute(getInput);
       if(getOutput.getRecord() == null)
       {

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/scripts/StoreAssociatedScriptAction.java

@@ -31,6 +31,8 @@ import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
 import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
 import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
+import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
 import com.kingsrook.qqq.backend.core.model.actions.scripts.StoreAssociatedScriptInput;
 import com.kingsrook.qqq.backend.core.model.actions.scripts.StoreAssociatedScriptOutput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetInput;
@@ -47,6 +49,7 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.AssociatedScript;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+import com.kingsrook.qqq.backend.core.processes.implementations.scripts.StoreScriptRevisionProcessStep;
 import com.kingsrook.qqq.backend.core.utils.StringUtils;
 
 
@@ -156,8 +159,7 @@ public class StoreAssociatedScriptAction
          queryInput.setFilter(new QQueryFilter()
             .withCriteria(new QFilterCriteria("scriptId", QCriteriaOperator.EQUALS, List.of(script.getValue("id"))))
             .withOrderBy(new QFilterOrderBy("sequenceNo", false))
-         );
-         queryInput.setLimit(1);
+            .withLimit(1));
          QueryOutput queryOutput = new QueryAction().execute(queryInput);
          if(!queryOutput.getRecords().isEmpty())
          {
@@ -181,39 +183,19 @@ public class StoreAssociatedScriptAction
          }
       }
 
-      QRecord scriptRevision = new QRecord()
-         .withValue("scriptId", script.getValue("id"))
-         .withValue("contents", input.getCode())
-         .withValue("commitMessage", commitMessage)
-         .withValue("sequenceNo", nextSequenceNo);
-
-      try
-      {
-         scriptRevision.setValue("author", input.getSession().getUser().getFullName());
-      }
-      catch(Exception e)
-      {
-         scriptRevision.setValue("author", "Unknown");
-      }
-
-      InsertInput insertInput = new InsertInput();
-      insertInput.setTableName("scriptRevision");
-      insertInput.setRecords(List.of(scriptRevision));
-      InsertOutput insertOutput = new InsertAction().execute(insertInput);
-      scriptRevision = insertOutput.getRecords().get(0);
-
-      ////////////////////////////////////////////////////
-      // update the script to point at the new revision //
-      ////////////////////////////////////////////////////
-      script.setValue("currentScriptRevisionId", scriptRevision.getValue("id"));
-      UpdateInput updateInput = new UpdateInput();
-      updateInput.setTableName("script");
-      updateInput.setRecords(List.of(script));
-      new UpdateAction().execute(updateInput);
+      RunBackendStepInput storeScriptRevisionInput = new RunBackendStepInput();
+      storeScriptRevisionInput.addValue("scriptId", script.getValue("id"));
+      storeScriptRevisionInput.addValue("commitMessage", commitMessage);
+      storeScriptRevisionInput.addValue("apiName", input.getApiName());
+      storeScriptRevisionInput.addValue("apiVersion", input.getApiVersion());
+      storeScriptRevisionInput.addValue("fileNames", "script");
+      storeScriptRevisionInput.addValue("fileContents:script", input.getCode());
+      RunBackendStepOutput storeScriptRevisionOutput = new RunBackendStepOutput();
+      new StoreScriptRevisionProcessStep().run(storeScriptRevisionInput, storeScriptRevisionOutput);
 
       output.setScriptId(script.getValueInteger("id"));
       output.setScriptName(script.getValueString("name"));
-      output.setScriptRevisionId(scriptRevision.getValueInteger("id"));
-      output.setScriptRevisionSequenceNo(scriptRevision.getValueInteger("sequenceNo"));
+      output.setScriptRevisionId(storeScriptRevisionOutput.getValueInteger("scriptRevisionId"));
+      output.setScriptRevisionSequenceNo(storeScriptRevisionOutput.getValueInteger("scriptRevisionSequenceNo"));
    }
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/scripts/TestScriptActionInterface.java

@@ -32,6 +32,7 @@ import com.kingsrook.qqq.backend.core.model.actions.scripts.ExecuteCodeOutput;
 import com.kingsrook.qqq.backend.core.model.actions.scripts.TestScriptInput;
 import com.kingsrook.qqq.backend.core.model.actions.scripts.TestScriptOutput;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
+import com.kingsrook.qqq.backend.core.model.scripts.ScriptRevision;
 
 
 /*******************************************************************************
@@ -47,7 +48,7 @@ public interface TestScriptActionInterface
     ** Note - such a method may want or need to put an "output" object into the
     ** executeCodeInput's context map.
     *******************************************************************************/
-   void setupTestScriptInput(TestScriptInput testScriptInput, ExecuteCodeInput executeCodeInput);
+   void setupTestScriptInput(TestScriptInput testScriptInput, ExecuteCodeInput executeCodeInput) throws QException;
 
 
    /*******************************************************************************
@@ -80,6 +81,9 @@ public interface TestScriptActionInterface
     *******************************************************************************/
    default void execute(TestScriptInput input, TestScriptOutput output) throws QException
    {
+      /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      // todo - could this be merged with the various other script runners, to use ExecuteCodeAction.setupExecuteCodeInput?? //
+      /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
       ExecuteCodeInput executeCodeInput = new ExecuteCodeInput();
       executeCodeInput.setContext(new HashMap<>());
 
@@ -87,12 +91,21 @@ public interface TestScriptActionInterface
       BuildScriptLogAndScriptLogLineExecutionLogger executionLogger = new BuildScriptLogAndScriptLogLineExecutionLogger(null, null);
       executeCodeInput.setExecutionLogger(executionLogger);
 
-      setupTestScriptInput(input, executeCodeInput);
-
-      ExecuteCodeOutput executeCodeOutput = new ExecuteCodeOutput();
-
       try
       {
+         setupTestScriptInput(input, executeCodeInput);
+
+         ScriptRevision scriptRevision = new ScriptRevision().withApiName(input.getApiName()).withApiVersion(input.getApiVersion());
+
+         if(this instanceof AssociatedScriptContextPrimerInterface associatedScriptContextPrimerInterface)
+         {
+            associatedScriptContextPrimerInterface.primeContext(executeCodeInput, scriptRevision);
+         }
+
+         ExecuteCodeOutput executeCodeOutput = new ExecuteCodeOutput();
+
+         ExecuteCodeAction.addApiUtilityToContext(executeCodeInput.getContext(), scriptRevision);
+
          new ExecuteCodeAction().run(executeCodeInput, executeCodeOutput);
          output.setOutputObject(processTestScriptOutput(executeCodeOutput));
       }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/scripts/logging/Log4jCodeExecutionLogger.java

@@ -41,6 +41,7 @@ public class Log4jCodeExecutionLogger implements QCodeExecutionLoggerInterface
    private QCodeReference qCodeReference;
    private String         uuid = UUID.randomUUID().toString();
 
+   private boolean includeUUID = true;
 
 
    /*******************************************************************************
@@ -52,7 +53,7 @@ public class Log4jCodeExecutionLogger implements QCodeExecutionLoggerInterface
       this.qCodeReference = executeCodeInput.getCodeReference();
 
       String inputString = StringUtils.safeTruncate(ValueUtils.getValueAsString(executeCodeInput.getInput()), 250, "...");
-      LOG.info("Starting script execution: " + qCodeReference.getName() + ", uuid: " + uuid + ", with input: " + inputString);
+      LOG.info("Starting script execution: " + qCodeReference.getName() + (includeUUID ? ", uuid: " + uuid : "") + ", with input: " + inputString);
    }
 
 
@@ -63,7 +64,7 @@ public class Log4jCodeExecutionLogger implements QCodeExecutionLoggerInterface
    @Override
    public void acceptLogLine(String logLine)
    {
-      LOG.info("Script log: " + uuid + ": " + logLine);
+      LOG.info("Script log: " + (includeUUID ? uuid + ": " : "") + logLine);
    }
 
 
@@ -74,7 +75,7 @@ public class Log4jCodeExecutionLogger implements QCodeExecutionLoggerInterface
    @Override
    public void acceptException(Exception exception)
    {
-      LOG.info("Script Exception: " + uuid, exception);
+      LOG.info("Script Exception: " + (includeUUID ? uuid : ""), exception);
    }
 
 
@@ -86,7 +87,38 @@ public class Log4jCodeExecutionLogger implements QCodeExecutionLoggerInterface
    public void acceptExecutionEnd(Serializable output)
    {
       String outputString = StringUtils.safeTruncate(ValueUtils.getValueAsString(output), 250, "...");
-      LOG.info("Finished script execution: " + qCodeReference.getName() + ", uuid: " + uuid + ", with output: " + outputString);
+      LOG.info("Finished script execution: " + qCodeReference.getName() + (includeUUID ? ", uuid: " + uuid : "") + ", with output: " + outputString);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for includeUUID
+    *******************************************************************************/
+   public boolean getIncludeUUID()
+   {
+      return (this.includeUUID);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for includeUUID
+    *******************************************************************************/
+   public void setIncludeUUID(boolean includeUUID)
+   {
+      this.includeUUID = includeUUID;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for includeUUID
+    *******************************************************************************/
+   public Log4jCodeExecutionLogger withIncludeUUID(boolean includeUUID)
+   {
+      this.includeUUID = includeUUID;
+      return (this);
    }
 
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/scripts/logging/QCodeExecutionLoggerInterface.java

@@ -29,7 +29,7 @@ import com.kingsrook.qqq.backend.core.model.actions.scripts.ExecuteCodeInput;
 /*******************************************************************************
  ** Interface to provide logging functionality to QCodeExecution (e.g., scripts)
  *******************************************************************************/
-public interface QCodeExecutionLoggerInterface
+public interface QCodeExecutionLoggerInterface extends Serializable
 {
 
    /*******************************************************************************

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/scripts/logging/SystemOutExecutionLogger.java

@@ -0,0 +1,88 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.actions.scripts.logging;
+
+
+import java.io.Serializable;
+import com.kingsrook.qqq.backend.core.model.actions.scripts.ExecuteCodeInput;
+import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
+import com.kingsrook.qqq.backend.core.utils.ValueUtils;
+
+
+/*******************************************************************************
+ ** Implementation of a code execution logger that logs to System.out and
+ ** System.err (for exceptions)
+ *******************************************************************************/
+public class SystemOutExecutionLogger implements QCodeExecutionLoggerInterface
+{
+   private QCodeReference qCodeReference;
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public void acceptExecutionStart(ExecuteCodeInput executeCodeInput)
+   {
+      this.qCodeReference = executeCodeInput.getCodeReference();
+
+      String inputString = ValueUtils.getValueAsString(executeCodeInput.getInput());
+      System.out.println("Starting script execution: " + qCodeReference.getName() + ", with input: " + inputString);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public void acceptLogLine(String logLine)
+   {
+      System.out.println("Script log: " + logLine);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public void acceptException(Exception exception)
+   {
+      System.out.println("Script Exception: " + exception.getMessage());
+      exception.printStackTrace();
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public void acceptExecutionEnd(Serializable output)
+   {
+      String outputString = ValueUtils.getValueAsString(output);
+      System.out.println("Finished script execution: " + qCodeReference.getName() + ", with output: " + outputString);
+   }
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/AggregateAction.java

@@ -23,9 +23,15 @@ package com.kingsrook.qqq.backend.core.actions.tables;
 
 
 import com.kingsrook.qqq.backend.core.actions.ActionHelper;
+import com.kingsrook.qqq.backend.core.actions.interfaces.AggregateInterface;
+import com.kingsrook.qqq.backend.core.actions.tables.helpers.QueryStatManager;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.logging.QLogger;
 import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateOutput;
+import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+import com.kingsrook.qqq.backend.core.model.querystats.QueryStat;
 import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleDispatcher;
 import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleInterface;
 
@@ -36,6 +42,12 @@ import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleInterface;
  *******************************************************************************/
 public class AggregateAction
 {
+   private static final QLogger LOG = QLogger.getLogger(AggregateAction.class);
+
+   private AggregateInterface aggregateInterface;
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/
@@ -43,11 +55,36 @@ public class AggregateAction
    {
       ActionHelper.validateSession(aggregateInput);
 
+      QTableMetaData   table   = aggregateInput.getTable();
+      QBackendMetaData backend = aggregateInput.getBackend();
+
+      QueryStat queryStat = QueryStatManager.newQueryStat(backend, table, aggregateInput.getFilter());
+
       QBackendModuleDispatcher qBackendModuleDispatcher = new QBackendModuleDispatcher();
       QBackendModuleInterface  qModule                  = qBackendModuleDispatcher.getQBackendModule(aggregateInput.getBackend());
-      // todo pre-customization - just get to modify the request?
-      AggregateOutput aggregateOutput = qModule.getAggregateInterface().execute(aggregateInput);
-      // todo post-customization - can do whatever w/ the result if you want
+
+      aggregateInterface = qModule.getAggregateInterface();
+      aggregateInterface.setQueryStat(queryStat);
+      AggregateOutput aggregateOutput = aggregateInterface.execute(aggregateInput);
+
+      QueryStatManager.getInstance().add(queryStat);
+
       return aggregateOutput;
    }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public void cancel()
+   {
+      if(aggregateInterface == null)
+      {
+         LOG.warn("aggregateInterface object was null when requested to cancel");
+         return;
+      }
+
+      aggregateInterface.cancelAction();
+   }
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/CountAction.java

@@ -23,9 +23,15 @@ package com.kingsrook.qqq.backend.core.actions.tables;
 
 
 import com.kingsrook.qqq.backend.core.actions.ActionHelper;
+import com.kingsrook.qqq.backend.core.actions.interfaces.CountInterface;
+import com.kingsrook.qqq.backend.core.actions.tables.helpers.QueryStatManager;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.logging.QLogger;
 import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountOutput;
+import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+import com.kingsrook.qqq.backend.core.model.querystats.QueryStat;
 import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleDispatcher;
 import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleInterface;
 
@@ -36,6 +42,12 @@ import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleInterface;
  *******************************************************************************/
 public class CountAction
 {
+   private static final QLogger LOG = QLogger.getLogger(CountAction.class);
+
+   private CountInterface countInterface;
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/
@@ -43,11 +55,36 @@ public class CountAction
    {
       ActionHelper.validateSession(countInput);
 
+      QTableMetaData   table   = countInput.getTable();
+      QBackendMetaData backend = countInput.getBackend();
+
+      QueryStat queryStat = QueryStatManager.newQueryStat(backend, table, countInput.getFilter());
+
       QBackendModuleDispatcher qBackendModuleDispatcher = new QBackendModuleDispatcher();
-      QBackendModuleInterface qModule = qBackendModuleDispatcher.getQBackendModule(countInput.getBackend());
-      // todo pre-customization - just get to modify the request?
-      CountOutput countOutput = qModule.getCountInterface().execute(countInput);
-      // todo post-customization - can do whatever w/ the result if you want
+      QBackendModuleInterface  qModule                  = qBackendModuleDispatcher.getQBackendModule(countInput.getBackend());
+
+      countInterface = qModule.getCountInterface();
+      countInterface.setQueryStat(queryStat);
+      CountOutput countOutput = countInterface.execute(countInput);
+
+      QueryStatManager.getInstance().add(queryStat);
+
       return countOutput;
    }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public void cancel()
+   {
+      if(countInterface == null)
+      {
+         LOG.warn("countInterface object was null when requested to cancel");
+         return;
+      }
+
+      countInterface.cancelAction();
+   }
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/DeleteAction.java

@@ -26,14 +26,24 @@ import java.io.Serializable;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.HashSet;
+import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
 import java.util.Set;
+import java.util.stream.Collectors;
 import com.kingsrook.qqq.backend.core.actions.ActionHelper;
 import com.kingsrook.qqq.backend.core.actions.audits.DMLAuditAction;
+import com.kingsrook.qqq.backend.core.actions.customizers.AbstractPostDeleteCustomizer;
+import com.kingsrook.qqq.backend.core.actions.customizers.AbstractPreDeleteCustomizer;
+import com.kingsrook.qqq.backend.core.actions.customizers.QCodeLoader;
+import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizers;
 import com.kingsrook.qqq.backend.core.actions.interfaces.DeleteInterface;
+import com.kingsrook.qqq.backend.core.actions.tables.helpers.ValidateRecordSecurityLockHelper;
 import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.logging.LogPair;
 import com.kingsrook.qqq.backend.core.logging.QLogger;
 import com.kingsrook.qqq.backend.core.model.actions.audits.DMLAuditInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput;
@@ -44,11 +54,12 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
-import com.kingsrook.qqq.backend.core.model.metadata.audits.AuditLevel;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.joins.QJoinMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.Association;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+import com.kingsrook.qqq.backend.core.model.statusmessages.NotFoundStatusMessage;
+import com.kingsrook.qqq.backend.core.model.statusmessages.QWarningMessage;
 import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleDispatcher;
 import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleInterface;
 import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
@@ -63,8 +74,6 @@ public class DeleteAction
 {
    private static final QLogger LOG = QLogger.getLogger(DeleteAction.class);
 
-   public static final String NOT_FOUND_ERROR_PREFIX = "No record was found to delete";
-
 
 
    /*******************************************************************************
@@ -74,20 +83,50 @@ public class DeleteAction
    {
       ActionHelper.validateSession(deleteInput);
 
-      QBackendModuleDispatcher qBackendModuleDispatcher = new QBackendModuleDispatcher();
-      QBackendModuleInterface  qModule                  = qBackendModuleDispatcher.getQBackendModule(deleteInput.getBackend());
+      QTableMetaData table               = deleteInput.getTable();
+      String         primaryKeyFieldName = table.getPrimaryKeyField();
+      QFieldMetaData primaryKeyField     = table.getField(primaryKeyFieldName);
 
-      if(CollectionUtils.nullSafeHasContents(deleteInput.getPrimaryKeys()) && deleteInput.getQueryFilter() != null)
+      List<Serializable> primaryKeys         = deleteInput.getPrimaryKeys();
+      List<Serializable> originalPrimaryKeys = primaryKeys == null ? null : new ArrayList<>(primaryKeys);
+      if(CollectionUtils.nullSafeHasContents(primaryKeys) && deleteInput.getQueryFilter() != null)
       {
          throw (new QException("A delete request may not contain both a list of primary keys and a query filter."));
       }
 
-      DeleteInterface deleteInterface = qModule.getDeleteInterface();
-      if(deleteInput.getQueryFilter() != null && !deleteInterface.supportsQueryFilterInput())
+      ////////////////////////////////////////////////////////
+      // make sure the primary keys are of the correct type //
+      ////////////////////////////////////////////////////////
+      if(CollectionUtils.nullSafeHasContents(primaryKeys))
       {
-         LOG.info("Querying for primary keys, for backend module " + qModule.getBackendType() + " which does not support queryFilter input for deletes");
+         for(int i = 0; i < primaryKeys.size(); i++)
+         {
+            Serializable primaryKey       = primaryKeys.get(i);
+            Serializable valueAsFieldType = ValueUtils.getValueAsFieldType(primaryKeyField.getType(), primaryKey);
+            if(!Objects.equals(primaryKey, valueAsFieldType))
+            {
+               primaryKeys.set(i, valueAsFieldType);
+            }
+         }
+      }
+
+      //////////////////////////////////////////////////////
+      // load the backend module and its delete interface //
+      //////////////////////////////////////////////////////
+      QBackendModuleDispatcher qBackendModuleDispatcher = new QBackendModuleDispatcher();
+      QBackendModuleInterface  qModule                  = qBackendModuleDispatcher.getQBackendModule(deleteInput.getBackend());
+      DeleteInterface          deleteInterface          = qModule.getDeleteInterface();
+
+      //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      // if there's a query filter, but the interface doesn't support using a query filter, then do a query for the filter, to get a list of primary keys instead //
+      // or - anytime there are associations on the table we want primary keys, as that's what the manage associations method uses                                //
+      //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      if(deleteInput.getQueryFilter() != null && (!deleteInterface.supportsQueryFilterInput() || CollectionUtils.nullSafeHasContents(table.getAssociations())))
+      {
+         LOG.info("Querying for primary keys, for table " + table.getName() + " in backend module " + qModule.getBackendType() + " which does not support queryFilter input for deletes (or the table has associations)");
          List<Serializable> primaryKeyList = getPrimaryKeysFromQueryFilter(deleteInput);
          deleteInput.setPrimaryKeys(primaryKeyList);
+         primaryKeys = primaryKeyList;
 
          if(primaryKeyList.isEmpty())
          {
@@ -99,36 +138,237 @@ public class DeleteAction
          }
       }
 
-      List<QRecord> recordListForAudit          = getRecordListForAuditIfNeeded(deleteInput);
-      List<QRecord> recordsWithValidationErrors = validateRecordsExistAndCanBeAccessed(deleteInput, recordListForAudit);
+      ////////////////////////////////////////////////////////////////////////////////
+      // fetch the old list of records (if the backend supports it), for audits,    //
+      // for "not-found detection", and for the pre-action to use (if there is one) //
+      ////////////////////////////////////////////////////////////////////////////////
+      Optional<List<QRecord>> oldRecordList = fetchOldRecords(deleteInput, deleteInterface);
 
-      DeleteOutput deleteOutput = deleteInterface.execute(deleteInput);
+      List<QRecord>              customizerResult              = performValidations(deleteInput, oldRecordList, false);
+      List<QRecord>              recordsWithValidationErrors   = new ArrayList<>();
+      Map<Serializable, QRecord> recordsWithValidationWarnings = new LinkedHashMap<>();
 
-      /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-      // merge the backend's output with any validation errors we found (whose ids wouldn't have gotten into the backend delete) //
-      /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-      List<QRecord> outputRecordsWithErrors = deleteOutput.getRecordsWithErrors();
-      if(outputRecordsWithErrors == null)
+      ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      // check if any records got errors in the customizer - if so, remove them from the input list of pkeys to delete //
+      ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      if(customizerResult != null)
       {
-         deleteOutput.setRecordsWithErrors(new ArrayList<>());
-         outputRecordsWithErrors = deleteOutput.getRecordsWithErrors();
+         Set<Serializable> primaryKeysToRemoveFromInput = new HashSet<>();
+         for(QRecord record : customizerResult)
+         {
+            if(CollectionUtils.nullSafeHasContents(record.getErrors()))
+            {
+               recordsWithValidationErrors.add(record);
+               primaryKeysToRemoveFromInput.add(record.getValue(primaryKeyFieldName));
+            }
+            else if(CollectionUtils.nullSafeHasContents(record.getWarnings()))
+            {
+               recordsWithValidationWarnings.put(record.getValue(primaryKeyFieldName), record);
+            }
+         }
+
+         if(!primaryKeysToRemoveFromInput.isEmpty())
+         {
+            if(primaryKeys == null)
+            {
+               LOG.warn("There were primary keys to remove from the input, but no primary key list (filter supplied as input?)", new LogPair("primaryKeysToRemoveFromInput", primaryKeysToRemoveFromInput));
+            }
+            else
+            {
+               primaryKeys.removeAll(primaryKeysToRemoveFromInput);
+            }
+         }
       }
 
+      ////////////////////////////////////////////////////////////////////////////////////////////////
+      // stash a copy of primary keys that didn't have errors (for use in manageAssociations below) //
+      ////////////////////////////////////////////////////////////////////////////////////////////////
+      Set<Serializable> primaryKeysWithoutErrors = new HashSet<>(CollectionUtils.nonNullList(primaryKeys));
+
+      ////////////////////////////////////
+      // have the backend do the delete //
+      ////////////////////////////////////
+      DeleteOutput deleteOutput = deleteInterface.execute(deleteInput);
+
+      /////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      // reset the input's list of primary keys -- callers may use & expect that to be what they had passed in!! //
+      /////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      deleteInput.setPrimaryKeys(originalPrimaryKeys);
+
+      ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      // merge the backend's output with any validation errors we found (whose pkeys wouldn't have gotten into the backend delete) //
+      ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      List<QRecord> outputRecordsWithErrors = Objects.requireNonNullElseGet(deleteOutput.getRecordsWithErrors(), () -> new ArrayList<>());
       outputRecordsWithErrors.addAll(recordsWithValidationErrors);
 
-      manageAssociations(deleteInput);
+      ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      // if a record had a validation warning, but then an execution error, remove it from the warning list - so it's only in one of them. //
+      // also, always remove from
+      ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      for(QRecord outputRecordWithError : outputRecordsWithErrors)
+      {
+         Serializable pkey = outputRecordWithError.getValue(primaryKeyFieldName);
+         recordsWithValidationWarnings.remove(pkey);
+         primaryKeysWithoutErrors.remove(pkey);
+      }
 
-      new DMLAuditAction().execute(new DMLAuditInput().withTableActionInput(deleteInput).withRecordList(recordListForAudit));
+      ///////////////////////////////////////////////////////////////////////////////////////////
+      // combine the warning list from validation to that from execution - avoiding duplicates //
+      // use a map to manage this list for the rest of this method                             //
+      ///////////////////////////////////////////////////////////////////////////////////////////
+      Map<Serializable, QRecord> outputRecordsWithWarningMap = CollectionUtils.nullSafeIsEmpty(deleteOutput.getRecordsWithWarnings()) ? new LinkedHashMap<>()
+         : deleteOutput.getRecordsWithWarnings().stream().collect(Collectors.toMap(r -> r.getValue(primaryKeyFieldName), r -> r, (a, b) -> a, () -> new LinkedHashMap<>()));
+      for(Map.Entry<Serializable, QRecord> entry : recordsWithValidationWarnings.entrySet())
+      {
+         if(!outputRecordsWithWarningMap.containsKey(entry.getKey()))
+         {
+            outputRecordsWithWarningMap.put(entry.getKey(), entry.getValue());
+         }
+      }
+
+      ////////////////////////////////////////
+      // delete associations, if applicable //
+      ////////////////////////////////////////
+      manageAssociations(primaryKeysWithoutErrors, deleteInput);
+
+      //////////////////
+      // do the audit //
+      //////////////////
+      if(deleteInput.getOmitDmlAudit())
+      {
+         LOG.debug("Requested to omit DML audit");
+      }
+      else
+      {
+         DMLAuditInput dmlAuditInput = new DMLAuditInput()
+            .withTableActionInput(deleteInput)
+            .withAuditContext(deleteInput.getAuditContext());
+         oldRecordList.ifPresent(l -> dmlAuditInput.setRecordList(l));
+         new DMLAuditAction().execute(dmlAuditInput);
+      }
+
+      //////////////////////////////////////////////////////////////
+      // finally, run the post-delete customizer, if there is one //
+      //////////////////////////////////////////////////////////////
+      Optional<AbstractPostDeleteCustomizer> postDeleteCustomizer = QCodeLoader.getTableCustomizer(AbstractPostDeleteCustomizer.class, table, TableCustomizers.POST_DELETE_RECORD.getRole());
+      if(postDeleteCustomizer.isPresent() && oldRecordList.isPresent())
+      {
+         ////////////////////////////////////////////////////////////////////////////
+         // make list of records that are still good - to pass into the customizer //
+         ////////////////////////////////////////////////////////////////////////////
+         List<QRecord> recordsForCustomizer = makeListOfRecordsNotInErrorList(primaryKeyFieldName, oldRecordList.get(), outputRecordsWithErrors);
+
+         try
+         {
+            postDeleteCustomizer.get().setDeleteInput(deleteInput);
+            List<QRecord> postCustomizerResult = postDeleteCustomizer.get().apply(recordsForCustomizer);
+
+            ///////////////////////////////////////////////////////
+            // check if any records got errors in the customizer //
+            ///////////////////////////////////////////////////////
+            for(QRecord record : postCustomizerResult)
+            {
+               Serializable pkey = record.getValue(primaryKeyFieldName);
+               if(CollectionUtils.nullSafeHasContents(record.getErrors()))
+               {
+                  outputRecordsWithErrors.add(record);
+                  outputRecordsWithWarningMap.remove(pkey);
+               }
+               else if(CollectionUtils.nullSafeHasContents(record.getWarnings()))
+               {
+                  outputRecordsWithWarningMap.put(pkey, record);
+               }
+            }
+         }
+         catch(Exception e)
+         {
+            for(QRecord record : recordsForCustomizer)
+            {
+               record.addWarning(new QWarningMessage("An error occurred after the delete: " + e.getMessage()));
+               outputRecordsWithWarningMap.put(record.getValue(primaryKeyFieldName), record);
+            }
+         }
+      }
+
+      deleteOutput.setRecordsWithErrors(outputRecordsWithErrors);
+      deleteOutput.setRecordsWithWarnings(new ArrayList<>(outputRecordsWithWarningMap.values()));
 
       return deleteOutput;
    }
 
 
 
+   /*******************************************************************************
+    ** this method takes in the deleteInput, and the list of old records that matched
+    ** the pkeys in that input.
+    **
+    ** it'll check if any of those pkeys aren't found (in a sub-method) - a record
+    ** with an error message will be added to oldRecordList for any such records.
+    **
+    ** it'll also then call the pre-customizer, if there is one - taking in the
+    ** oldRecordList.  it can add other errors or warnings to records.
+    **
+    ** The return value here is basically oldRecordList - possibly with some new
+    ** entries for the pkey-not-founds, and possibly w/ errors and warnings from the
+    ** customizer.
+    *******************************************************************************/
+   public List<QRecord> performValidations(DeleteInput deleteInput, Optional<List<QRecord>> oldRecordList, boolean isPreview) throws QException
+   {
+      if(oldRecordList.isEmpty())
+      {
+         return (null);
+      }
+
+      QTableMetaData table               = deleteInput.getTable();
+      List<QRecord>  primaryKeysNotFound = validateRecordsExistAndCanBeAccessed(deleteInput, oldRecordList.get());
+
+      ValidateRecordSecurityLockHelper.validateSecurityFields(table, oldRecordList.get(), ValidateRecordSecurityLockHelper.Action.DELETE);
+
+      ///////////////////////////////////////////////////////////////////////////
+      // after all validations, run the pre-delete customizer, if there is one //
+      ///////////////////////////////////////////////////////////////////////////
+      Optional<AbstractPreDeleteCustomizer> preDeleteCustomizer = QCodeLoader.getTableCustomizer(AbstractPreDeleteCustomizer.class, table, TableCustomizers.PRE_DELETE_RECORD.getRole());
+      List<QRecord>                         customizerResult    = oldRecordList.get();
+      if(preDeleteCustomizer.isPresent())
+      {
+         preDeleteCustomizer.get().setDeleteInput(deleteInput);
+         preDeleteCustomizer.get().setIsPreview(isPreview);
+         customizerResult = preDeleteCustomizer.get().apply(oldRecordList.get());
+      }
+
+      /////////////////////////////////////////////////////////////////////////
+      // add any pkey-not-found records to the front of the customizerResult //
+      /////////////////////////////////////////////////////////////////////////
+      customizerResult.addAll(primaryKeysNotFound);
+
+      return customizerResult;
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/
-   private void manageAssociations(DeleteInput deleteInput) throws QException
+   private static List<QRecord> makeListOfRecordsNotInErrorList(String primaryKeyField, List<QRecord> oldRecordList, List<QRecord> outputRecordsWithErrors)
+   {
+      Map<Serializable, QRecord> recordsWithErrorsMap = outputRecordsWithErrors.stream().collect(Collectors.toMap(r -> r.getValue(primaryKeyField), r -> r));
+      List<QRecord>              recordsForCustomizer = new ArrayList<>();
+      for(QRecord record : oldRecordList)
+      {
+         if(!recordsWithErrorsMap.containsKey(record.getValue(primaryKeyField)))
+         {
+            recordsForCustomizer.add(record);
+         }
+      }
+      return recordsForCustomizer;
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private void manageAssociations(Set<Serializable> primaryKeysWithoutErrors, DeleteInput deleteInput) throws QException
    {
       QTableMetaData table = deleteInput.getTable();
       for(Association association : CollectionUtils.nonNullList(table.getAssociations()))
@@ -141,7 +381,7 @@ public class DeleteAction
 
          if(join.getJoinOns().size() == 1 && join.getJoinOns().get(0).getLeftField().equals(table.getPrimaryKeyField()))
          {
-            filter.addCriteria(new QFilterCriteria(join.getJoinOns().get(0).getRightField(), QCriteriaOperator.IN, deleteInput.getPrimaryKeys()));
+            filter.addCriteria(new QFilterCriteria(join.getJoinOns().get(0).getRightField(), QCriteriaOperator.IN, new ArrayList<>(primaryKeysWithoutErrors)));
          }
          else
          {
@@ -157,11 +397,14 @@ public class DeleteAction
          QueryOutput        queryOutput    = new QueryAction().execute(queryInput);
          List<Serializable> associatedKeys = queryOutput.getRecords().stream().map(r -> r.getValue(associatedTable.getPrimaryKeyField())).toList();
 
-         DeleteInput nextLevelDeleteInput = new DeleteInput();
-         nextLevelDeleteInput.setTransaction(deleteInput.getTransaction());
-         nextLevelDeleteInput.setTableName(association.getAssociatedTableName());
-         nextLevelDeleteInput.setPrimaryKeys(associatedKeys);
-         DeleteOutput nextLevelDeleteOutput = new DeleteAction().execute(nextLevelDeleteInput);
+         if(CollectionUtils.nullSafeHasContents(associatedKeys))
+         {
+            DeleteInput nextLevelDeleteInput = new DeleteInput();
+            nextLevelDeleteInput.setTransaction(deleteInput.getTransaction());
+            nextLevelDeleteInput.setTableName(association.getAssociatedTableName());
+            nextLevelDeleteInput.setPrimaryKeys(associatedKeys);
+            DeleteOutput nextLevelDeleteOutput = new DeleteAction().execute(nextLevelDeleteInput);
+         }
       }
    }
 
@@ -170,12 +413,9 @@ public class DeleteAction
    /*******************************************************************************
     **
     *******************************************************************************/
-   private static List<QRecord> getRecordListForAuditIfNeeded(DeleteInput deleteInput) throws QException
+   private static Optional<List<QRecord>> fetchOldRecords(DeleteInput deleteInput, DeleteInterface deleteInterface) throws QException
    {
-      List<QRecord> recordListForAudit = null;
-
-      AuditLevel auditLevel = DMLAuditAction.getAuditLevel(deleteInput);
-      if(AuditLevel.RECORD.equals(auditLevel) || AuditLevel.FIELD.equals(auditLevel))
+      if(deleteInterface.supportsPreFetchQuery())
       {
          List<Serializable> primaryKeyList = deleteInput.getPrimaryKeys();
          if(CollectionUtils.nullSafeIsEmpty(deleteInput.getPrimaryKeys()) && deleteInput.getQueryFilter() != null)
@@ -185,19 +425,16 @@ public class DeleteAction
 
          if(CollectionUtils.nullSafeHasContents(primaryKeyList))
          {
-            ////////////////////////////////////////////////////////////////////////////////////
-            // always fetch the records - we'll use them anyway for checking not-exist  below //
-            ////////////////////////////////////////////////////////////////////////////////////
             QueryInput queryInput = new QueryInput();
             queryInput.setTransaction(deleteInput.getTransaction());
             queryInput.setTableName(deleteInput.getTableName());
             queryInput.setFilter(new QQueryFilter(new QFilterCriteria(deleteInput.getTable().getPrimaryKeyField(), QCriteriaOperator.IN, primaryKeyList)));
             QueryOutput queryOutput = new QueryAction().execute(queryInput);
-            recordListForAudit = queryOutput.getRecords();
+            return (Optional.of(queryOutput.getRecords()));
          }
       }
 
-      return (recordListForAudit);
+      return (Optional.empty());
    }
 
 
@@ -207,9 +444,9 @@ public class DeleteAction
     ** records that you can't see because of security - that they won't be found
     ** by the query here, so it's the same to you as if they don't exist at all!
     **
-    ** This method, if it finds any missing records, will:
-    ** - remove those ids from the deleteInput
-    ** - create a QRecord with that id and a not-found error message.
+    ** If this method identifies any missing records (e.g., from PKeys that are
+    ** requested to be deleted, but don't exist (or can't be seen)), then it will
+    ** return those as new QRecords, with error messages.
     *******************************************************************************/
    private List<QRecord> validateRecordsExistAndCanBeAccessed(DeleteInput deleteInput, List<QRecord> oldRecordList) throws QException
    {
@@ -218,64 +455,28 @@ public class DeleteAction
       QTableMetaData table           = deleteInput.getTable();
       QFieldMetaData primaryKeyField = table.getField(table.getPrimaryKeyField());
 
-      Set<Serializable> primaryKeysToRemoveFromInput = new HashSet<>();
-
       List<List<Serializable>> pages = CollectionUtils.getPages(deleteInput.getPrimaryKeys(), 1000);
       for(List<Serializable> page : pages)
       {
-         List<Serializable> primaryKeysToLookup = new ArrayList<>();
-         for(Serializable primaryKeyValue : page)
+         Map<Serializable, QRecord> oldRecordMapByPrimaryKey = new HashMap<>();
+         for(QRecord record : oldRecordList)
          {
-            if(primaryKeyValue != null)
-            {
-               primaryKeysToLookup.add(primaryKeyValue);
-            }
-         }
-
-         Map<Serializable, QRecord> lookedUpRecords = new HashMap<>();
-         if(CollectionUtils.nullSafeHasContents(oldRecordList))
-         {
-            for(QRecord record : oldRecordList)
-            {
-               Serializable primaryKeyValue = record.getValue(table.getPrimaryKeyField());
-               primaryKeyValue = ValueUtils.getValueAsFieldType(primaryKeyField.getType(), primaryKeyValue);
-               lookedUpRecords.put(primaryKeyValue, record);
-            }
-         }
-         else if(!primaryKeysToLookup.isEmpty())
-         {
-            QueryInput queryInput = new QueryInput();
-            queryInput.setTransaction(deleteInput.getTransaction());
-            queryInput.setTableName(table.getName());
-            queryInput.setFilter(new QQueryFilter(new QFilterCriteria(table.getPrimaryKeyField(), QCriteriaOperator.IN, primaryKeysToLookup)));
-            QueryOutput queryOutput = new QueryAction().execute(queryInput);
-            for(QRecord record : queryOutput.getRecords())
-            {
-               lookedUpRecords.put(record.getValue(table.getPrimaryKeyField()), record);
-            }
+            Serializable primaryKeyValue = record.getValue(table.getPrimaryKeyField());
+            primaryKeyValue = ValueUtils.getValueAsFieldType(primaryKeyField.getType(), primaryKeyValue);
+            oldRecordMapByPrimaryKey.put(primaryKeyValue, record);
          }
 
          for(Serializable primaryKeyValue : page)
          {
             primaryKeyValue = ValueUtils.getValueAsFieldType(primaryKeyField.getType(), primaryKeyValue);
-            if(!lookedUpRecords.containsKey(primaryKeyValue))
+            if(!oldRecordMapByPrimaryKey.containsKey(primaryKeyValue))
             {
                QRecord recordWithError = new QRecord();
                recordsWithErrors.add(recordWithError);
                recordWithError.setValue(primaryKeyField.getName(), primaryKeyValue);
-               recordWithError.addError(NOT_FOUND_ERROR_PREFIX + " for " + primaryKeyField.getLabel() + " = " + primaryKeyValue);
-               primaryKeysToRemoveFromInput.add(primaryKeyValue);
+               recordWithError.addError(new NotFoundStatusMessage("No record was found to delete for " + primaryKeyField.getLabel() + " = " + primaryKeyValue));
             }
          }
-
-         /////////////////////////////////////////////////////////////////
-         // do one mass removal of any bad keys from the input key list //
-         /////////////////////////////////////////////////////////////////
-         if(!primaryKeysToRemoveFromInput.isEmpty())
-         {
-            deleteInput.getPrimaryKeys().removeAll(primaryKeysToRemoveFromInput);
-            primaryKeysToRemoveFromInput.clear();
-         }
       }
 
       return (recordsWithErrors);

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/GetAction.java

@@ -23,8 +23,6 @@ package com.kingsrook.qqq.backend.core.actions.tables;
 
 
 import java.io.Serializable;
-import java.time.Instant;
-import java.time.temporal.ChronoUnit;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
@@ -33,32 +31,24 @@ import com.kingsrook.qqq.backend.core.actions.customizers.AbstractPostQueryCusto
 import com.kingsrook.qqq.backend.core.actions.customizers.QCodeLoader;
 import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizers;
 import com.kingsrook.qqq.backend.core.actions.interfaces.GetInterface;
+import com.kingsrook.qqq.backend.core.actions.tables.helpers.GetActionCacheHelper;
 import com.kingsrook.qqq.backend.core.actions.values.QPossibleValueTranslator;
 import com.kingsrook.qqq.backend.core.actions.values.QValueFormatter;
+import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
-import com.kingsrook.qqq.backend.core.logging.LogPair;
-import com.kingsrook.qqq.backend.core.logging.QLogger;
-import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetOutput;
-import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
-import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertOutput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput;
-import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
-import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateOutput;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.AdornmentType;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
-import com.kingsrook.qqq.backend.core.model.metadata.tables.cache.CacheUseCase;
 import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleDispatcher;
 import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleInterface;
-import com.kingsrook.qqq.backend.core.modules.backend.implementations.utils.BackendQueryFilterUtils;
-import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
-import com.kingsrook.qqq.backend.core.utils.StringUtils;
-import org.apache.commons.lang.NotImplementedException;
 
 
 /*******************************************************************************
@@ -67,8 +57,6 @@ import org.apache.commons.lang.NotImplementedException;
  *******************************************************************************/
 public class GetAction
 {
-   private static final QLogger LOG = QLogger.getLogger(InsertAction.class);
-
    private Optional<AbstractPostQueryCustomizer> postGetRecordCustomizer;
 
    private GetInput                 getInput;
@@ -76,6 +64,16 @@ public class GetAction
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public QRecord executeForRecord(GetInput getInput) throws QException
+   {
+      return (execute(getInput).getRecord());
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/
@@ -122,51 +120,7 @@ public class GetAction
       ////////////////////////////
       if(table.getCacheOf() != null)
       {
-         if(getOutput.getRecord() == null)
-         {
-            ///////////////////////////////////////////////////////////////////////
-            // if the record wasn't found, see if we should look in cache-source //
-            ///////////////////////////////////////////////////////////////////////
-            QRecord recordFromSource = tryToGetFromCacheSource(getInput, getOutput);
-            if(recordFromSource != null)
-            {
-               QRecord recordToCache     = mapSourceRecordToCacheRecord(table, recordFromSource);
-               boolean shouldCacheRecord = true;
-
-               ////////////////////////////////////////////////////////////////////////////////
-               // see if there are any exclustions that need to be considered for this table //
-               ////////////////////////////////////////////////////////////////////////////////
-               recordMatchExclusionLoop:
-               for(CacheUseCase useCase : CollectionUtils.nonNullList(table.getCacheOf().getUseCases()))
-               {
-                  for(QQueryFilter filter : CollectionUtils.nonNullList(useCase.getExcludeRecordsMatching()))
-                  {
-                     if(BackendQueryFilterUtils.doesRecordMatch(filter, recordToCache))
-                     {
-                        LOG.info("Not caching record because it matches a use case's filter exclusion", new LogPair("record", recordToCache), new LogPair("filter", filter));
-                        shouldCacheRecord = false;
-                        break recordMatchExclusionLoop;
-                     }
-                  }
-               }
-
-               if(shouldCacheRecord)
-               {
-                  InsertInput insertInput = new InsertInput();
-                  insertInput.setTableName(getInput.getTableName());
-                  insertInput.setRecords(List.of(recordToCache));
-                  InsertOutput insertOutput = new InsertAction().execute(insertInput);
-                  getOutput.setRecord(insertOutput.getRecords().get(0));
-               }
-            }
-         }
-         else
-         {
-            /////////////////////////////////////////////////////////////////////////////////
-            // if the record was found, but it's too old, maybe re-fetch from cache source //
-            /////////////////////////////////////////////////////////////////////////////////
-            refreshCacheIfExpired(getInput, getOutput);
-         }
+         new GetActionCacheHelper().handleCaching(getInput, getOutput);
       }
 
       ////////////////////////////////////////////////////////
@@ -183,109 +137,12 @@ public class GetAction
 
 
    /*******************************************************************************
-    **
+    ** Run a GetAction by using the QueryAction instead (e.g., with a filter made
+    ** from the pkey/ukey, and returning the single record if found).
     *******************************************************************************/
-   private static QRecord mapSourceRecordToCacheRecord(QTableMetaData table, QRecord recordFromSource)
+   public GetOutput executeViaQuery(GetInput getInput) throws QException
    {
-      QRecord cacheRecord = new QRecord(recordFromSource);
-      if(StringUtils.hasContent(table.getCacheOf().getCachedDateFieldName()))
-      {
-         cacheRecord.setValue(table.getCacheOf().getCachedDateFieldName(), Instant.now());
-      }
-      return (cacheRecord);
-   }
-
-
-
-   /*******************************************************************************
-    **
-    *******************************************************************************/
-   private void refreshCacheIfExpired(GetInput getInput, GetOutput getOutput) throws QException
-   {
-      QTableMetaData table             = getInput.getTable();
-      Integer        expirationSeconds = table.getCacheOf().getExpirationSeconds();
-      if(expirationSeconds != null)
-      {
-         QRecord cachedRecord = getOutput.getRecord();
-         Instant cachedDate   = cachedRecord.getValueInstant(table.getCacheOf().getCachedDateFieldName());
-         if(cachedDate == null || cachedDate.isBefore(Instant.now().minus(expirationSeconds, ChronoUnit.SECONDS)))
-         {
-            QRecord recordFromSource = tryToGetFromCacheSource(getInput, getOutput);
-            if(recordFromSource != null)
-            {
-               ///////////////////////////////////////////////////////////////////
-               // if the record was found in the source, update it in the cache //
-               ///////////////////////////////////////////////////////////////////
-               QRecord recordToCache = mapSourceRecordToCacheRecord(table, recordFromSource);
-               recordToCache.setValue(table.getPrimaryKeyField(), cachedRecord.getValue(table.getPrimaryKeyField()));
-
-               UpdateInput updateInput = new UpdateInput();
-               updateInput.setTableName(getInput.getTableName());
-               updateInput.setRecords(List.of(recordToCache));
-               UpdateOutput updateOutput = new UpdateAction().execute(updateInput);
-
-               getOutput.setRecord(updateOutput.getRecords().get(0));
-            }
-            else
-            {
-               /////////////////////////////////////////////////////////////////////////////
-               // if the record is no longer in the source, then remove it from the cache //
-               /////////////////////////////////////////////////////////////////////////////
-               DeleteInput deleteInput = new DeleteInput();
-               deleteInput.setTableName(getInput.getTableName());
-               deleteInput.setPrimaryKeys(List.of(getOutput.getRecord().getValue(table.getPrimaryKeyField())));
-               new DeleteAction().execute(deleteInput);
-
-               getOutput.setRecord(null);
-            }
-         }
-      }
-   }
-
-
-
-   /*******************************************************************************
-    **
-    *******************************************************************************/
-   private QRecord tryToGetFromCacheSource(GetInput getInput, GetOutput getOutput) throws QException
-   {
-      QRecord        recordFromSource = null;
-      QTableMetaData table            = getInput.getTable();
-
-      for(CacheUseCase cacheUseCase : CollectionUtils.nonNullList(table.getCacheOf().getUseCases()))
-      {
-         if(CacheUseCase.Type.UNIQUE_KEY_TO_UNIQUE_KEY.equals(cacheUseCase.getType()) && getInput.getUniqueKey() != null)
-         {
-            recordFromSource = getFromCachedSourceForUniqueKeyToUniqueKey(getInput, table.getCacheOf().getSourceTable());
-            break;
-         }
-         else
-         {
-            // todo!!
-            throw new NotImplementedException("Not-yet-implemented cache use case type: " + cacheUseCase.getType());
-         }
-      }
-
-      return (recordFromSource);
-   }
-
-
-
-   /*******************************************************************************
-    **
-    *******************************************************************************/
-   private QRecord getFromCachedSourceForUniqueKeyToUniqueKey(GetInput getInput, String sourceTableName) throws QException
-   {
-      /////////////////////////////////////////////////////
-      // do a Get on the source table, by the unique key //
-      /////////////////////////////////////////////////////
-      GetInput sourceGetInput = new GetInput();
-      sourceGetInput.setTableName(sourceTableName);
-      sourceGetInput.setUniqueKey(getInput.getUniqueKey());
-      GetOutput sourceGetOutput = new GetAction().execute(sourceGetInput);
-      QRecord   outputRecord    = sourceGetOutput.getRecord();
-
-      return (outputRecord);
+      return (new DefaultGetInterface().execute(getInput));
    }
 
 
@@ -298,40 +155,7 @@ public class GetAction
       @Override
       public GetOutput execute(GetInput getInput) throws QException
       {
-         QueryInput queryInput = new QueryInput();
-         queryInput.setTableName(getInput.getTableName());
-
-         //////////////////////////////////////////////////
-         // build filter using either pkey or unique key //
-         //////////////////////////////////////////////////
-         QQueryFilter filter = new QQueryFilter();
-         if(getInput.getPrimaryKey() != null)
-         {
-            filter.addCriteria(new QFilterCriteria(getInput.getTable().getPrimaryKeyField(), QCriteriaOperator.EQUALS, getInput.getPrimaryKey()));
-         }
-         else if(getInput.getUniqueKey() != null)
-         {
-            for(Map.Entry<String, Serializable> entry : getInput.getUniqueKey().entrySet())
-            {
-               if(entry.getValue() == null)
-               {
-                  filter.addCriteria(new QFilterCriteria(entry.getKey(), QCriteriaOperator.IS_BLANK));
-               }
-               else
-               {
-                  filter.addCriteria(new QFilterCriteria(entry.getKey(), QCriteriaOperator.EQUALS, entry.getValue()));
-               }
-            }
-         }
-         else
-         {
-            throw (new QException("No primaryKey or uniqueKey was passed to Get"));
-         }
-
-         queryInput.setFilter(filter);
-         queryInput.setIncludeAssociations(getInput.getIncludeAssociations());
-         queryInput.setAssociationNamesToInclude(getInput.getAssociationNamesToInclude());
-         queryInput.setShouldFetchHeavyFields(getInput.getShouldFetchHeavyFields());
+         QueryInput queryInput = convertGetInputToQueryInput(getInput);
 
          QueryOutput queryOutput = new QueryAction().execute(queryInput);
 
@@ -346,6 +170,48 @@ public class GetAction
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public static QueryInput convertGetInputToQueryInput(GetInput getInput) throws QException
+   {
+      QueryInput queryInput = new QueryInput();
+      queryInput.setTableName(getInput.getTableName());
+
+      //////////////////////////////////////////////////
+      // build filter using either pkey or unique key //
+      //////////////////////////////////////////////////
+      QQueryFilter filter = new QQueryFilter();
+      if(getInput.getPrimaryKey() != null)
+      {
+         filter.addCriteria(new QFilterCriteria(getInput.getTable().getPrimaryKeyField(), QCriteriaOperator.EQUALS, getInput.getPrimaryKey()));
+      }
+      else if(getInput.getUniqueKey() != null)
+      {
+         for(Map.Entry<String, Serializable> entry : getInput.getUniqueKey().entrySet())
+         {
+            if(entry.getValue() == null)
+            {
+               filter.addCriteria(new QFilterCriteria(entry.getKey(), QCriteriaOperator.IS_BLANK));
+            }
+            else
+            {
+               filter.addCriteria(new QFilterCriteria(entry.getKey(), QCriteriaOperator.EQUALS, entry.getValue()));
+            }
+         }
+      }
+      else
+      {
+         throw (new QException("No primaryKey or uniqueKey was passed to Get"));
+      }
+
+      queryInput.setFilter(filter);
+      queryInput.setCommonParamsFrom(getInput);
+      return queryInput;
+   }
+
+
+
    /*******************************************************************************
     ** Run the necessary actions on a record.  This may include setting display values,
     ** translating possible values, and running post-record customizations.
@@ -372,6 +238,32 @@ public class GetAction
          QValueFormatter.setDisplayValuesInRecords(getInput.getTable(), List.of(returnRecord));
       }
 
+      if(getInput.getShouldOmitHiddenFields() || getInput.getShouldMaskPasswords())
+      {
+         Map<String, QFieldMetaData> fields = QContext.getQInstance().getTable(getInput.getTableName()).getFields();
+         for(String fieldName : fields.keySet())
+         {
+            QFieldMetaData field = fields.get(fieldName);
+            if(getInput.getShouldOmitHiddenFields() && field.getIsHidden())
+            {
+               returnRecord.removeValue(fieldName);
+            }
+            else if(getInput.getShouldMaskPasswords() && field.getType() != null && field.getType().needsMasked() && !field.hasAdornmentType(AdornmentType.REVEAL))
+            {
+               //////////////////////////////////////////////////////////////////////
+               // empty out the value completely first (which will remove from     //
+               // display fields as well) then update display value if flag is set //
+               //////////////////////////////////////////////////////////////////////
+               returnRecord.removeValue(fieldName);
+               returnRecord.setValue(fieldName, "************");
+               if(getInput.getShouldGenerateDisplayValues())
+               {
+                  returnRecord.setDisplayValue(fieldName, record.getValueString(fieldName));
+               }
+            }
+         }
+      }
+
       //////////////////////////////////////////////////////////////////////////////
       // note - shouldFetchHeavyFields should be handled by the underlying action //
       //////////////////////////////////////////////////////////////////////////////

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/InsertAction.java

@@ -38,8 +38,10 @@ import com.kingsrook.qqq.backend.core.actions.audits.DMLAuditAction;
 import com.kingsrook.qqq.backend.core.actions.automation.AutomationStatus;
 import com.kingsrook.qqq.backend.core.actions.automation.RecordAutomationStatusUpdater;
 import com.kingsrook.qqq.backend.core.actions.customizers.AbstractPostInsertCustomizer;
+import com.kingsrook.qqq.backend.core.actions.customizers.AbstractPreInsertCustomizer;
 import com.kingsrook.qqq.backend.core.actions.customizers.QCodeLoader;
 import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizers;
+import com.kingsrook.qqq.backend.core.actions.interfaces.InsertInterface;
 import com.kingsrook.qqq.backend.core.actions.tables.helpers.UniqueKeyHelper;
 import com.kingsrook.qqq.backend.core.actions.tables.helpers.ValidateRecordSecurityLockHelper;
 import com.kingsrook.qqq.backend.core.actions.values.ValueBehaviorApplier;
@@ -51,14 +53,18 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertOutput;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
 import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinOn;
 import com.kingsrook.qqq.backend.core.model.metadata.joins.QJoinMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.Association;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.UniqueKey;
+import com.kingsrook.qqq.backend.core.model.statusmessages.BadInputStatusMessage;
+import com.kingsrook.qqq.backend.core.model.statusmessages.QWarningMessage;
 import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleDispatcher;
 import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleInterface;
 import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
+import com.kingsrook.qqq.backend.core.utils.ValueUtils;
 import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
 
 
@@ -72,6 +78,28 @@ public class InsertAction extends AbstractQActionFunction<InsertInput, InsertOut
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public QRecord executeForRecord(InsertInput insertInput) throws QException
+   {
+      InsertOutput insertOutput = new InsertAction().execute(insertInput);
+      return (insertOutput.getRecords().get(0));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public static List<QRecord> executeForRecords(InsertInput insertInput) throws QException
+   {
+      InsertOutput insertOutput = new InsertAction().execute(insertInput);
+      return (insertOutput.getRecords());
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/
@@ -86,41 +114,71 @@ public class InsertAction extends AbstractQActionFunction<InsertInput, InsertOut
          throw (new QException("Error:  Undefined table: " + insertInput.getTableName()));
       }
 
-      Optional<AbstractPostInsertCustomizer> postInsertCustomizer = QCodeLoader.getTableCustomizer(AbstractPostInsertCustomizer.class, table, TableCustomizers.POST_INSERT_RECORD.getRole());
       setAutomationStatusField(insertInput);
 
-      QBackendModuleInterface qModule = getBackendModuleInterface(insertInput);
-      // todo pre-customization - just get to modify the request?
+      //////////////////////////////////////////////////////
+      // load the backend module and its insert interface //
+      //////////////////////////////////////////////////////
+      QBackendModuleInterface qModule         = getBackendModuleInterface(insertInput);
+      InsertInterface         insertInterface = qModule.getInsertInterface();
 
-      ValueBehaviorApplier.applyFieldBehaviors(insertInput.getInstance(), table, insertInput.getRecords());
-      setErrorsIfUniqueKeyErrors(insertInput, table);
-      validateRequiredFields(insertInput);
-      ValidateRecordSecurityLockHelper.validateSecurityFields(insertInput.getTable(), insertInput.getRecords(), ValidateRecordSecurityLockHelper.Action.INSERT);
+      /////////////////////////////
+      // run standard validators //
+      /////////////////////////////
+      performValidations(insertInput, false);
 
-      InsertOutput insertOutput = qModule.getInsertInterface().execute(insertInput);
-      List<String> errors       = insertOutput.getRecords().stream().flatMap(r -> r.getErrors().stream()).toList();
+      ////////////////////////////////////
+      // have the backend do the insert //
+      ////////////////////////////////////
+      InsertOutput insertOutput = insertInterface.execute(insertInput);
+
+      //////////////////////////////
+      // log if there were errors //
+      //////////////////////////////
+      List<String> errors = insertOutput.getRecords().stream().flatMap(r -> r.getErrors().stream().map(Object::toString)).toList();
       if(CollectionUtils.nullSafeHasContents(errors))
       {
-         LOG.warn("Errors in insertAction", logPair("tableName", table.getName()), logPair("errorCount", errors.size()), errors.size() < 10 ? logPair("errors", errors) : logPair("first10Errors", errors.subList(0, 10)));
+         LOG.info("Errors in insertAction", logPair("tableName", table.getName()), logPair("errorCount", errors.size()), errors.size() < 10 ? logPair("errors", errors) : logPair("first10Errors", errors.subList(0, 10)));
       }
 
+      //////////////////////////////////////////////////
+      // insert any associations in the input records //
+      //////////////////////////////////////////////////
       manageAssociations(table, insertOutput.getRecords(), insertInput.getTransaction());
 
-      // todo post-customization - can do whatever w/ the result if you want
-
+      //////////////////
+      // do the audit //
+      //////////////////
       if(insertInput.getOmitDmlAudit())
       {
          LOG.debug("Requested to omit DML audit");
       }
       else
       {
-         new DMLAuditAction().execute(new DMLAuditInput().withTableActionInput(insertInput).withRecordList(insertOutput.getRecords()));
+         new DMLAuditAction().execute(new DMLAuditInput()
+            .withTableActionInput(insertInput)
+            .withAuditContext(insertInput.getAuditContext())
+            .withRecordList(insertOutput.getRecords()));
       }
 
+      //////////////////////////////////////////////////////////////
+      // finally, run the post-insert customizer, if there is one //
+      //////////////////////////////////////////////////////////////
+      Optional<AbstractPostInsertCustomizer> postInsertCustomizer = QCodeLoader.getTableCustomizer(AbstractPostInsertCustomizer.class, table, TableCustomizers.POST_INSERT_RECORD.getRole());
       if(postInsertCustomizer.isPresent())
       {
-         postInsertCustomizer.get().setInsertInput(insertInput);
-         insertOutput.setRecords(postInsertCustomizer.get().apply(insertOutput.getRecords()));
+         try
+         {
+            postInsertCustomizer.get().setInsertInput(insertInput);
+            insertOutput.setRecords(postInsertCustomizer.get().apply(insertOutput.getRecords()));
+         }
+         catch(Exception e)
+         {
+            for(QRecord record : insertOutput.getRecords())
+            {
+               record.addWarning(new QWarningMessage("An error occurred after the insert: " + e.getMessage()));
+            }
+         }
       }
 
       return insertOutput;
@@ -128,6 +186,88 @@ public class InsertAction extends AbstractQActionFunction<InsertInput, InsertOut
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public void performValidations(InsertInput insertInput, boolean isPreview) throws QException
+   {
+      QTableMetaData table = insertInput.getTable();
+
+      ///////////////////////////////////////////////////////////////////
+      // load the pre-insert customizer and set it up, if there is one //
+      // then we'll run it based on its WhenToRun value                //
+      ///////////////////////////////////////////////////////////////////
+      Optional<AbstractPreInsertCustomizer> preInsertCustomizer = QCodeLoader.getTableCustomizer(AbstractPreInsertCustomizer.class, table, TableCustomizers.PRE_INSERT_RECORD.getRole());
+      if(preInsertCustomizer.isPresent())
+      {
+         preInsertCustomizer.get().setInsertInput(insertInput);
+         preInsertCustomizer.get().setIsPreview(isPreview);
+         runPreInsertCustomizerIfItIsTime(insertInput, preInsertCustomizer, AbstractPreInsertCustomizer.WhenToRun.BEFORE_ALL_VALIDATIONS);
+      }
+
+      setDefaultValuesInRecords(table, insertInput.getRecords());
+
+      ValueBehaviorApplier.applyFieldBehaviors(insertInput.getInstance(), table, insertInput.getRecords());
+
+      runPreInsertCustomizerIfItIsTime(insertInput, preInsertCustomizer, AbstractPreInsertCustomizer.WhenToRun.BEFORE_UNIQUE_KEY_CHECKS);
+      setErrorsIfUniqueKeyErrors(insertInput, table);
+
+      runPreInsertCustomizerIfItIsTime(insertInput, preInsertCustomizer, AbstractPreInsertCustomizer.WhenToRun.BEFORE_REQUIRED_FIELD_CHECKS);
+      if(insertInput.getInputSource().shouldValidateRequiredFields())
+      {
+         validateRequiredFields(insertInput);
+      }
+
+      runPreInsertCustomizerIfItIsTime(insertInput, preInsertCustomizer, AbstractPreInsertCustomizer.WhenToRun.BEFORE_SECURITY_CHECKS);
+      ValidateRecordSecurityLockHelper.validateSecurityFields(insertInput.getTable(), insertInput.getRecords(), ValidateRecordSecurityLockHelper.Action.INSERT);
+
+      runPreInsertCustomizerIfItIsTime(insertInput, preInsertCustomizer, AbstractPreInsertCustomizer.WhenToRun.AFTER_ALL_VALIDATIONS);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private void setDefaultValuesInRecords(QTableMetaData table, List<QRecord> records)
+   {
+      ////////////////////////////////////////////////////////////////////////////////////////////////
+      // for all fields in the table - if any have a default value, then look at all input records, //
+      // and if they have null value, then apply the default                                        //
+      ////////////////////////////////////////////////////////////////////////////////////////////////
+      for(QFieldMetaData field : table.getFields().values())
+      {
+         if(field.getDefaultValue() != null)
+         {
+            for(QRecord record : records)
+            {
+               if(record.getValue(field.getName()) == null)
+               {
+                  record.setValue(field.getName(), field.getDefaultValue());
+               }
+            }
+         }
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private void runPreInsertCustomizerIfItIsTime(InsertInput insertInput, Optional<AbstractPreInsertCustomizer> preInsertCustomizer, AbstractPreInsertCustomizer.WhenToRun whenToRun) throws QException
+   {
+      if(preInsertCustomizer.isPresent())
+      {
+         if(whenToRun.equals(preInsertCustomizer.get().getWhenToRun()))
+         {
+            insertInput.setRecords(preInsertCustomizer.get().apply(insertInput.getRecords()));
+         }
+      }
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/
@@ -146,7 +286,7 @@ public class InsertAction extends AbstractQActionFunction<InsertInput, InsertOut
             {
                if(record.getValue(requiredField.getName()) == null || (requiredField.getType().isStringLike() && record.getValueString(requiredField.getName()).trim().equals("")))
                {
-                  record.addError("Missing value in required field: " + requiredField.getLabel());
+                  record.addError(new BadInputStatusMessage("Missing value in required field: " + requiredField.getLabel()));
                }
             }
          }
@@ -169,24 +309,33 @@ public class InsertAction extends AbstractQActionFunction<InsertInput, InsertOut
          List<QRecord> nextLevelInserts = new ArrayList<>();
          for(QRecord record : insertedRecords)
          {
+            if(CollectionUtils.nullSafeHasContents(record.getErrors()))
+            {
+               continue;
+            }
+
             if(record.getAssociatedRecords() != null && record.getAssociatedRecords().containsKey(association.getName()))
             {
                for(QRecord associatedRecord : CollectionUtils.nonNullList(record.getAssociatedRecords().get(association.getName())))
                {
                   for(JoinOn joinOn : join.getJoinOns())
                   {
-                     associatedRecord.setValue(joinOn.getRightField(), record.getValue(joinOn.getLeftField()));
+                     QFieldType type = table.getField(joinOn.getLeftField()).getType();
+                     associatedRecord.setValue(joinOn.getRightField(), ValueUtils.getValueAsFieldType(type, record.getValue(joinOn.getLeftField())));
                   }
                   nextLevelInserts.add(associatedRecord);
                }
             }
          }
 
-         InsertInput nextLevelInsertInput = new InsertInput();
-         nextLevelInsertInput.setTransaction(transaction);
-         nextLevelInsertInput.setTableName(association.getAssociatedTableName());
-         nextLevelInsertInput.setRecords(nextLevelInserts);
-         InsertOutput nextLevelInsertOutput = new InsertAction().execute(nextLevelInsertInput);
+         if(CollectionUtils.nullSafeHasContents(nextLevelInserts))
+         {
+            InsertInput nextLevelInsertInput = new InsertInput();
+            nextLevelInsertInput.setTransaction(transaction);
+            nextLevelInsertInput.setTableName(association.getAssociatedTableName());
+            nextLevelInsertInput.setRecords(nextLevelInserts);
+            InsertOutput nextLevelInsertOutput = new InsertAction().execute(nextLevelInsertInput);
+         }
       }
    }
 
@@ -232,7 +381,7 @@ public class InsertAction extends AbstractQActionFunction<InsertInput, InsertOut
                Optional<List<Serializable>> keyValues = UniqueKeyHelper.getKeyValues(table, uniqueKey, record);
                if(keyValues.isPresent() && (existingKeys.get(uniqueKey).contains(keyValues.get()) || keysInThisList.get(uniqueKey).contains(keyValues.get())))
                {
-                  record.addError("Another record already exists with this " + uniqueKey.getDescription(table));
+                  record.addError(new BadInputStatusMessage("Another record already exists with this " + uniqueKey.getDescription(table)));
                   foundDupe = true;
                   break;
                }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/QueryAction.java

@@ -27,13 +27,18 @@ import java.util.ArrayList;
 import java.util.Collection;
 import java.util.HashSet;
 import java.util.List;
+import java.util.Map;
 import java.util.Optional;
 import java.util.Set;
 import com.kingsrook.qqq.backend.core.actions.ActionHelper;
 import com.kingsrook.qqq.backend.core.actions.customizers.AbstractPostQueryCustomizer;
 import com.kingsrook.qqq.backend.core.actions.customizers.QCodeLoader;
 import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizers;
+import com.kingsrook.qqq.backend.core.actions.interfaces.QueryInterface;
 import com.kingsrook.qqq.backend.core.actions.reporting.BufferedRecordPipe;
+import com.kingsrook.qqq.backend.core.actions.reporting.RecordPipeBufferedWrapper;
+import com.kingsrook.qqq.backend.core.actions.tables.helpers.QueryActionCacheHelper;
+import com.kingsrook.qqq.backend.core.actions.tables.helpers.QueryStatManager;
 import com.kingsrook.qqq.backend.core.actions.values.QPossibleValueTranslator;
 import com.kingsrook.qqq.backend.core.actions.values.QValueFormatter;
 import com.kingsrook.qqq.backend.core.context.QContext;
@@ -45,14 +50,19 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.AdornmentType;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinOn;
 import com.kingsrook.qqq.backend.core.model.metadata.joins.QJoinMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.Association;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+import com.kingsrook.qqq.backend.core.model.querystats.QueryStat;
 import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleDispatcher;
 import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleInterface;
 import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
 import com.kingsrook.qqq.backend.core.utils.ListingHash;
+import com.kingsrook.qqq.backend.core.utils.ValueUtils;
 
 
 /*******************************************************************************
@@ -66,6 +76,7 @@ public class QueryAction
    private Optional<AbstractPostQueryCustomizer> postQueryRecordCustomizer;
 
    private QueryInput               queryInput;
+   private QueryInterface           queryInterface;
    private QPossibleValueTranslator qPossibleValueTranslator;
 
 
@@ -77,27 +88,53 @@ public class QueryAction
    {
       ActionHelper.validateSession(queryInput);
 
-      postQueryRecordCustomizer = QCodeLoader.getTableCustomizer(AbstractPostQueryCustomizer.class, queryInput.getTable(), TableCustomizers.POST_QUERY_RECORD.getRole());
+      if(queryInput.getTableName() == null)
+      {
+         throw (new QException("Table name was not specified in query input"));
+      }
+
+      QTableMetaData table = queryInput.getTable();
+      if(table == null)
+      {
+         throw (new QException("A table named [" + queryInput.getTableName() + "] was not found in the active QInstance"));
+      }
+
+      QBackendMetaData backend = queryInput.getBackend();
+      postQueryRecordCustomizer = QCodeLoader.getTableCustomizer(AbstractPostQueryCustomizer.class, table, TableCustomizers.POST_QUERY_RECORD.getRole());
       this.queryInput = queryInput;
 
       if(queryInput.getRecordPipe() != null)
       {
          queryInput.getRecordPipe().setPostRecordActions(this::postRecordActions);
+
+         if(queryInput.getIncludeAssociations())
+         {
+            //////////////////////////////////////////////////////////////////////////////////////////
+            // if the user requested to include associations, it's important that that is buffered, //
+            // (for performance reasons), so, wrap the user's pipe with a buffer                    //
+            //////////////////////////////////////////////////////////////////////////////////////////
+            queryInput.setRecordPipe(new RecordPipeBufferedWrapper(queryInput.getRecordPipe()));
+         }
       }
 
-      if(queryInput.getIncludeAssociations() && queryInput.getRecordPipe() != null)
-      {
-         //////////////////////////////////////////////
-         // todo - support this in the future maybe? //
-         //////////////////////////////////////////////
-         throw (new QException("Associations may not be fetched into a RecordPipe."));
-      }
+      QueryStat queryStat = QueryStatManager.newQueryStat(backend, table, queryInput.getFilter());
 
       QBackendModuleDispatcher qBackendModuleDispatcher = new QBackendModuleDispatcher();
-      QBackendModuleInterface  qModule                  = qBackendModuleDispatcher.getQBackendModule(queryInput.getBackend());
-      // todo pre-customization - just get to modify the request?
-      QueryOutput queryOutput = qModule.getQueryInterface().execute(queryInput);
-      // todo post-customization - can do whatever w/ the result if you want
+      QBackendModuleInterface  qModule                  = qBackendModuleDispatcher.getQBackendModule(backend);
+
+      queryInterface = qModule.getQueryInterface();
+      queryInterface.setQueryStat(queryStat);
+      QueryOutput queryOutput = queryInterface.execute(queryInput);
+
+      QueryStatManager.getInstance().add(queryStat);
+
+      ////////////////////////////
+      // handle cache use-cases //
+      ////////////////////////////
+      if(table.getCacheOf() != null)
+      {
+         new QueryActionCacheHelper().handleCaching(queryInput, queryOutput);
+      }
 
       if(queryInput.getRecordPipe() instanceof BufferedRecordPipe bufferedRecordPipe)
       {
@@ -109,11 +146,6 @@ public class QueryAction
          postRecordActions(queryOutput.getRecords());
       }
 
-      if(queryInput.getIncludeAssociations())
-      {
-         manageAssociations(queryInput, queryOutput);
-      }
-
       return queryOutput;
    }
 
@@ -122,7 +154,7 @@ public class QueryAction
    /*******************************************************************************
     **
     *******************************************************************************/
-   private void manageAssociations(QueryInput queryInput, QueryOutput queryOutput) throws QException
+   private void manageAssociations(QueryInput queryInput, List<QRecord> queryOutputRecords) throws QException
    {
       QTableMetaData table = queryInput.getTable();
       for(Association association : CollectionUtils.nonNullList(table.getAssociations()))
@@ -147,11 +179,12 @@ public class QueryAction
             {
                JoinOn            joinOn = join.getJoinOns().get(0);
                Set<Serializable> values = new HashSet<>();
-               for(QRecord record : queryOutput.getRecords())
+               for(QRecord record : queryOutputRecords)
                {
-                  Serializable value = record.getValue(joinOn.getLeftField());
-                  values.add(value);
-                  outerResultMap.add(List.of(value), record);
+                  Serializable value       = record.getValue(joinOn.getLeftField());
+                  Serializable valueAsType = ValueUtils.getValueAsFieldType(table.getField(joinOn.getLeftField()).getType(), value);
+                  values.add(valueAsType);
+                  outerResultMap.add(List.of(valueAsType), record);
                }
                filter.addCriteria(new QFilterCriteria(joinOn.getRightField(), QCriteriaOperator.IN, new ArrayList<>(values)));
             }
@@ -159,7 +192,7 @@ public class QueryAction
             {
                filter.setBooleanOperator(QQueryFilter.BooleanOperator.OR);
 
-               for(QRecord record : queryOutput.getRecords())
+               for(QRecord record : queryOutputRecords)
                {
                   QQueryFilter subFilter = new QQueryFilter();
                   filter.addSubFilter(subFilter);
@@ -227,7 +260,7 @@ public class QueryAction
     ** not one created via List.of()).  This may include setting display values,
     ** translating possible values, and running post-record customizations.
     *******************************************************************************/
-   public void postRecordActions(List<QRecord> records)
+   public void postRecordActions(List<QRecord> records) throws QException
    {
       if(this.postQueryRecordCustomizer.isPresent())
       {
@@ -247,5 +280,80 @@ public class QueryAction
       {
          QValueFormatter.setDisplayValuesInRecords(queryInput.getTable(), records);
       }
+
+      if(queryInput.getIncludeAssociations())
+      {
+         manageAssociations(queryInput, records);
+      }
+
+      //////////////////////////////
+      // mask any password fields //
+      //////////////////////////////
+      if(queryInput.getShouldOmitHiddenFields() || queryInput.getShouldMaskPasswords())
+      {
+         Set<String> maskedFields = new HashSet<>();
+         Set<String> hiddenFields = new HashSet<>();
+
+         //////////////////////////////////////////////////
+         // build up sets of passwords and hidden fields //
+         //////////////////////////////////////////////////
+         Map<String, QFieldMetaData> fields = QContext.getQInstance().getTable(queryInput.getTableName()).getFields();
+         for(String fieldName : fields.keySet())
+         {
+            QFieldMetaData field = fields.get(fieldName);
+            if(queryInput.getShouldOmitHiddenFields() && field.getIsHidden())
+            {
+               hiddenFields.add(fieldName);
+            }
+            else if(queryInput.getShouldMaskPasswords() && field.getType() != null && field.getType().needsMasked() && !field.hasAdornmentType(AdornmentType.REVEAL))
+            {
+               maskedFields.add(fieldName);
+            }
+         }
+
+         /////////////////////////////////////////////////////
+         // iterate over records replacing values with mask //
+         /////////////////////////////////////////////////////
+         for(QRecord record : records)
+         {
+            /////////////////////////
+            // clear hidden fields //
+            /////////////////////////
+            for(String hiddenFieldName : hiddenFields)
+            {
+               record.removeValue(hiddenFieldName);
+            }
+
+            for(String maskedFieldName : maskedFields)
+            {
+               //////////////////////////////////////////////////////////////////////
+               // empty out the value completely first (which will remove from     //
+               // display fields as well) then update display value if flag is set //
+               //////////////////////////////////////////////////////////////////////
+               record.removeValue(maskedFieldName);
+               record.setValue(maskedFieldName, "************");
+               if(queryInput.getShouldGenerateDisplayValues())
+               {
+                  record.setDisplayValue(maskedFieldName, record.getValueString(maskedFieldName));
+               }
+            }
+         }
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public void cancel()
+   {
+      if(queryInterface == null)
+      {
+         LOG.warn("queryInterface object was null when requested to cancel");
+         return;
+      }
+
+      queryInterface.cancelAction();
    }
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/ReplaceAction.java

@@ -0,0 +1,179 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2023.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.actions.tables;
+
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import com.kingsrook.qqq.backend.core.actions.AbstractQActionFunction;
+import com.kingsrook.qqq.backend.core.actions.QBackendTransaction;
+import com.kingsrook.qqq.backend.core.actions.tables.helpers.UniqueKeyHelper;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.logging.QLogger;
+import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteOutput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertOutput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
+import com.kingsrook.qqq.backend.core.model.actions.tables.replace.ReplaceInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.replace.ReplaceOutput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateOutput;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.UniqueKey;
+import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
+
+
+/*******************************************************************************
+ ** Action to do a "replace" - e.g: Update rows with unique-key values that are
+ ** already in the table; insert rows whose unique keys weren't already in the
+ ** table, and delete rows that weren't in the input (all based on a
+ ** UniqueKey that's part of the input)
+ **
+ ** Note - the filter in the ReplaceInput - its role is to limit what rows are
+ ** potentially deleted.  e.g., if you have a table that's segmented, and you're
+ ** only replacing a particular segment of it (say, for 1 client), then you pass
+ ** in a filter that finds  rows matching that segment.  See Test for example.
+ *******************************************************************************/
+public class ReplaceAction extends AbstractQActionFunction<ReplaceInput, ReplaceOutput>
+{
+   private static final QLogger LOG = QLogger.getLogger(ReplaceAction.class);
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public ReplaceOutput execute(ReplaceInput input) throws QException
+   {
+      ReplaceOutput output = new ReplaceOutput();
+
+      QBackendTransaction transaction         = input.getTransaction();
+      boolean             weOwnTheTransaction = false;
+
+      try
+      {
+         QTableMetaData table           = input.getTable();
+         UniqueKey      uniqueKey       = input.getKey();
+         String         primaryKeyField = table.getPrimaryKeyField();
+         if(transaction == null)
+         {
+            InsertInput insertInput = new InsertInput();
+            insertInput.setTableName(input.getTableName());
+            transaction = new InsertAction().openTransaction(insertInput);
+            weOwnTheTransaction = true;
+         }
+
+         List<QRecord>      insertList        = new ArrayList<>();
+         List<QRecord>      updateList        = new ArrayList<>();
+         List<Serializable> primaryKeysToKeep = new ArrayList<>();
+
+         for(List<QRecord> page : CollectionUtils.getPages(input.getRecords(), 1000))
+         {
+            ///////////////////////////////////////////////////////////////////////////////////
+            // originally it was thought that we'd need to pass the filter in here           //
+            // but, it's been decided not to.  the filter only applies to what we can delete //
+            ///////////////////////////////////////////////////////////////////////////////////
+            Map<List<Serializable>, Serializable> existingKeys = UniqueKeyHelper.getExistingKeys(transaction, table, page, uniqueKey);
+            for(QRecord record : page)
+            {
+               Optional<List<Serializable>> keyValues = UniqueKeyHelper.getKeyValues(table, uniqueKey, record);
+               if(keyValues.isPresent())
+               {
+                  if(existingKeys.containsKey(keyValues.get()))
+                  {
+                     Serializable primaryKey = existingKeys.get(keyValues.get());
+                     record.setValue(primaryKeyField, primaryKey);
+                     updateList.add(record);
+                     primaryKeysToKeep.add(primaryKey);
+                  }
+                  else
+                  {
+                     insertList.add(record);
+                  }
+               }
+            }
+         }
+
+         InsertInput insertInput = new InsertInput();
+         insertInput.setTableName(table.getName());
+         insertInput.setRecords(insertList);
+         insertInput.setTransaction(transaction);
+         insertInput.setOmitDmlAudit(input.getOmitDmlAudit());
+         InsertOutput insertOutput = new InsertAction().execute(insertInput);
+         primaryKeysToKeep.addAll(insertOutput.getRecords().stream().map(r -> r.getValue(primaryKeyField)).toList());
+         output.setInsertOutput(insertOutput);
+
+         UpdateInput updateInput = new UpdateInput();
+         updateInput.setTableName(table.getName());
+         updateInput.setRecords(updateList);
+         updateInput.setTransaction(transaction);
+         updateInput.setOmitDmlAudit(input.getOmitDmlAudit());
+         UpdateOutput updateOutput = new UpdateAction().execute(updateInput);
+         output.setUpdateOutput(updateOutput);
+
+         QQueryFilter deleteFilter = new QQueryFilter(new QFilterCriteria(primaryKeyField, QCriteriaOperator.NOT_IN, primaryKeysToKeep));
+         if(input.getFilter() != null)
+         {
+            deleteFilter.addSubFilter(input.getFilter());
+         }
+
+         DeleteInput deleteInput = new DeleteInput();
+         deleteInput.setTableName(table.getName());
+         deleteInput.setQueryFilter(deleteFilter);
+         deleteInput.setTransaction(transaction);
+         deleteInput.setOmitDmlAudit(input.getOmitDmlAudit());
+         DeleteOutput deleteOutput = new DeleteAction().execute(deleteInput);
+         output.setDeleteOutput(deleteOutput);
+
+         if(weOwnTheTransaction)
+         {
+            transaction.commit();
+         }
+
+         return (output);
+      }
+      catch(Exception e)
+      {
+         if(weOwnTheTransaction)
+         {
+            LOG.warn("Caught top-level ReplaceAction exception - rolling back exception", e);
+            transaction.rollback();
+         }
+         throw (new QException("Error executing replace action", e));
+      }
+      finally
+      {
+         if(weOwnTheTransaction)
+         {
+            transaction.close();
+         }
+      }
+   }
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/UpdateAction.java

@@ -27,12 +27,18 @@ import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Optional;
 import java.util.Set;
 import java.util.stream.Collectors;
 import com.kingsrook.qqq.backend.core.actions.ActionHelper;
 import com.kingsrook.qqq.backend.core.actions.audits.DMLAuditAction;
 import com.kingsrook.qqq.backend.core.actions.automation.AutomationStatus;
 import com.kingsrook.qqq.backend.core.actions.automation.RecordAutomationStatusUpdater;
+import com.kingsrook.qqq.backend.core.actions.customizers.AbstractPostUpdateCustomizer;
+import com.kingsrook.qqq.backend.core.actions.customizers.AbstractPreUpdateCustomizer;
+import com.kingsrook.qqq.backend.core.actions.customizers.QCodeLoader;
+import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizers;
+import com.kingsrook.qqq.backend.core.actions.interfaces.UpdateInterface;
 import com.kingsrook.qqq.backend.core.actions.tables.helpers.ValidateRecordSecurityLockHelper;
 import com.kingsrook.qqq.backend.core.actions.values.ValueBehaviorApplier;
 import com.kingsrook.qqq.backend.core.context.QContext;
@@ -51,12 +57,17 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateOutput;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
-import com.kingsrook.qqq.backend.core.model.metadata.audits.AuditLevel;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
 import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinOn;
 import com.kingsrook.qqq.backend.core.model.metadata.joins.QJoinMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.security.RecordSecurityLock;
+import com.kingsrook.qqq.backend.core.model.metadata.security.RecordSecurityLockFilters;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.Association;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+import com.kingsrook.qqq.backend.core.model.statusmessages.BadInputStatusMessage;
+import com.kingsrook.qqq.backend.core.model.statusmessages.NotFoundStatusMessage;
+import com.kingsrook.qqq.backend.core.model.statusmessages.QWarningMessage;
 import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleDispatcher;
 import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleInterface;
 import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
@@ -72,7 +83,27 @@ public class UpdateAction
 {
    private static final QLogger LOG = QLogger.getLogger(UpdateAction.class);
 
-   public static final String NOT_FOUND_ERROR_PREFIX = "No record was found to update";
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public QRecord executeForRecord(UpdateInput updateInput) throws QException
+   {
+      UpdateOutput updateOutput = new UpdateAction().execute(updateInput);
+      return (updateOutput.getRecords().get(0));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public static List<QRecord> executeForRecords(UpdateInput updateInput) throws QException
+   {
+      UpdateOutput updateOutput = new UpdateAction().execute(updateInput);
+      return (updateOutput.getRecords());
+   }
 
 
 
@@ -84,38 +115,78 @@ public class UpdateAction
       ActionHelper.validateSession(updateInput);
       setAutomationStatusField(updateInput);
 
-      ValueBehaviorApplier.applyFieldBehaviors(updateInput.getInstance(), updateInput.getTable(), updateInput.getRecords());
-      // todo - need to handle records with errors coming out of here...
-
-      List<QRecord> oldRecordList = getOldRecordListForAuditIfNeeded(updateInput);
+      QTableMetaData table = updateInput.getTable();
 
+      //////////////////////////////////////////////////////
+      // load the backend module and its update interface //
+      //////////////////////////////////////////////////////
       QBackendModuleDispatcher qBackendModuleDispatcher = new QBackendModuleDispatcher();
       QBackendModuleInterface  qModule                  = qBackendModuleDispatcher.getQBackendModule(updateInput.getBackend());
+      UpdateInterface          updateInterface          = qModule.getUpdateInterface();
 
-      validatePrimaryKeysAreGiven(updateInput);
-      validateRecordsExistAndCanBeAccessed(updateInput, oldRecordList);
-      validateRequiredFields(updateInput);
-      ValidateRecordSecurityLockHelper.validateSecurityFields(updateInput.getTable(), updateInput.getRecords(), ValidateRecordSecurityLockHelper.Action.UPDATE);
+      ////////////////////////////////////////////////////////////////////////////////
+      // fetch the old list of records (if the backend supports it), for audits,    //
+      // for "not-found detection", and for the pre-action to use (if there is one) //
+      ////////////////////////////////////////////////////////////////////////////////
+      Optional<List<QRecord>> oldRecordList = fetchOldRecords(updateInput, updateInterface);
 
-      // todo pre-customization - just get to modify the request?
-      UpdateOutput updateOutput = qModule.getUpdateInterface().execute(updateInput);
-      // todo post-customization - can do whatever w/ the result if you want
+      performValidations(updateInput, oldRecordList, false);
 
-      List<String> errors = updateOutput.getRecords().stream().flatMap(r -> r.getErrors().stream()).toList();
+      ////////////////////////////////////
+      // have the backend do the update //
+      ////////////////////////////////////
+      UpdateOutput updateOutput = updateInterface.execute(updateInput);
+
+      //////////////////////////////
+      // log if there were errors //
+      //////////////////////////////
+      List<String> errors = updateOutput.getRecords().stream().flatMap(r -> r.getErrors().stream().map(Object::toString)).toList();
       if(CollectionUtils.nullSafeHasContents(errors))
       {
-         LOG.warn("Errors in updateAction", logPair("tableName", updateInput.getTableName()), logPair("errorCount", errors.size()), errors.size() < 10 ? logPair("errors", errors) : logPair("first10Errors", errors.subList(0, 10)));
+         LOG.info("Errors in updateAction", logPair("tableName", updateInput.getTableName()), logPair("errorCount", errors.size()), errors.size() < 10 ? logPair("errors", errors) : logPair("first10Errors", errors.subList(0, 10)));
       }
 
+      /////////////////////////////////////////////////////////////////////////////////////
+      // update (inserting and deleting as needed) any associations in the input records //
+      /////////////////////////////////////////////////////////////////////////////////////
       manageAssociations(updateInput);
 
+      //////////////////
+      // do the audit //
+      //////////////////
       if(updateInput.getOmitDmlAudit())
       {
          LOG.debug("Requested to omit DML audit");
       }
       else
       {
-         new DMLAuditAction().execute(new DMLAuditInput().withTableActionInput(updateInput).withRecordList(updateOutput.getRecords()).withOldRecordList(oldRecordList));
+         DMLAuditInput dmlAuditInput = new DMLAuditInput()
+            .withTableActionInput(updateInput)
+            .withRecordList(updateOutput.getRecords())
+            .withAuditContext(updateInput.getAuditContext());
+         oldRecordList.ifPresent(l -> dmlAuditInput.setOldRecordList(l));
+         new DMLAuditAction().execute(dmlAuditInput);
+      }
+
+      //////////////////////////////////////////////////////////////
+      // finally, run the post-update customizer, if there is one //
+      //////////////////////////////////////////////////////////////
+      Optional<AbstractPostUpdateCustomizer> postUpdateCustomizer = QCodeLoader.getTableCustomizer(AbstractPostUpdateCustomizer.class, table, TableCustomizers.POST_UPDATE_RECORD.getRole());
+      if(postUpdateCustomizer.isPresent())
+      {
+         try
+         {
+            postUpdateCustomizer.get().setUpdateInput(updateInput);
+            oldRecordList.ifPresent(l -> postUpdateCustomizer.get().setOldRecordList(l));
+            updateOutput.setRecords(postUpdateCustomizer.get().apply(updateOutput.getRecords()));
+         }
+         catch(Exception e)
+         {
+            for(QRecord record : updateOutput.getRecords())
+            {
+               record.addWarning(new QWarningMessage("An error occurred after the update: " + e.getMessage()));
+            }
+         }
       }
 
       return updateOutput;
@@ -123,6 +194,73 @@ public class UpdateAction
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public void performValidations(UpdateInput updateInput, Optional<List<QRecord>> oldRecordList, boolean isPreview) throws QException
+   {
+      QTableMetaData table = updateInput.getTable();
+
+      /////////////////////////////
+      // run standard validators //
+      /////////////////////////////
+      ValueBehaviorApplier.applyFieldBehaviors(updateInput.getInstance(), table, updateInput.getRecords());
+      validatePrimaryKeysAreGiven(updateInput);
+
+      if(oldRecordList.isPresent())
+      {
+         validateRecordsExistAndCanBeAccessed(updateInput, oldRecordList.get());
+      }
+      else
+      {
+         ValidateRecordSecurityLockHelper.validateSecurityFields(table, updateInput.getRecords(), ValidateRecordSecurityLockHelper.Action.UPDATE);
+      }
+
+      if(updateInput.getInputSource().shouldValidateRequiredFields())
+      {
+         validateRequiredFields(updateInput);
+      }
+
+      ///////////////////////////////////////////////////////////////////////////
+      // after all validations, run the pre-update customizer, if there is one //
+      ///////////////////////////////////////////////////////////////////////////
+      Optional<AbstractPreUpdateCustomizer> preUpdateCustomizer = QCodeLoader.getTableCustomizer(AbstractPreUpdateCustomizer.class, table, TableCustomizers.PRE_UPDATE_RECORD.getRole());
+      if(preUpdateCustomizer.isPresent())
+      {
+         preUpdateCustomizer.get().setUpdateInput(updateInput);
+         preUpdateCustomizer.get().setIsPreview(isPreview);
+         oldRecordList.ifPresent(l -> preUpdateCustomizer.get().setOldRecordList(l));
+         updateInput.setRecords(preUpdateCustomizer.get().apply(updateInput.getRecords()));
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private Optional<List<QRecord>> fetchOldRecords(UpdateInput updateInput, UpdateInterface updateInterface) throws QException
+   {
+      if(updateInterface.supportsPreFetchQuery())
+      {
+         String             primaryKeyField   = updateInput.getTable().getPrimaryKeyField();
+         List<Serializable> pkeysBeingUpdated = CollectionUtils.nonNullList(updateInput.getRecords()).stream().map(r -> r.getValue(primaryKeyField)).toList();
+
+         QueryInput queryInput = new QueryInput();
+         queryInput.setTransaction(updateInput.getTransaction());
+         queryInput.setTableName(updateInput.getTableName());
+         queryInput.setFilter(new QQueryFilter(new QFilterCriteria(primaryKeyField, QCriteriaOperator.IN, pkeysBeingUpdated)));
+         // todo - need a limit?  what if too many??
+         QueryOutput queryOutput = new QueryAction().execute(queryInput);
+
+         return (Optional.of(queryOutput.getRecords()));
+      }
+
+      return (Optional.empty());
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/
@@ -136,7 +274,7 @@ public class UpdateAction
          ////////////////////////////////////////////////////////////////////////////////////////////////////////////////
          if(record.getValue(table.getPrimaryKeyField()) == null)
          {
-            record.addError("Missing value in primary key field");
+            record.addError(new BadInputStatusMessage("Missing value in primary key field"));
          }
       }
    }
@@ -153,6 +291,8 @@ public class UpdateAction
       QTableMetaData table           = updateInput.getTable();
       QFieldMetaData primaryKeyField = table.getField(table.getPrimaryKeyField());
 
+      List<RecordSecurityLock> onlyWriteLocks = RecordSecurityLockFilters.filterForOnlyWriteLocks(CollectionUtils.nonNullList(table.getRecordSecurityLocks()));
+
       for(List<QRecord> page : CollectionUtils.getPages(updateInput.getRecords(), 1000))
       {
          List<Serializable> primaryKeysToLookup = new ArrayList<>();
@@ -186,6 +326,8 @@ public class UpdateAction
             }
          }
 
+         ValidateRecordSecurityLockHelper.validateSecurityFields(table, updateInput.getRecords(), ValidateRecordSecurityLockHelper.Action.UPDATE);
+
          for(QRecord record : page)
          {
             Serializable value = ValueUtils.getValueAsFieldType(primaryKeyField.getType(), record.getValue(table.getPrimaryKeyField()));
@@ -196,7 +338,20 @@ public class UpdateAction
 
             if(!lookedUpRecords.containsKey(value))
             {
-               record.addError(NOT_FOUND_ERROR_PREFIX + " for " + primaryKeyField.getLabel() + " = " + value);
+               record.addError(new NotFoundStatusMessage("No record was found to update for " + primaryKeyField.getLabel() + " = " + value));
+            }
+            else
+            {
+               ///////////////////////////////////////////////////////////////////////////////////////////
+               // if the table has any write-only locks, validate their values here, on the old-records //
+               ///////////////////////////////////////////////////////////////////////////////////////////
+               for(RecordSecurityLock lock : onlyWriteLocks)
+               {
+                  QRecord      oldRecord = lookedUpRecords.get(value);
+                  QFieldType   fieldType = table.getField(lock.getFieldName()).getType();
+                  Serializable lockValue = ValueUtils.getValueAsFieldType(fieldType, oldRecord.getValue(lock.getFieldName()));
+                  ValidateRecordSecurityLockHelper.validateRecordSecurityValue(table, record, lock, lockValue, fieldType, ValidateRecordSecurityLockHelper.Action.UPDATE);
+               }
             }
          }
       }
@@ -225,9 +380,9 @@ public class UpdateAction
                /////////////////////////////////////////////////////////////////////////////////////////////
                if(record.getValues().containsKey(requiredField.getName()))
                {
-                  if(record.getValue(requiredField.getName()) == null || (requiredField.getType().isStringLike() && record.getValueString(requiredField.getName()).trim().equals("")))
+                  if(record.getValue(requiredField.getName()) == null || record.getValueString(requiredField.getName()).trim().equals(""))
                   {
-                     record.addError("Missing value in required field: " + requiredField.getLabel());
+                     record.addError(new BadInputStatusMessage("Missing value in required field: " + requiredField.getLabel()));
                   }
                }
             }
@@ -262,6 +417,11 @@ public class UpdateAction
             //////////////////////////////////////////////////////
             for(QRecord record : page)
             {
+               if(CollectionUtils.nullSafeHasContents(record.getErrors()))
+               {
+                  continue;
+               }
+
                if(record.getAssociatedRecords() != null && record.getAssociatedRecords().containsKey(association.getName()))
                {
                   ///////////////////////////////////////////////////////////////////////////////////////////////////////////////
@@ -289,7 +449,8 @@ public class UpdateAction
                         //////////////////////////////////////////////////////////////////////////////////////////////////////////
                         for(JoinOn joinOn : join.getJoinOns())
                         {
-                           associatedRecord.setValue(joinOn.getRightField(), record.getValue(joinOn.getLeftField()));
+                           QFieldType type = table.getField(joinOn.getLeftField()).getType();
+                           associatedRecord.setValue(joinOn.getRightField(), ValueUtils.getValueAsFieldType(type, record.getValue(joinOn.getLeftField())));
                         }
                         nextLevelInserts.add(associatedRecord);
                      }
@@ -300,6 +461,16 @@ public class UpdateAction
                         ///////////////////////////////////////////////////////////////////////////////
                         idsBeingUpdated.add(associatedId);
                         nextLevelUpdates.add(associatedRecord);
+
+                        /////////////////////////////////////////////////////////////////////////////////////////////////
+                        // make sure the child record being updated has its join fields populated (same as an insert). //
+                        // this will make the next update action much happier                                          //
+                        /////////////////////////////////////////////////////////////////////////////////////////////////
+                        for(JoinOn joinOn : join.getJoinOns())
+                        {
+                           QFieldType type = table.getField(joinOn.getLeftField()).getType();
+                           associatedRecord.setValue(joinOn.getRightField(), ValueUtils.getValueAsFieldType(type, record.getValue(joinOn.getLeftField())));
+                        }
                      }
                   }
 
@@ -356,45 +527,6 @@ public class UpdateAction
 
 
 
-   /*******************************************************************************
-    **
-    *******************************************************************************/
-   private static List<QRecord> getOldRecordListForAuditIfNeeded(UpdateInput updateInput)
-   {
-      if(updateInput.getOmitDmlAudit())
-      {
-         return (null);
-      }
-
-      try
-      {
-         AuditLevel    auditLevel    = DMLAuditAction.getAuditLevel(updateInput);
-         List<QRecord> oldRecordList = null;
-         if(AuditLevel.FIELD.equals(auditLevel))
-         {
-            String             primaryKeyField   = updateInput.getTable().getPrimaryKeyField();
-            List<Serializable> pkeysBeingUpdated = CollectionUtils.nonNullList(updateInput.getRecords()).stream().map(r -> r.getValue(primaryKeyField)).toList();
-
-            QueryInput queryInput = new QueryInput();
-            queryInput.setTransaction(updateInput.getTransaction());
-            queryInput.setTableName(updateInput.getTableName());
-            queryInput.setFilter(new QQueryFilter(new QFilterCriteria(primaryKeyField, QCriteriaOperator.IN, pkeysBeingUpdated)));
-            // todo - need a limit?  what if too many??
-            QueryOutput queryOutput = new QueryAction().execute(queryInput);
-
-            oldRecordList = queryOutput.getRecords();
-         }
-         return oldRecordList;
-      }
-      catch(Exception e)
-      {
-         LOG.warn("Error getting old record list for audit", e, logPair("table", updateInput.getTableName()));
-         return (null);
-      }
-   }
-
-
-
    /*******************************************************************************
     ** If the table being updated uses an automation-status field, populate it now.
     *******************************************************************************/

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/helpers/ActionTimeoutHelper.java

@@ -0,0 +1,109 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2023.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.actions.tables.helpers;
+
+
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledFuture;
+import java.util.concurrent.TimeUnit;
+
+
+/*******************************************************************************
+ ** For actions that may want to set a timeout, and cancel themselves if they run
+ ** too long - this class helps.
+ **
+ ** Construct with the timeout (delay & timeUnit), and a runnable that takes care
+ ** of doing the cancel (e.g., cancelling a JDBC statement).
+ **
+ ** Call start() to make a future get scheduled (note, if delay was null or <= 0,
+ ** then it doesn't get scheduled at all).
+ **
+ ** Call cancel() if the action got far enough/completed, to cancel the future.
+ **
+ ** You can check didTimeout (getDidTimeout()) to know if the timeout did occur.
+ *******************************************************************************/
+public class ActionTimeoutHelper
+{
+   private final Integer            delay;
+   private final TimeUnit           timeUnit;
+   private final Runnable           runnable;
+   private       ScheduledFuture<?> future;
+
+   private boolean didTimeout = false;
+
+
+
+   /*******************************************************************************
+    ** Constructor
+    **
+    *******************************************************************************/
+   public ActionTimeoutHelper(Integer delay, TimeUnit timeUnit, Runnable runnable)
+   {
+      this.delay = delay;
+      this.timeUnit = timeUnit;
+      this.runnable = runnable;
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public void start()
+   {
+      if(delay == null || delay <= 0)
+      {
+         return;
+      }
+
+      future = Executors.newSingleThreadScheduledExecutor().schedule(() ->
+      {
+         didTimeout = true;
+         runnable.run();
+      }, delay, timeUnit);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public void cancel()
+   {
+      if(future != null)
+      {
+         future.cancel(true);
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for didTimeout
+    **
+    *******************************************************************************/
+   public boolean getDidTimeout()
+   {
+      return didTimeout;
+   }
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/helpers/CacheUtils.java

@@ -0,0 +1,104 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2023.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.actions.tables.helpers;
+
+
+import java.time.Instant;
+import com.kingsrook.qqq.backend.core.logging.LogPair;
+import com.kingsrook.qqq.backend.core.logging.QLogger;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.cache.CacheUseCase;
+import com.kingsrook.qqq.backend.core.modules.backend.implementations.utils.BackendQueryFilterUtils;
+import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
+import com.kingsrook.qqq.backend.core.utils.StringUtils;
+
+
+/*******************************************************************************
+ **
+ *******************************************************************************/
+public class CacheUtils
+{
+   private static final QLogger LOG = QLogger.getLogger(CacheUtils.class);
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   static QRecord mapSourceRecordToCacheRecord(QTableMetaData table, QRecord recordFromSource, CacheUseCase cacheUseCase)
+   {
+      QRecord cacheRecord = new QRecord(recordFromSource);
+
+      //////////////////////////////////////////////////////////////////////////////////////////////
+      // make sure every value in the qRecord is set, because we will possibly be doing an update //
+      // on this record and want to null out any fields not set, not leave them populated         //
+      //////////////////////////////////////////////////////////////////////////////////////////////
+      for(String fieldName : table.getFields().keySet())
+      {
+         if(fieldName.equals(table.getPrimaryKeyField()))
+         {
+            if(!cacheUseCase.getDoCopySourcePrimaryKeyToCache())
+            {
+               cacheRecord.removeValue(fieldName);
+            }
+         }
+         else if(!cacheRecord.getValues().containsKey(fieldName))
+         {
+            cacheRecord.setValue(fieldName, null);
+         }
+      }
+
+      if(StringUtils.hasContent(table.getCacheOf().getCachedDateFieldName()))
+      {
+         cacheRecord.setValue(table.getCacheOf().getCachedDateFieldName(), Instant.now());
+      }
+      return (cacheRecord);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   static boolean shouldCacheRecord(QTableMetaData table, QRecord recordToCache)
+   {
+      boolean shouldCacheRecord = true;
+      recordMatchExclusionLoop:
+      for(CacheUseCase useCase : CollectionUtils.nonNullList(table.getCacheOf().getUseCases()))
+      {
+         for(QQueryFilter filter : CollectionUtils.nonNullList(useCase.getExcludeRecordsMatching()))
+         {
+            if(BackendQueryFilterUtils.doesRecordMatch(filter, recordToCache))
+            {
+               LOG.info("Not caching record because it matches a use case's filter exclusion", new LogPair("record", recordToCache), new LogPair("filter", filter));
+               shouldCacheRecord = false;
+               break recordMatchExclusionLoop;
+            }
+         }
+      }
+
+      return (shouldCacheRecord);
+   }
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/helpers/GetActionCacheHelper.java

@@ -0,0 +1,241 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2023.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.actions.tables.helpers;
+
+
+import com.kingsrook.qqq.backend.core.actions.tables.GetAction;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.logging.QLogger;
+import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetOutput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput;
+import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
+
+
+/*******************************************************************************
+ **
+ *******************************************************************************/
+public class GetActionCacheHelper
+{
+   private static final QLogger LOG = QLogger.getLogger(GetActionCacheHelper.class);
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public void handleCaching(GetInput getInput, GetOutput getOutput) throws QException
+   {
+      ///////////////////////////////////////////////////////
+      // copy Get input & output into Query input & output //
+      ///////////////////////////////////////////////////////
+      QueryInput queryInput = GetAction.convertGetInputToQueryInput(getInput);
+
+      QueryOutput queryOutput = new QueryOutput(queryInput);
+      if(getOutput.getRecord() != null)
+      {
+         queryOutput.addRecord(getOutput.getRecord());
+      }
+
+      ////////////////////////////////////
+      // run the QueryActionCacheHelper //
+      ////////////////////////////////////
+      new QueryActionCacheHelper().handleCaching(queryInput, queryOutput);
+
+      ///////////////////////////////////
+      // set result back in get output //
+      ///////////////////////////////////
+      if(CollectionUtils.nullSafeHasContents(queryOutput.getRecords()))
+      {
+         getOutput.setRecord(queryOutput.getRecords().get(0));
+      }
+      else
+      {
+         getOutput.setRecord(null);
+      }
+   }
+
+   /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+   // In July 2023, initial caching was added in QueryAction.                                                                             //
+   // at this time, it felt wrong to essentially duplicate this code between Get & Query - as Get is a simplified use-case of Query.      //
+   // so - we'll keep this code here, as a potential quick/easy fallback - but - see above - where we use QueryActionCacheHelper instead. //
+   /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
+   /*
+   public void handleCaching(GetInput getInput, GetOutput getOutput) throws QException
+   {
+      if(getOutput.getRecord() == null)
+      {
+         ///////////////////////////////////////////////////////////////////////
+         // if the record wasn't found, see if we should look in cache-source //
+         ///////////////////////////////////////////////////////////////////////
+         QRecord recordFromSource = tryToGetFromCacheSource(getInput);
+         if(recordFromSource != null)
+         {
+            /////////////////////////////////////////////////////////////////////////////////////////////////
+            // good, we found a record from the source, make sure we should cache it, and if so, do it now //
+            // note, we always return the record from the source, even if we don't cache it.               //
+            /////////////////////////////////////////////////////////////////////////////////////////////////
+            QTableMetaData table         = getInput.getTable();
+            QRecord        recordToCache = CacheUtils.mapSourceRecordToCacheRecord(table, recordFromSource);
+            getOutput.setRecord(recordToCache);
+
+            boolean shouldCacheRecord = CacheUtils.shouldCacheRecord(table, recordToCache);
+            if(shouldCacheRecord)
+            {
+               InsertInput insertInput = new InsertInput();
+               insertInput.setTableName(getInput.getTableName());
+               insertInput.setRecords(List.of(recordToCache));
+               InsertOutput insertOutput = new InsertAction().execute(insertInput);
+
+               /////////////////////////////////////////////////////////////////////////////////////////////
+               // update the result record from the insert (e.g., so we get its id, just in case we care) //
+               /////////////////////////////////////////////////////////////////////////////////////////////
+               getOutput.setRecord(insertOutput.getRecords().get(0));
+            }
+         }
+      }
+      else
+      {
+         /////////////////////////////////////////////////////////////////////////////////
+         // if the record was found, but it's too old, maybe re-fetch from cache source //
+         /////////////////////////////////////////////////////////////////////////////////
+         refreshCacheIfExpired(getInput, getOutput);
+      }
+   }
+
+
+
+   private QRecord tryToGetFromCacheSource(GetInput getInput) throws QException
+   {
+      QRecord        recordFromSource = null;
+      QTableMetaData table            = getInput.getTable();
+
+      for(CacheUseCase cacheUseCase : CollectionUtils.nonNullList(table.getCacheOf().getUseCases()))
+      {
+         if(CacheUseCase.Type.UNIQUE_KEY_TO_UNIQUE_KEY.equals(cacheUseCase.getType()) && getInput.getUniqueKey() != null)
+         {
+            recordFromSource = getFromCachedSourceForUniqueKeyToUniqueKey(getInput, table.getCacheOf().getSourceTable());
+            break;
+         }
+         else
+         {
+            // todo!!
+            throw new NotImplementedException("Not-yet-implemented cache use case type: " + cacheUseCase.getType());
+         }
+      }
+
+      return (recordFromSource);
+   }
+
+
+
+   private void refreshCacheIfExpired(GetInput getInput, GetOutput getOutput) throws QException
+   {
+      QTableMetaData table             = getInput.getTable();
+      Integer        expirationSeconds = table.getCacheOf().getExpirationSeconds();
+      if(expirationSeconds != null)
+      {
+         QRecord cachedRecord = getOutput.getRecord();
+         Instant cachedDate   = cachedRecord.getValueInstant(table.getCacheOf().getCachedDateFieldName());
+         if(cachedDate == null || cachedDate.isBefore(Instant.now().minus(expirationSeconds, ChronoUnit.SECONDS)))
+         {
+            //////////////////////////////////////////////////////////////////////////
+            // keep the serial key from the old record in case we need to delete it //
+            //////////////////////////////////////////////////////////////////////////
+            Serializable oldRecordPrimaryKey = cachedRecord.getValue(table.getPrimaryKeyField());
+            boolean      shouldDeleteCachedRecord;
+
+            ///////////////////////////////////////////
+            // fetch record from original source now //
+            ///////////////////////////////////////////
+            QRecord recordFromSource = tryToGetFromCacheSource(getInput);
+            if(recordFromSource != null)
+            {
+               ///////////////////////////////////////////////////////////////////
+               // if the record was found in the source, put it into the output //
+               // object so returned back to caller                             //
+               ///////////////////////////////////////////////////////////////////
+               QRecord recordToCache = CacheUtils.mapSourceRecordToCacheRecord(table, recordFromSource);
+               recordToCache.setValue(table.getPrimaryKeyField(), cachedRecord.getValue(table.getPrimaryKeyField()));
+               getOutput.setRecord(recordToCache);
+
+               //////////////////////////////////////////////////////////////////////////////////////////////////////////////
+               // if the record should be cached, update the cache record - else set the flag to delete the cached record. //
+               //////////////////////////////////////////////////////////////////////////////////////////////////////////////
+               if(CacheUtils.shouldCacheRecord(table, recordToCache))
+               {
+                  UpdateInput updateInput = new UpdateInput();
+                  updateInput.setTableName(getInput.getTableName());
+                  updateInput.setRecords(List.of(recordToCache));
+                  UpdateOutput updateOutput = new UpdateAction().execute(updateInput);
+                  getOutput.setRecord(updateOutput.getRecords().get(0));
+                  shouldDeleteCachedRecord = false;
+               }
+               else
+               {
+                  shouldDeleteCachedRecord = true;
+               }
+            }
+            else
+            {
+               ///////////////////////////////////////////////////////////////////////////////////////
+               // if we did not get a record back from the source, empty out the getOutput's record //
+               // and set the flag to delete the cached record                                      //
+               ///////////////////////////////////////////////////////////////////////////////////////
+               getOutput.setRecord(null);
+               shouldDeleteCachedRecord = true;
+            }
+
+            if(shouldDeleteCachedRecord)
+            {
+               //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+               // if the record is no longer in the source (or it was in the source, but failed the should-cache check), then remove it from the cache //
+               //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+               DeleteInput deleteInput = new DeleteInput();
+               deleteInput.setTableName(getInput.getTableName());
+               deleteInput.setPrimaryKeys(List.of(oldRecordPrimaryKey));
+               new DeleteAction().execute(deleteInput);
+            }
+         }
+      }
+   }
+
+
+
+   private QRecord getFromCachedSourceForUniqueKeyToUniqueKey(GetInput getInput, String sourceTableName) throws QException
+   {
+      /////////////////////////////////////////////////////
+      // do a Get on the source table, by the unique key //
+      /////////////////////////////////////////////////////
+      GetInput sourceGetInput = new GetInput();
+      sourceGetInput.setTableName(sourceTableName);
+      sourceGetInput.setUniqueKey(getInput.getUniqueKey());
+      GetOutput sourceGetOutput = new GetAction().execute(sourceGetInput);
+      QRecord   outputRecord    = sourceGetOutput.getRecord();
+
+      return (outputRecord);
+   }
+   */
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/helpers/QueryActionCacheHelper.java

@@ -0,0 +1,622 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2023.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.actions.tables.helpers;
+
+
+import java.io.Serializable;
+import java.time.Instant;
+import java.time.temporal.ChronoUnit;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.ListIterator;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+import com.kingsrook.qqq.backend.core.actions.tables.DeleteAction;
+import com.kingsrook.qqq.backend.core.actions.tables.GetAction;
+import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
+import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
+import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction;
+import com.kingsrook.qqq.backend.core.context.QContext;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.logging.QLogger;
+import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetOutput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertOutput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateOutput;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.Capability;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.UniqueKey;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.cache.CacheUseCase;
+import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
+import com.kingsrook.qqq.backend.core.utils.ListingHash;
+import org.apache.commons.lang.NotImplementedException;
+import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
+
+
+/*******************************************************************************
+ ** After running a query, if it's for a table that's a CacheOf another table,
+ ** see if there are any cache use-cases to apply to the query result.
+ **
+ ** Such as:
+ ** - if it's a query for one or more values in a UniqueKey:
+ **   - if any particular UniqueKeys weren't found, look in the source table
+ ** - if any cached records are expired, refresh them from the source
+ **   - possibly updating the cached record; possibly deleting it.
+ *******************************************************************************/
+public class QueryActionCacheHelper
+{
+   private static final QLogger LOG = QLogger.getLogger(QueryActionCacheHelper.class);
+
+   private boolean                              isQueryInputCacheable = false;
+   private Map<CacheUseCase.Type, CacheUseCase> cacheUseCaseMap       = new HashMap<>();
+   private CacheUseCase                         activeCacheUseCase    = null;
+
+   private UniqueKey                         cacheUniqueKey  = null;
+   private ListingHash<String, Serializable> uniqueKeyValues = new ListingHash<>();
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public void handleCaching(QueryInput queryInput, QueryOutput queryOutput) throws QException
+   {
+      analyzeInput(queryInput);
+      if(!isQueryInputCacheable)
+      {
+         return;
+      }
+
+      //////////////////////////////////////////////////////////////////////////
+      // figure out which keys in the query were found, and which were missed //
+      //////////////////////////////////////////////////////////////////////////
+      List<QRecord>           recordsFoundInCache           = new ArrayList<>(queryOutput.getRecords());
+      Set<List<Serializable>> uniqueKeyValuesInFoundRecords = getUniqueKeyValuesFromFoundRecords(queryOutput.getRecords());
+      Set<List<Serializable>> missedUniqueKeyValues         = getUniqueKeyValuesFromQuery();
+      missedUniqueKeyValues.removeAll(uniqueKeyValuesInFoundRecords);
+
+      ///////////////////////////////////////////////////////////////////////////////////
+      // if any requested records weren't found, see if we should look in cache-source //
+      ///////////////////////////////////////////////////////////////////////////////////
+      if(CollectionUtils.nullSafeHasContents(missedUniqueKeyValues))
+      {
+         List<QRecord> recordsFromSource = tryToGetFromCacheSource(queryInput, missedUniqueKeyValues);
+         if(CollectionUtils.nullSafeHasContents(recordsFromSource))
+         {
+            //////////////////////////////////////////////////////////////////////////////////////////////////
+            // good, we found records from the source, make sure we should cache them, and if so, do it now //
+            // note, we always return the record from the source, even if we don't cache it.                //
+            //////////////////////////////////////////////////////////////////////////////////////////////////
+            QTableMetaData table = queryInput.getTable();
+
+            List<QRecord> recordsToReturn = recordsFromSource.stream()
+               .map(r -> CacheUtils.mapSourceRecordToCacheRecord(table, r, activeCacheUseCase))
+               .toList();
+            queryOutput.addRecords(recordsToReturn);
+
+            List<QRecord> recordsToCache = recordsToReturn.stream()
+               .filter(r -> CacheUtils.shouldCacheRecord(table, r))
+               .toList();
+
+            if(CollectionUtils.nullSafeHasContents(recordsToCache))
+            {
+               try
+               {
+                  InsertInput insertInput = new InsertInput();
+                  insertInput.setTableName(queryInput.getTableName());
+                  insertInput.setRecords(recordsToCache);
+                  insertInput.setSkipUniqueKeyCheck(true);
+                  InsertOutput insertOutput = new InsertAction().execute(insertInput);
+
+                  //////////////////////////////////////////////////////////
+                  // set the (generated) ids in the records being returne //
+                  //////////////////////////////////////////////////////////
+                  Map<List<Serializable>, QRecord> insertedRecordsByUniqueKey = new HashMap<>();
+                  for(QRecord record : insertOutput.getRecords())
+                  {
+                     insertedRecordsByUniqueKey.put(getUniqueKeyValues(record), record);
+                  }
+                  for(QRecord record : recordsToReturn)
+                  {
+                     QRecord insertedRecord = insertedRecordsByUniqueKey.get(getUniqueKeyValues(record));
+                     if(insertedRecord != null)
+                     {
+                        record.setValue(table.getPrimaryKeyField(), insertedRecord.getValue(table.getPrimaryKeyField()));
+                     }
+                  }
+               }
+               catch(Exception e)
+               {
+                  //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+                  // don't let an exception break this query - it (probably) just indicates some data that didn't get cached - so - that's generally "ok" //
+                  //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+                  LOG.warn("Error inserting cached records", e, logPair("cacheTable", queryInput.getTableName()));
+               }
+            }
+         }
+      }
+
+      //////////////////////////////////////////////////////////////////////////
+      // for records that were found, if they're too old, maybe re-fetch them //
+      //////////////////////////////////////////////////////////////////////////
+      if(CollectionUtils.nullSafeHasContents(recordsFoundInCache))
+      {
+         refreshCacheIfExpired(recordsFoundInCache, queryInput, queryOutput);
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private void refreshCacheIfExpired(List<QRecord> recordsFoundInCache, QueryInput queryInput, QueryOutput queryOutput) throws QException
+   {
+      QTableMetaData table             = queryInput.getTable();
+      Integer        expirationSeconds = table.getCacheOf().getExpirationSeconds();
+
+      if(expirationSeconds != null)
+      {
+         List<QRecord> expiredRecords = new ArrayList<>();
+         for(QRecord cachedRecord : recordsFoundInCache)
+         {
+            Instant cachedDate = cachedRecord.getValueInstant(table.getCacheOf().getCachedDateFieldName());
+            if(cachedDate == null || cachedDate.isBefore(Instant.now().minus(expirationSeconds, ChronoUnit.SECONDS)))
+            {
+               expiredRecords.add(cachedRecord);
+            }
+         }
+
+         if(CollectionUtils.nullSafeHasContents(expiredRecords))
+         {
+            Map<List<Serializable>, Serializable> uniqueKeyToPrimaryKeyMap = getUniqueKeyToPrimaryKeyMap(table.getPrimaryKeyField(), expiredRecords);
+            Set<List<Serializable>>               uniqueKeyValuesToRefresh = uniqueKeyToPrimaryKeyMap.keySet();
+
+            ////////////////////////////////////////////
+            // fetch records from original source now //
+            ////////////////////////////////////////////
+            List<QRecord> recordsFromSource = tryToGetFromCacheSource(queryInput, uniqueKeyValuesToRefresh);
+
+            Set<List<Serializable>> uniqueKeyValuesInFoundRecords = getUniqueKeyValuesFromFoundRecords(recordsFromSource);
+            Set<List<Serializable>> missedUniqueKeyValues         = getUniqueKeyValuesFromQuery();
+            missedUniqueKeyValues.retainAll(getUniqueKeyValuesFromFoundRecords(expiredRecords));
+            missedUniqueKeyValues.removeAll(uniqueKeyValuesInFoundRecords);
+
+            //////////////////////////////////////////////////////////////////////////////////////////////////////
+            // build records to cache - setting their original (from cache) ids back in them, so they'll update //
+            //////////////////////////////////////////////////////////////////////////////////////////////////////
+            List<QRecord> refreshedRecordsToReturn = recordsFromSource.stream()
+               .map(r ->
+               {
+                  QRecord recordToCache = CacheUtils.mapSourceRecordToCacheRecord(table, r, activeCacheUseCase);
+                  recordToCache.setValue(table.getPrimaryKeyField(), uniqueKeyToPrimaryKeyMap.get(getUniqueKeyValues(recordToCache)));
+                  return (recordToCache);
+               })
+               .toList();
+
+            ///////////////////////////////////////////////////////////////////////////////////////////////////////
+            // if the records were found in the source, put it into the output object so returned back to caller //
+            ///////////////////////////////////////////////////////////////////////////////////////////////////////
+            Map<List<Serializable>, QRecord> refreshedRecordsByUniqueKeyValues = refreshedRecordsToReturn.stream().collect(Collectors.toMap(this::getUniqueKeyValues, r -> r, (a, b) -> a));
+
+            ListIterator<QRecord> queryOutputListIterator = queryOutput.getRecords().listIterator();
+            while(queryOutputListIterator.hasNext())
+            {
+               QRecord            originalRecord        = queryOutputListIterator.next();
+               List<Serializable> recordUniqueKeyValues = getUniqueKeyValues(originalRecord);
+               QRecord            refreshedRecord       = refreshedRecordsByUniqueKeyValues.get(recordUniqueKeyValues);
+
+               if(refreshedRecord != null)
+               {
+                  queryOutputListIterator.set(refreshedRecord);
+               }
+               else if(missedUniqueKeyValues.contains(recordUniqueKeyValues))
+               {
+                  queryOutputListIterator.remove();
+               }
+            }
+
+            ////////////////////////////////////////////////////////////////////////////
+            // for refreshed records which should be cached, update them in the cache //
+            ////////////////////////////////////////////////////////////////////////////
+            List<QRecord> recordsToUpdate = refreshedRecordsToReturn.stream().filter(r -> CacheUtils.shouldCacheRecord(table, r)).toList();
+            if(CollectionUtils.nullSafeHasContents(recordsToUpdate))
+            {
+               try
+               {
+                  UpdateInput updateInput = new UpdateInput();
+                  updateInput.setTableName(queryInput.getTableName());
+                  updateInput.setRecords(recordsToUpdate);
+                  UpdateOutput updateOutput = new UpdateAction().execute(updateInput);
+               }
+               catch(Exception e)
+               {
+                  //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+                  // don't let an exception break this query - it (probably) just indicates some data that didn't get cached - so - that's generally "ok" //
+                  //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+                  LOG.warn("Error updating cached records", e, logPair("cacheTable", queryInput.getTableName()));
+               }
+            }
+
+            ////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+            // if the records were missed in the source - OR if they shouldn't be cached now, then mark them for deleting //
+            ////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+            Set<Serializable> cachedRecordIdsToDelete = missedUniqueKeyValues.stream()
+               .map(uniqueKeyToPrimaryKeyMap::get)
+               .collect(Collectors.toSet());
+
+            cachedRecordIdsToDelete.addAll(refreshedRecordsToReturn.stream()
+               .filter(r -> !CacheUtils.shouldCacheRecord(table, r))
+               .map(r -> r.getValue(table.getPrimaryKeyField()))
+               .collect(Collectors.toSet()));
+
+            if(CollectionUtils.nullSafeHasContents(cachedRecordIdsToDelete))
+            {
+               /////////////////////////////////////////////////////////////////////////////////
+               // if the records are no longer in the source, then remove them from the cache //
+               /////////////////////////////////////////////////////////////////////////////////
+               try
+               {
+                  DeleteInput deleteInput = new DeleteInput();
+                  deleteInput.setTableName(queryInput.getTableName());
+                  deleteInput.setPrimaryKeys(new ArrayList<>(cachedRecordIdsToDelete));
+                  new DeleteAction().execute(deleteInput);
+
+               }
+               catch(Exception e)
+               {
+                  ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+                  // don't let an exception break this query - it (probably) just indicates some data that didn't get uncached - so - that's generally "ok" //
+                  ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+                  LOG.warn("Error deleting cached records", e, logPair("cacheTable", queryInput.getTableName()));
+               }
+            }
+         }
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private Set<List<Serializable>> getUniqueKeyValuesFromQuery()
+   {
+      Set<List<Serializable>> rs = new HashSet<>();
+
+      int noOfUniqueKeys = uniqueKeyValues.get(cacheUniqueKey.getFieldNames().get(0)).size();
+      for(int i = 0; i < noOfUniqueKeys; i++)
+      {
+         List<Serializable> values = new ArrayList<>();
+
+         for(String fieldName : cacheUniqueKey.getFieldNames())
+         {
+            values.add(uniqueKeyValues.get(fieldName).get(i));
+         }
+
+         ////////////////////////////////////////////////////////////////////////////////
+         // critical - leave this here so hashCode from the list is correctly computed //
+         ////////////////////////////////////////////////////////////////////////////////
+         rs.add(values);
+      }
+
+      return (rs);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private Set<List<Serializable>> getUniqueKeyValuesFromFoundRecords(List<QRecord> records)
+   {
+      return (getUniqueKeyToPrimaryKeyMap("ignore", records).keySet());
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private Map<List<Serializable>, Serializable> getUniqueKeyToPrimaryKeyMap(String primaryKeyField, List<QRecord> records)
+   {
+      Map<List<Serializable>, Serializable> rs = new HashMap<>();
+
+      for(QRecord record : records)
+      {
+         List<Serializable> uniqueKeyValues = getUniqueKeyValues(record);
+         rs.put(uniqueKeyValues, record.getValue(primaryKeyField));
+      }
+
+      return (rs);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private List<Serializable> getUniqueKeyValues(QRecord record)
+   {
+      List<Serializable> uniqueKeyValues = new ArrayList<>();
+      for(String fieldName : cacheUniqueKey.getFieldNames())
+      {
+         uniqueKeyValues.add(record.getValue(fieldName));
+      }
+      return uniqueKeyValues;
+   }
+
+
+
+   /*******************************************************************************
+    ** figure out if this was a request that we can cache records for -
+    ** e.g., if it's a request for unique-key EQUALS or IN
+    ** build up fields for the unique keys, the values, etc.
+    *******************************************************************************/
+   private void analyzeInput(QueryInput queryInput)
+   {
+      QTableMetaData table = queryInput.getTable();
+
+      for(CacheUseCase cacheUseCase : CollectionUtils.nonNullList(table.getCacheOf().getUseCases()))
+      {
+         cacheUseCaseMap.put(cacheUseCase.getType(), cacheUseCase);
+      }
+
+      if(cacheUseCaseMap.containsKey(CacheUseCase.Type.UNIQUE_KEY_TO_UNIQUE_KEY))
+      {
+         if(queryInput.getFilter() == null)
+         {
+            LOG.trace("Unable to cache: there is no filter");
+            return;
+         }
+
+         QQueryFilter filter      = queryInput.getFilter();
+         Set<String>  queryFields = new HashSet<>();
+         if(CollectionUtils.nullSafeHasContents(filter.getSubFilters()))
+         {
+            if(CollectionUtils.nullSafeHasContents(filter.getCriteria()))
+            {
+               LOG.trace("Unable to cache: we have sub-filters and criteria");
+               return;
+            }
+
+            if(!QQueryFilter.BooleanOperator.OR.equals(filter.getBooleanOperator()))
+            {
+               LOG.trace("Unable to cache: we have sub-filters but not an OR query");
+               return;
+            }
+
+            /////////////////////////
+            // look at sub-filters //
+            /////////////////////////
+            for(QQueryFilter subFilter : filter.getSubFilters())
+            {
+               Set<String> thisSubFilterFields = getQueryFieldsIfCacheableFilter(subFilter, false);
+               if(thisSubFilterFields == null)
+               {
+                  return;
+               }
+
+               if(queryFields.isEmpty())
+               {
+                  queryFields.addAll(thisSubFilterFields);
+               }
+               else
+               {
+                  if(!queryFields.equals(thisSubFilterFields))
+                  {
+                     LOG.trace("Unable to cache: sub-filters have different sets of fields");
+                     return;
+                  }
+               }
+            }
+
+            if(doQueryFieldsMatchAUniqueKey(table, queryFields))
+            {
+               return;
+            }
+
+            LOG.trace("Unable to cache: we have sub-filters that do match a unique key");
+            return;
+         }
+         else
+         {
+            //////////////////////////////////////////
+            // look at the criteria in the query:   //
+            // - build a set of field names         //
+            // - fail upon unsupported operators    //
+            // - collect the values in the criteria //
+            //////////////////////////////////////////
+            queryFields = getQueryFieldsIfCacheableFilter(filter, true);
+            if(queryFields == null)
+            {
+               return;
+            }
+         }
+
+         if(doQueryFieldsMatchAUniqueKey(table, queryFields))
+         {
+            return;
+         }
+
+         LOG.trace("Unable to cache: we have query fields that don't match a unique key: " + queryFields);
+         return;
+      }
+
+      LOG.trace("Unable to cache: No supported use case: " + cacheUseCaseMap.keySet());
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private boolean doQueryFieldsMatchAUniqueKey(QTableMetaData table, Set<String> queryFields)
+   {
+      for(UniqueKey uniqueKey : CollectionUtils.nonNullList(table.getUniqueKeys()))
+      {
+         if(queryFields.equals(new HashSet<>(uniqueKey.getFieldNames())))
+         {
+            this.cacheUniqueKey = uniqueKey;
+            isQueryInputCacheable = true;
+            activeCacheUseCase = cacheUseCaseMap.get(CacheUseCase.Type.UNIQUE_KEY_TO_UNIQUE_KEY);
+            return true;
+         }
+      }
+      return false;
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private Set<String> getQueryFieldsIfCacheableFilter(QQueryFilter filter, boolean allowOperatorIn)
+   {
+      Set<String> rs = new HashSet<>();
+      for(QFilterCriteria criterion : filter.getCriteria())
+      {
+         boolean isEquals = criterion.getOperator().equals(QCriteriaOperator.EQUALS);
+         boolean isIn     = criterion.getOperator().equals(QCriteriaOperator.IN);
+
+         if(isEquals || (isIn && allowOperatorIn))
+         {
+            rs.add(criterion.getFieldName());
+            this.uniqueKeyValues.addAll(criterion.getFieldName(), criterion.getValues());
+         }
+         else
+         {
+            LOG.trace("Unable to cache: we have an unsupported criteria operator: " + criterion.getOperator());
+            isQueryInputCacheable = false;
+            return (null);
+         }
+      }
+      return (rs);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private List<QRecord> tryToGetFromCacheSource(QueryInput queryInput, Set<List<Serializable>> uniqueKeyValues) throws QException
+   {
+      List<QRecord>  recordsFromSource = null;
+      QTableMetaData table             = queryInput.getTable();
+
+      if(CacheUseCase.Type.UNIQUE_KEY_TO_UNIQUE_KEY.equals(activeCacheUseCase.getType()))
+      {
+         recordsFromSource = getFromCachedSourceForUniqueKeyToUniqueKey(queryInput, uniqueKeyValues, table.getCacheOf().getSourceTable());
+      }
+      else
+      {
+         // todo!!
+         throw (new NotImplementedException("Not-yet-implemented cache use case type: " + activeCacheUseCase.getType()));
+      }
+
+      return (recordsFromSource);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private List<QRecord> getFromCachedSourceForUniqueKeyToUniqueKey(QueryInput cacheQueryInput, Set<List<Serializable>> uniqueKeyValues, String sourceTableName) throws QException
+   {
+      QTableMetaData   sourceTable   = QContext.getQInstance().getTable(sourceTableName);
+      QBackendMetaData sourceBackend = QContext.getQInstance().getBackendForTable(sourceTableName);
+
+      if(sourceTable.isCapabilityEnabled(sourceBackend, Capability.TABLE_QUERY))
+      {
+         ///////////////////////////////////////////////////////
+         // do a Query on the source table, by the unique key //
+         ///////////////////////////////////////////////////////
+         QueryInput sourceQueryInput = new QueryInput();
+         sourceQueryInput.setTableName(sourceTableName);
+
+         QQueryFilter filter = new QQueryFilter().withBooleanOperator(QQueryFilter.BooleanOperator.OR);
+         sourceQueryInput.setFilter(filter);
+         sourceQueryInput.setCommonParamsFrom(cacheQueryInput);
+
+         for(List<Serializable> uniqueKeyValue : uniqueKeyValues)
+         {
+            QQueryFilter subFilter = new QQueryFilter();
+            filter.addSubFilter(subFilter);
+
+            for(int i = 0; i < cacheUniqueKey.getFieldNames().size(); i++)
+            {
+               subFilter.addCriteria(new QFilterCriteria(cacheUniqueKey.getFieldNames().get(i), QCriteriaOperator.EQUALS, uniqueKeyValue.get(i)));
+            }
+         }
+
+         QueryOutput sourceQueryOutput = new QueryAction().execute(sourceQueryInput);
+         return (sourceQueryOutput.getRecords());
+      }
+      else if(sourceTable.isCapabilityEnabled(sourceBackend, Capability.TABLE_GET))
+      {
+         ///////////////////////////////////////////////////////////////////////
+         // if the table only supports GET, then do a GET for each unique key //
+         ///////////////////////////////////////////////////////////////////////
+         List<QRecord> outputRecords = new ArrayList<>();
+         for(List<Serializable> uniqueKeyValue : uniqueKeyValues)
+         {
+            Map<String, Serializable> uniqueKey = new HashMap<>();
+            for(int i = 0; i < cacheUniqueKey.getFieldNames().size(); i++)
+            {
+               uniqueKey.put(cacheUniqueKey.getFieldNames().get(i), uniqueKeyValue.get(i));
+            }
+
+            GetInput getInput = new GetInput();
+            getInput.setTableName(sourceTableName);
+            getInput.setUniqueKey(uniqueKey);
+            getInput.setCommonParamsFrom(cacheQueryInput);
+            GetOutput getOutput = new GetAction().execute(getInput);
+
+            if(getOutput.getRecord() != null)
+            {
+               outputRecords.add(getOutput.getRecord());
+            }
+         }
+
+         return (outputRecords);
+      }
+      else
+      {
+         throw (new QException("Cache source table " + sourceTableName + " does not support Query or Get capability."));
+      }
+   }
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/helpers/QueryStatManager.java

@@ -0,0 +1,640 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2023.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.actions.tables.helpers;
+
+
+import java.time.Instant;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Objects;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+import java.util.function.Supplier;
+import com.kingsrook.qqq.backend.core.actions.tables.GetAction;
+import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
+import com.kingsrook.qqq.backend.core.context.QContext;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.instances.QMetaDataVariableInterpreter;
+import com.kingsrook.qqq.backend.core.logging.QLogger;
+import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetOutput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertOutput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.Capability;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+import com.kingsrook.qqq.backend.core.model.querystats.QueryStat;
+import com.kingsrook.qqq.backend.core.model.querystats.QueryStatCriteriaField;
+import com.kingsrook.qqq.backend.core.model.querystats.QueryStatJoinTable;
+import com.kingsrook.qqq.backend.core.model.querystats.QueryStatOrderByField;
+import com.kingsrook.qqq.backend.core.model.session.QSession;
+import com.kingsrook.qqq.backend.core.model.tables.QQQTable;
+import com.kingsrook.qqq.backend.core.model.tables.QQQTablesMetaDataProvider;
+import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
+import com.kingsrook.qqq.backend.core.utils.StringUtils;
+import com.kingsrook.qqq.backend.core.utils.collections.MapBuilder;
+import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
+
+
+/*******************************************************************************
+ ** Singleton, which starts a thread, to store query stats into a table.
+ **
+ ** Supports these systemProperties or ENV_VARS:
+ ** qqq.queryStatManager.enabled / QQQ_QUERY_STAT_MANAGER_ENABLED
+ ** qqq.queryStatManager.minMillisToStore / QQQ_QUERY_STAT_MANAGER_MIN_MILLIS_TO_STORE
+ ** qqq.queryStatManager.jobPeriodSeconds / QQQ_QUERY_STAT_MANAGER_JOB_PERIOD_SECONDS
+ ** qqq.queryStatManager.jobInitialDelay / QQQ_QUERY_STAT_MANAGER_JOB_INITIAL_DELAY
+ *******************************************************************************/
+public class QueryStatManager
+{
+   private static final QLogger LOG = QLogger.getLogger(QueryStatManager.class);
+
+   private static QueryStatManager queryStatManager = null;
+
+   // todo - support multiple qInstances?
+   private QInstance          qInstance;
+   private Supplier<QSession> sessionSupplier;
+
+   private boolean         active     = false;
+   private List<QueryStat> queryStats = new ArrayList<>();
+
+   private ScheduledExecutorService executorService;
+
+   private int jobPeriodSeconds = 60;
+   private int jobInitialDelay  = 60;
+   private int minMillisToStore = 0;
+
+
+
+   /*******************************************************************************
+    ** Singleton constructor
+    *******************************************************************************/
+   private QueryStatManager()
+   {
+
+   }
+
+
+
+   /*******************************************************************************
+    ** Singleton accessor
+    *******************************************************************************/
+   public static QueryStatManager getInstance()
+   {
+      if(queryStatManager == null)
+      {
+         queryStatManager = new QueryStatManager();
+
+         QMetaDataVariableInterpreter interpreter = new QMetaDataVariableInterpreter();
+
+         Integer propertyMinMillisToStore = interpreter.getIntegerFromPropertyOrEnvironment("qqq.queryStatManager.minMillisToStore", "QQQ_QUERY_STAT_MANAGER_MIN_MILLIS_TO_STORE", null);
+         if(propertyMinMillisToStore != null)
+         {
+            queryStatManager.setMinMillisToStore(propertyMinMillisToStore);
+         }
+
+         Integer propertyJobPeriodSeconds = interpreter.getIntegerFromPropertyOrEnvironment("qqq.queryStatManager.jobPeriodSeconds", "QQQ_QUERY_STAT_MANAGER_JOB_PERIOD_SECONDS", null);
+         if(propertyJobPeriodSeconds != null)
+         {
+            queryStatManager.setJobPeriodSeconds(propertyJobPeriodSeconds);
+         }
+
+         Integer propertyJobInitialDelay = interpreter.getIntegerFromPropertyOrEnvironment("qqq.queryStatManager.jobInitialDelay", "QQQ_QUERY_STAT_MANAGER_JOB_INITIAL_DELAY", null);
+         if(propertyJobInitialDelay != null)
+         {
+            queryStatManager.setJobInitialDelay(propertyJobInitialDelay);
+         }
+
+      }
+      return (queryStatManager);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public static QueryStat newQueryStat(QBackendMetaData backend, QTableMetaData table, QQueryFilter filter)
+   {
+      QueryStat queryStat = null;
+
+      if(table.isCapabilityEnabled(backend, Capability.QUERY_STATS))
+      {
+         queryStat = new QueryStat();
+         queryStat.setTableName(table.getName());
+         queryStat.setQueryFilter(Objects.requireNonNullElse(filter, new QQueryFilter()));
+         queryStat.setStartTimestamp(Instant.now());
+      }
+
+      return (queryStat);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public void start(QInstance qInstance, Supplier<QSession> sessionSupplier)
+   {
+      if(!isEnabled())
+      {
+         LOG.info("Not starting QueryStatManager per settings.");
+         return;
+      }
+
+      LOG.info("Starting QueryStatManager");
+
+      this.qInstance = qInstance;
+      this.sessionSupplier = sessionSupplier;
+
+      active = true;
+      queryStats = new ArrayList<>();
+
+      executorService = Executors.newSingleThreadScheduledExecutor();
+      executorService.scheduleAtFixedRate(new QueryStatManagerInsertJob(), jobInitialDelay, jobPeriodSeconds, TimeUnit.SECONDS);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private static boolean isEnabled()
+   {
+      return new QMetaDataVariableInterpreter().getBooleanFromPropertyOrEnvironment("qqq.queryStatManager.enabled", "QQQ_QUERY_STAT_MANAGER_ENABLED", true);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public void stop()
+   {
+      active = false;
+      queryStats.clear();
+
+      if(executorService != null)
+      {
+         executorService.shutdown();
+         executorService = null;
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public void add(QueryStat queryStat)
+   {
+      if(queryStat == null)
+      {
+         return;
+      }
+
+      if(active)
+      {
+         ////////////////////////////////////////////////////////////////////////////////////////
+         // set fields that we need to capture now (rather than when the thread to store runs) //
+         ////////////////////////////////////////////////////////////////////////////////////////
+         if(queryStat.getFirstResultTimestamp() == null)
+         {
+            queryStat.setFirstResultTimestamp(Instant.now());
+         }
+
+         if(queryStat.getStartTimestamp() != null && queryStat.getFirstResultTimestamp() != null && queryStat.getFirstResultMillis() == null)
+         {
+            long millis = queryStat.getFirstResultTimestamp().toEpochMilli() - queryStat.getStartTimestamp().toEpochMilli();
+            queryStat.setFirstResultMillis((int) millis);
+         }
+
+         if(queryStat.getFirstResultMillis() != null && queryStat.getFirstResultMillis() < minMillisToStore)
+         {
+            //////////////////////////////////////////////////////////////
+            // discard this record if it's under the min millis setting //
+            //////////////////////////////////////////////////////////////
+            return;
+         }
+
+         if(queryStat.getSessionId() == null && QContext.getQSession() != null)
+         {
+            queryStat.setSessionId(QContext.getQSession().getUuid());
+         }
+
+         if(queryStat.getAction() == null)
+         {
+            if(!QContext.getActionStack().isEmpty())
+            {
+               queryStat.setAction(QContext.getActionStack().peek().getActionIdentity());
+            }
+            else
+            {
+               boolean   expected = false;
+               Exception e        = new Exception("Unexpected empty action stack");
+               for(StackTraceElement stackTraceElement : e.getStackTrace())
+               {
+                  String className = stackTraceElement.getClassName();
+                  if(className.contains(QueryStatManagerInsertJob.class.getName()))
+                  {
+                     expected = true;
+                     break;
+                  }
+               }
+
+               if(!expected)
+               {
+                  LOG.debug(e);
+               }
+            }
+         }
+
+         synchronized(this)
+         {
+            queryStats.add(queryStat);
+         }
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private List<QueryStat> getListAndReset()
+   {
+      if(queryStats.isEmpty())
+      {
+         return Collections.emptyList();
+      }
+
+      synchronized(this)
+      {
+         List<QueryStat> returnList = queryStats;
+         queryStats = new ArrayList<>();
+         return (returnList);
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** force stats to be stored right now (rather than letting the scheduled job do it)
+    *******************************************************************************/
+   public void storeStatsNow()
+   {
+      new QueryStatManagerInsertJob().run();
+   }
+
+
+
+   /*******************************************************************************
+    ** Runnable that gets scheduled to periodically reset and store the list of collected stats
+    *******************************************************************************/
+   private static class QueryStatManagerInsertJob implements Runnable
+   {
+      private static final QLogger LOG = QLogger.getLogger(QueryStatManagerInsertJob.class);
+
+
+
+      /*******************************************************************************
+       **
+       *******************************************************************************/
+      @Override
+      public void run()
+      {
+         try
+         {
+            QContext.init(getInstance().qInstance, getInstance().sessionSupplier.get());
+
+            /////////////////////////////////////////////////////////////////////////////////////
+            // every time we re-run, check if we've been turned off - if so, stop the service. //
+            /////////////////////////////////////////////////////////////////////////////////////
+            if(!isEnabled())
+            {
+               LOG.info("Stopping QueryStatManager.");
+               getInstance().stop();
+               return;
+            }
+
+            List<QueryStat> list = getInstance().getListAndReset();
+
+            LOG.info(logPair("queryStatListSize", list.size()));
+
+            if(list.isEmpty())
+            {
+               return;
+            }
+
+            ////////////////////////////////////
+            // prime the entities for storing //
+            ////////////////////////////////////
+            List<QRecord> queryStatQRecordsToInsert = new ArrayList<>();
+            for(QueryStat queryStat : list)
+            {
+               try
+               {
+                  //////////////////////
+                  // set the table id //
+                  //////////////////////
+                  Integer qqqTableId = getQQQTableId(queryStat.getTableName());
+                  queryStat.setQqqTableId(qqqTableId);
+
+                  //////////////////////////////
+                  // build join-table records //
+                  //////////////////////////////
+                  if(CollectionUtils.nullSafeHasContents(queryStat.getJoinTableNames()))
+                  {
+                     List<QueryStatJoinTable> queryStatJoinTableList = new ArrayList<>();
+                     for(String joinTableName : queryStat.getJoinTableNames())
+                     {
+                        queryStatJoinTableList.add(new QueryStatJoinTable().withQqqTableId(getQQQTableId(joinTableName)));
+                     }
+                     queryStat.setQueryStatJoinTableList(queryStatJoinTableList);
+                  }
+
+                  ////////////////////////////
+                  // build criteria records //
+                  ////////////////////////////
+                  if(queryStat.getQueryFilter() != null && queryStat.getQueryFilter().hasAnyCriteria())
+                  {
+                     List<QueryStatCriteriaField> queryStatCriteriaFieldList = new ArrayList<>();
+                     processCriteriaFromFilter(qqqTableId, queryStatCriteriaFieldList, queryStat.getQueryFilter());
+                     queryStat.setQueryStatCriteriaFieldList(queryStatCriteriaFieldList);
+                  }
+
+                  if(CollectionUtils.nullSafeHasContents(queryStat.getQueryFilter().getOrderBys()))
+                  {
+                     List<QueryStatOrderByField> queryStatOrderByFieldList = new ArrayList<>();
+                     processOrderByFromFilter(qqqTableId, queryStatOrderByFieldList, queryStat.getQueryFilter());
+                     queryStat.setQueryStatOrderByFieldList(queryStatOrderByFieldList);
+                  }
+
+                  queryStatQRecordsToInsert.add(queryStat.toQRecord());
+               }
+               catch(Exception e)
+               {
+                  //////////////////////
+                  // skip this record //
+                  //////////////////////
+                  LOG.warn("Error priming a query stat for storing", e);
+               }
+            }
+
+            try
+            {
+               InsertInput insertInput = new InsertInput();
+               insertInput.setTableName(QueryStat.TABLE_NAME);
+               insertInput.setRecords(queryStatQRecordsToInsert);
+               new InsertAction().execute(insertInput);
+            }
+            catch(Exception e)
+            {
+               LOG.error("Error inserting query stats", e);
+            }
+         }
+         catch(Exception e)
+         {
+            LOG.warn("Error storing query stats", e);
+         }
+         finally
+         {
+            QContext.clear();
+         }
+      }
+
+
+
+      /*******************************************************************************
+       **
+       *******************************************************************************/
+      private static void processCriteriaFromFilter(Integer qqqTableId, List<QueryStatCriteriaField> queryStatCriteriaFieldList, QQueryFilter queryFilter) throws QException
+      {
+         for(QFilterCriteria criteria : CollectionUtils.nonNullList(queryFilter.getCriteria()))
+         {
+            String                 fieldName              = criteria.getFieldName();
+            QueryStatCriteriaField queryStatCriteriaField = new QueryStatCriteriaField();
+            queryStatCriteriaField.setOperator(String.valueOf(criteria.getOperator()));
+
+            if(criteria.getValues() != null)
+            {
+               queryStatCriteriaField.setValues(StringUtils.join(",", criteria.getValues()));
+            }
+
+            if(fieldName.contains("."))
+            {
+               String[] parts = fieldName.split("\\.");
+               if(parts.length > 1)
+               {
+                  queryStatCriteriaField.setQqqTableId(getQQQTableId(parts[0]));
+                  queryStatCriteriaField.setName(parts[1]);
+               }
+            }
+            else
+            {
+               queryStatCriteriaField.setQqqTableId(qqqTableId);
+               queryStatCriteriaField.setName(fieldName);
+            }
+
+            queryStatCriteriaFieldList.add(queryStatCriteriaField);
+         }
+
+         for(QQueryFilter subFilter : CollectionUtils.nonNullList(queryFilter.getSubFilters()))
+         {
+            processCriteriaFromFilter(qqqTableId, queryStatCriteriaFieldList, subFilter);
+         }
+      }
+
+
+
+      /*******************************************************************************
+       **
+       *******************************************************************************/
+      private static void processOrderByFromFilter(Integer qqqTableId, List<QueryStatOrderByField> queryStatOrderByFieldList, QQueryFilter queryFilter) throws QException
+      {
+         for(QFilterOrderBy orderBy : CollectionUtils.nonNullList(queryFilter.getOrderBys()))
+         {
+            String                fieldName             = orderBy.getFieldName();
+            QueryStatOrderByField queryStatOrderByField = new QueryStatOrderByField();
+
+            if(fieldName != null)
+            {
+               if(fieldName.contains("."))
+               {
+                  String[] parts = fieldName.split("\\.");
+                  if(parts.length > 1)
+                  {
+                     queryStatOrderByField.setQqqTableId(getQQQTableId(parts[0]));
+                     queryStatOrderByField.setName(parts[1]);
+                  }
+               }
+               else
+               {
+                  queryStatOrderByField.setQqqTableId(qqqTableId);
+                  queryStatOrderByField.setName(fieldName);
+               }
+
+               queryStatOrderByFieldList.add(queryStatOrderByField);
+            }
+         }
+      }
+
+
+
+      /*******************************************************************************
+       **
+       *******************************************************************************/
+      private static Integer getQQQTableId(String tableName) throws QException
+      {
+         /////////////////////////////
+         // look in the cache table //
+         /////////////////////////////
+         GetInput getInput = new GetInput();
+         getInput.setTableName(QQQTablesMetaDataProvider.QQQ_TABLE_CACHE_TABLE_NAME);
+         getInput.setUniqueKey(MapBuilder.of("name", tableName));
+         GetOutput getOutput = new GetAction().execute(getInput);
+
+         ////////////////////////
+         // upon cache miss... //
+         ////////////////////////
+         if(getOutput.getRecord() == null)
+         {
+            ///////////////////////////////////////////////////////
+            // insert the record (into the table, not the cache) //
+            ///////////////////////////////////////////////////////
+            QTableMetaData tableMetaData = getInstance().qInstance.getTable(tableName);
+            InsertInput    insertInput   = new InsertInput();
+            insertInput.setTableName(QQQTable.TABLE_NAME);
+            insertInput.setRecords(List.of(new QRecord().withValue("name", tableName).withValue("label", tableMetaData.getLabel())));
+            InsertOutput insertOutput = new InsertAction().execute(insertInput);
+
+            ///////////////////////////////////
+            // repeat the get from the cache //
+            ///////////////////////////////////
+            getOutput = new GetAction().execute(getInput);
+         }
+
+         return getOutput.getRecord().getValueInteger("id");
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for jobPeriodSeconds
+    *******************************************************************************/
+   public int getJobPeriodSeconds()
+   {
+      return (this.jobPeriodSeconds);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for jobPeriodSeconds
+    *******************************************************************************/
+   public void setJobPeriodSeconds(int jobPeriodSeconds)
+   {
+      this.jobPeriodSeconds = jobPeriodSeconds;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for jobPeriodSeconds
+    *******************************************************************************/
+   public QueryStatManager withJobPeriodSeconds(int jobPeriodSeconds)
+   {
+      this.jobPeriodSeconds = jobPeriodSeconds;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for jobInitialDelay
+    *******************************************************************************/
+   public int getJobInitialDelay()
+   {
+      return (this.jobInitialDelay);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for jobInitialDelay
+    *******************************************************************************/
+   public void setJobInitialDelay(int jobInitialDelay)
+   {
+      this.jobInitialDelay = jobInitialDelay;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for jobInitialDelay
+    *******************************************************************************/
+   public QueryStatManager withJobInitialDelay(int jobInitialDelay)
+   {
+      this.jobInitialDelay = jobInitialDelay;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for minMillisToStore
+    *******************************************************************************/
+   public int getMinMillisToStore()
+   {
+      return (this.minMillisToStore);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for minMillisToStore
+    *******************************************************************************/
+   public void setMinMillisToStore(int minMillisToStore)
+   {
+      this.minMillisToStore = minMillisToStore;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for minMillisToStore
+    *******************************************************************************/
+   public QueryStatManager withMinMillisToStore(int minMillisToStore)
+   {
+      this.minMillisToStore = minMillisToStore;
+      return (this);
+   }
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/helpers/ValidateRecordSecurityLockHelper.java

@@ -44,10 +44,13 @@ import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinOn;
 import com.kingsrook.qqq.backend.core.model.metadata.joins.QJoinMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.security.QSecurityKeyType;
 import com.kingsrook.qqq.backend.core.model.metadata.security.RecordSecurityLock;
+import com.kingsrook.qqq.backend.core.model.metadata.security.RecordSecurityLockFilters;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+import com.kingsrook.qqq.backend.core.model.statusmessages.PermissionDeniedMessage;
 import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
 import com.kingsrook.qqq.backend.core.utils.ListingHash;
 import com.kingsrook.qqq.backend.core.utils.StringUtils;
+import com.kingsrook.qqq.backend.core.utils.ValueUtils;
 
 
 /*******************************************************************************
@@ -66,6 +69,7 @@ public class ValidateRecordSecurityLockHelper
    {
       INSERT,
       UPDATE,
+      DELETE,
       SELECT
    }
 
@@ -76,7 +80,7 @@ public class ValidateRecordSecurityLockHelper
     *******************************************************************************/
    public static void validateSecurityFields(QTableMetaData table, List<QRecord> records, Action action) throws QException
    {
-      List<RecordSecurityLock> locksToCheck = getRecordSecurityLocks(table);
+      List<RecordSecurityLock> locksToCheck = getRecordSecurityLocks(table, action);
       if(CollectionUtils.nullSafeIsEmpty(locksToCheck))
       {
          return;
@@ -96,11 +100,12 @@ public class ValidateRecordSecurityLockHelper
 
             for(QRecord record : records)
             {
-               if(action.equals(Action.UPDATE) && !record.getValues().containsKey(field.getName()))
+               if(action.equals(Action.UPDATE) && !record.getValues().containsKey(field.getName()) && RecordSecurityLock.LockScope.READ_AND_WRITE.equals(recordSecurityLock.getLockScope()))
                {
-                  /////////////////////////////////////////////////////////////////////////
-                  // if not updating the security field, then no error can come from it! //
-                  /////////////////////////////////////////////////////////////////////////
+                  /////////////////////////////////////////////////////////////////////////////////////////////////////////
+                  // if this is a read-write lock, then if we have the record, it means we were able to read the record. //
+                  // So if we're not updating the security field, then no error can come from it!                        //
+                  /////////////////////////////////////////////////////////////////////////////////////////////////////////
                   continue;
                }
 
@@ -113,9 +118,10 @@ public class ValidateRecordSecurityLockHelper
             ////////////////////////////////////////////////////////////////////////////////////////////////////////////////
             // else look for the joined record - if it isn't found, assume a fail - else validate security value if found //
             ////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-            QJoinMetaData  leftMostJoin      = QContext.getQInstance().getJoin(recordSecurityLock.getJoinNameChain().get(0));
-            QJoinMetaData  rightMostJoin     = QContext.getQInstance().getJoin(recordSecurityLock.getJoinNameChain().get(recordSecurityLock.getJoinNameChain().size() - 1));
-            QTableMetaData leftMostJoinTable = QContext.getQInstance().getTable(leftMostJoin.getLeftTable());
+            QJoinMetaData  leftMostJoin       = QContext.getQInstance().getJoin(recordSecurityLock.getJoinNameChain().get(0));
+            QJoinMetaData  rightMostJoin      = QContext.getQInstance().getJoin(recordSecurityLock.getJoinNameChain().get(recordSecurityLock.getJoinNameChain().size() - 1));
+            QTableMetaData rightMostJoinTable = QContext.getQInstance().getTable(rightMostJoin.getRightTable());
+            QTableMetaData leftMostJoinTable  = QContext.getQInstance().getTable(leftMostJoin.getLeftTable());
 
             for(List<QRecord> inputRecordPage : CollectionUtils.getPages(records, 500))
             {
@@ -153,7 +159,8 @@ public class ValidateRecordSecurityLockHelper
 
                   for(JoinOn joinOn : rightMostJoin.getJoinOns())
                   {
-                     Serializable inputRecordValue = inputRecord.getValue(joinOn.getRightField());
+                     QFieldType   type             = rightMostJoinTable.getField(joinOn.getRightField()).getType();
+                     Serializable inputRecordValue = ValueUtils.getValueAsFieldType(type, inputRecord.getValue(joinOn.getRightField()));
                      inputRecordJoinValues.add(inputRecordValue);
 
                      subFilter.addCriteria(inputRecordValue == null
@@ -225,7 +232,7 @@ public class ValidateRecordSecurityLockHelper
                      {
                         if(RecordSecurityLock.NullValueBehavior.DENY.equals(recordSecurityLock.getNullValueBehavior()))
                         {
-                           inputRecord.addError("You do not have permission to " + action.name().toLowerCase() + " this record - the referenced " + leftMostJoinTable.getLabel() + " was not found.");
+                           inputRecord.addError(new PermissionDeniedMessage("You do not have permission to " + action.name().toLowerCase() + " this record - the referenced " + leftMostJoinTable.getLabel() + " was not found."));
                         }
                      }
                   }
@@ -240,11 +247,18 @@ public class ValidateRecordSecurityLockHelper
    /*******************************************************************************
     **
     *******************************************************************************/
-   private static List<RecordSecurityLock> getRecordSecurityLocks(QTableMetaData table)
+   private static List<RecordSecurityLock> getRecordSecurityLocks(QTableMetaData table, Action action)
    {
-      List<RecordSecurityLock> recordSecurityLocks = table.getRecordSecurityLocks();
+      List<RecordSecurityLock> recordSecurityLocks = CollectionUtils.nonNullList(table.getRecordSecurityLocks());
       List<RecordSecurityLock> locksToCheck        = new ArrayList<>();
 
+      recordSecurityLocks = switch(action)
+      {
+         case INSERT, UPDATE, DELETE -> RecordSecurityLockFilters.filterForWriteLocks(recordSecurityLocks);
+         case SELECT -> RecordSecurityLockFilters.filterForReadLocks(recordSecurityLocks);
+         default -> throw (new IllegalArgumentException("Unsupported action: " + action));
+      };
+
       ////////////////////////////////////////
       // if there are no locks, just return //
       ////////////////////////////////////////
@@ -261,7 +275,7 @@ public class ValidateRecordSecurityLockHelper
          QSecurityKeyType securityKeyType = QContext.getQInstance().getSecurityKeyType(recordSecurityLock.getSecurityKeyType());
          if(StringUtils.hasContent(securityKeyType.getAllAccessKeyName()) && QContext.getQSession().hasSecurityKeyValue(securityKeyType.getAllAccessKeyName(), true, QFieldType.BOOLEAN))
          {
-            LOG.debug("Session has " + securityKeyType.getAllAccessKeyName() + " - not checking this lock.");
+            LOG.trace("Session has " + securityKeyType.getAllAccessKeyName() + " - not checking this lock.");
          }
          else
          {
@@ -277,7 +291,7 @@ public class ValidateRecordSecurityLockHelper
    /*******************************************************************************
     **
     *******************************************************************************/
-   static void validateRecordSecurityValue(QTableMetaData table, QRecord record, RecordSecurityLock recordSecurityLock, Serializable recordSecurityValue, QFieldType fieldType, Action action)
+   public static void validateRecordSecurityValue(QTableMetaData table, QRecord record, RecordSecurityLock recordSecurityLock, Serializable recordSecurityValue, QFieldType fieldType, Action action)
    {
       if(recordSecurityValue == null)
       {
@@ -287,7 +301,7 @@ public class ValidateRecordSecurityLockHelper
          if(RecordSecurityLock.NullValueBehavior.DENY.equals(recordSecurityLock.getNullValueBehavior()))
          {
             String lockLabel = CollectionUtils.nullSafeHasContents(recordSecurityLock.getJoinNameChain()) ? recordSecurityLock.getSecurityKeyType() : table.getField(recordSecurityLock.getFieldName()).getLabel();
-            record.addError("You do not have permission to " + action.name().toLowerCase() + " a record without a value in the field: " + lockLabel);
+            record.addError(new PermissionDeniedMessage("You do not have permission to " + action.name().toLowerCase() + " a record without a value in the field: " + lockLabel));
          }
       }
       else
@@ -299,12 +313,12 @@ public class ValidateRecordSecurityLockHelper
                ///////////////////////////////////////////////////////////////////////////////////////////////
                // avoid telling the user a value from a foreign record that they didn't pass in themselves. //
                ///////////////////////////////////////////////////////////////////////////////////////////////
-               record.addError("You do not have permission to " + action.name().toLowerCase() + " this record.");
+               record.addError(new PermissionDeniedMessage("You do not have permission to " + action.name().toLowerCase() + " this record."));
             }
             else
             {
                QFieldMetaData field = table.getField(recordSecurityLock.getFieldName());
-               record.addError("You do not have permission to " + action.name().toLowerCase() + " a record with a value of " + recordSecurityValue + " in the field: " + field.getLabel());
+               record.addError(new PermissionDeniedMessage("You do not have permission to " + action.name().toLowerCase() + " a record with a value of " + recordSecurityValue + " in the field: " + field.getLabel()));
             }
          }
       }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/templates/ConvertHtmlToPdfAction.java

@@ -26,8 +26,8 @@ import java.nio.file.Path;
 import java.util.Map;
 import com.kingsrook.qqq.backend.core.actions.AbstractQActionFunction;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
-import com.kingsrook.qqq.backend.core.model.templates.ConvertHtmlToPdfInput;
-import com.kingsrook.qqq.backend.core.model.templates.ConvertHtmlToPdfOutput;
+import com.kingsrook.qqq.backend.core.model.actions.templates.ConvertHtmlToPdfInput;
+import com.kingsrook.qqq.backend.core.model.actions.templates.ConvertHtmlToPdfOutput;
 import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
 import org.jsoup.Jsoup;
 import org.jsoup.nodes.Document;

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/templates/RenderTemplateAction.java

@@ -26,20 +26,30 @@ import java.io.StringWriter;
 import java.util.Map;
 import com.kingsrook.qqq.backend.core.actions.AbstractQActionFunction;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.logging.QLogger;
 import com.kingsrook.qqq.backend.core.model.actions.AbstractActionInput;
-import com.kingsrook.qqq.backend.core.model.templates.RenderTemplateInput;
-import com.kingsrook.qqq.backend.core.model.templates.RenderTemplateOutput;
+import com.kingsrook.qqq.backend.core.model.actions.templates.RenderTemplateInput;
+import com.kingsrook.qqq.backend.core.model.actions.templates.RenderTemplateOutput;
 import com.kingsrook.qqq.backend.core.model.templates.TemplateType;
+import com.kingsrook.qqq.backend.core.utils.StringUtils;
 import org.apache.velocity.VelocityContext;
 import org.apache.velocity.app.Velocity;
+import org.apache.velocity.app.event.EventCartridge;
+import org.apache.velocity.app.event.MethodExceptionEventHandler;
 import org.apache.velocity.context.Context;
+import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
 
 
 /*******************************************************************************
  ** Basic action to render a template!
+ **
+ ** hard-coded built to only assume Velocity right now.  could expand (and refactor) in future.
  *******************************************************************************/
 public class RenderTemplateAction extends AbstractQActionFunction<RenderTemplateInput, RenderTemplateOutput>
 {
+   private static final QLogger LOG = QLogger.getLogger(RenderTemplateAction.class);
+
+
 
    /*******************************************************************************
     **
@@ -52,9 +62,20 @@ public class RenderTemplateAction extends AbstractQActionFunction<RenderTemplate
       if(TemplateType.VELOCITY.equals(input.getTemplateType()))
       {
          Velocity.init();
-         Context      context      = new VelocityContext(input.getContext());
+         Context context = new VelocityContext();
+
+         if(input.getContext() != null)
+         {
+            for(Map.Entry<String, ?> entry : input.getContext().entrySet())
+            {
+               context.put(entry.getKey(), entry.getValue());
+            }
+         }
+
+         setupEventHandlers(context);
+
          StringWriter stringWriter = new StringWriter();
-         Velocity.evaluate(context, stringWriter, "logTag", input.getCode());
+         Velocity.evaluate(context, stringWriter, StringUtils.hasContent(input.getTemplateIdentifier()) ? input.getTemplateIdentifier() : "anonymous", input.getCode());
          output.setResult(stringWriter.getBuffer().toString());
       }
       else
@@ -67,12 +88,28 @@ public class RenderTemplateAction extends AbstractQActionFunction<RenderTemplate
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private static void setupEventHandlers(Context context)
+   {
+      EventCartridge eventCartridge = new EventCartridge();
+      eventCartridge.addEventHandler((MethodExceptionEventHandler) (ctx, aClass, method, exception, info) ->
+      {
+         LOG.info("Exception in velocity template", exception, logPair("at", info.toString()));
+         return (null);
+      });
+      eventCartridge.attachToContext(context);
+   }
+
+
+
    /*******************************************************************************
     ** Most convenient static wrapper to render a Velocity template.
     *******************************************************************************/
    public static String renderVelocity(AbstractActionInput parentActionInput, Map<String, Object> context, String code) throws QException
    {
-      return (render(parentActionInput, TemplateType.VELOCITY, context, code));
+      return (render(TemplateType.VELOCITY, context, code));
    }
 
 
@@ -80,7 +117,7 @@ public class RenderTemplateAction extends AbstractQActionFunction<RenderTemplate
    /*******************************************************************************
     ** Convenient static wrapper to render a template of an arbitrary type (language).
     *******************************************************************************/
-   public static String render(AbstractActionInput parentActionInput, TemplateType templateType, Map<String, Object> context, String code) throws QException
+   public static String render(TemplateType templateType, Map<String, Object> context, String code) throws QException
    {
       RenderTemplateInput renderTemplateInput = new RenderTemplateInput();
       renderTemplateInput.setCode(code);

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/QPossibleValueTranslator.java

@@ -32,11 +32,14 @@ import java.util.List;
 import java.util.Map;
 import java.util.Objects;
 import java.util.Set;
+import com.kingsrook.qqq.backend.core.actions.QBackendTransaction;
 import com.kingsrook.qqq.backend.core.actions.customizers.QCodeLoader;
+import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
 import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
 import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.exceptions.QValueException;
 import com.kingsrook.qqq.backend.core.logging.QLogger;
+import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
@@ -74,9 +77,47 @@ public class QPossibleValueTranslator
    ///////////////////////////////////////////////////////
    private Map<String, Map<Serializable, String>> possibleValueCache = new HashMap<>();
 
+   private int maxSizePerPvsCache = 50_000;
+
+   private Map<String, QBackendTransaction> transactionsPerTable = new HashMap<>();
+
    // todo not commit - remove instance & session - use Context
 
 
+   boolean useTransactionsAsConnectionPool = false;
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private QBackendTransaction getTransaction(String tableName)
+   {
+      /////////////////////////////////////////////////////////////
+      // mmm, this does cut down on connections used -           //
+      // especially seems helpful in big exports.                //
+      // but, let's just start using connection pools instead... //
+      /////////////////////////////////////////////////////////////
+      if(useTransactionsAsConnectionPool)
+      {
+         try
+         {
+            if(!transactionsPerTable.containsKey(tableName))
+            {
+               transactionsPerTable.put(tableName, new InsertAction().openTransaction(new InsertInput(tableName)));
+            }
+
+            return (transactionsPerTable.get(tableName));
+         }
+         catch(Exception e)
+         {
+            LOG.warn("Error opening transaction for table", logPair("tableName", tableName));
+         }
+      }
+
+      return null;
+   }
+
 
    /*******************************************************************************
     ** Constructor
@@ -123,7 +164,7 @@ public class QPossibleValueTranslator
          return;
       }
 
-      LOG.debug("Translating possible values in [" + records.size() + "] records from the [" + table.getName() + "] table.");
+      LOG.trace("Translating possible values in [" + records.size() + "] records from the [" + table.getName() + "] table.");
       primePvsCache(table, records, queryJoins, limitedToFieldNames);
 
       for(QRecord record : records)
@@ -378,11 +419,11 @@ public class QPossibleValueTranslator
          for(String valueField : valueFields)
          {
             Object value = switch(valueField)
-               {
-                  case "id" -> id;
-                  case "label" -> label;
-                  default -> throw new IllegalArgumentException("Unexpected value field: " + valueField);
-               };
+            {
+               case "id" -> id;
+               case "label" -> label;
+               default -> throw new IllegalArgumentException("Unexpected value field: " + valueField);
+            };
             values.add(Objects.requireNonNullElse(value, ""));
          }
       }
@@ -425,9 +466,10 @@ public class QPossibleValueTranslator
       for(Map.Entry<String, Map<Serializable, String>> entry : possibleValueCache.entrySet())
       {
          int size = entry.getValue().size();
-         if(size > 50_000)
+         if(size > maxSizePerPvsCache)
          {
-            LOG.debug("Found a big PVS cache - clearing it.", logPair("name", entry.getKey()), logPair("size", size));
+            LOG.info("Found a big PVS cache - clearing it.", logPair("name", entry.getKey()), logPair("size", size));
+            entry.getValue().clear();
          }
       }
 
@@ -483,7 +525,7 @@ public class QPossibleValueTranslator
          {
             if(limitedToFieldNames != null && !limitedToFieldNames.contains(fieldNamePrefix + field.getName()))
             {
-               LOG.debug("Skipping cache priming for translation of possible value field [" + fieldNamePrefix + field.getName() + "] - it's not in the limitedToFieldNames set.");
+               LOG.trace("Skipping cache priming for translation of possible value field [" + fieldNamePrefix + field.getName() + "] - it's not in the limitedToFieldNames set.");
                continue;
             }
 
@@ -521,6 +563,7 @@ public class QPossibleValueTranslator
             QueryInput queryInput = new QueryInput();
             queryInput.setTableName(tableName);
             queryInput.setFilter(new QQueryFilter().withCriteria(new QFilterCriteria(primaryKeyField, QCriteriaOperator.IN, page)));
+            queryInput.setTransaction(getTransaction(tableName));
 
             /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
             // when querying for possible values, we do want to generate their display values, which makes record labels, which are usually used as PVS labels //
@@ -556,7 +599,7 @@ public class QPossibleValueTranslator
                queryInput.setFieldsToTranslatePossibleValues(possibleValueFieldsToTranslate);
             }
 
-            LOG.debug("Priming PVS cache for [" + page.size() + "] ids from [" + tableName + "] table.");
+            LOG.trace("Priming PVS cache for [" + page.size() + "] ids from [" + tableName + "] table.");
             QueryOutput queryOutput = new QueryAction().execute(queryInput);
 
             ///////////////////////////////////////////////////////////////////////////////////
@@ -613,4 +656,24 @@ public class QPossibleValueTranslator
       return (count < 5);
    }
 
+
+
+   /*******************************************************************************
+    ** Getter for maxSizePerPvsCache
+    *******************************************************************************/
+   public int getMaxSizePerPvsCache()
+   {
+      return (this.maxSizePerPvsCache);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for maxSizePerPvsCache
+    *******************************************************************************/
+   public void setMaxSizePerPvsCache(int maxSizePerPvsCache)
+   {
+      this.maxSizePerPvsCache = maxSizePerPvsCache;
+   }
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/QValueFormatter.java

@@ -29,14 +29,24 @@ import java.time.LocalTime;
 import java.time.ZonedDateTime;
 import java.time.format.DateTimeFormatter;
 import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Optional;
+import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.logging.QLogger;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.AdornmentType;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.FieldAdornment;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.ExposedJoin;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
 import com.kingsrook.qqq.backend.core.utils.StringUtils;
 import com.kingsrook.qqq.backend.core.utils.ValueUtils;
+import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
 
 
 /*******************************************************************************
@@ -102,6 +112,12 @@ public class QValueFormatter
          {
             return formatLocalTime(lt);
          }
+
+         //////////////////////////////////////////////////////////////////////////////////////////
+         // else, just return the value as a string, rather than going through String.formatted  //
+         // this saves some overhead incurred by String.formatted when called millions of times. //
+         //////////////////////////////////////////////////////////////////////////////////////////
+         return (ValueUtils.getValueAsString(value));
       }
 
       ////////////////////////////////////////////////////////
@@ -126,7 +142,7 @@ public class QValueFormatter
                {
                   return formatValue(displayFormat, ValueUtils.getValueAsBigDecimal(value));
                }
-               else if(e.getMessage().equals("d != java.math.BigDecimal"))
+               else if(e.getMessage().equals("d != java.math.BigDecimal") || e.getMessage().equals("d != java.lang.String"))
                {
                   return formatValue(displayFormat, ValueUtils.getValueAsInteger(value));
                }
@@ -258,6 +274,14 @@ public class QValueFormatter
     *******************************************************************************/
    private static String formatRecordLabelExceptionalCases(QTableMetaData table, QRecord record)
    {
+      //////////////////////////////////////////////////////////////////////////////////////
+      // if the record already has a label (say, from a query-customizer), then return it //
+      //////////////////////////////////////////////////////////////////////////////////////
+      if(record.getRecordLabel() != null)
+      {
+         return (record.getRecordLabel());
+      }
+
       ///////////////////////////////////////////////////////////////////////////////////////
       // if there's no record label format, then just return the primary key display value //
       ///////////////////////////////////////////////////////////////////////////////////////
@@ -292,9 +316,45 @@ public class QValueFormatter
          return;
       }
 
+      Map<String, QFieldMetaData> fieldMap = new HashMap<>();
+
       for(QRecord record : records)
       {
-         setDisplayValuesInRecord(table.getFields().values(), record);
+         for(String fieldName : record.getValues().keySet())
+         {
+            if(!fieldMap.containsKey(fieldName))
+            {
+               try
+               {
+                  if(fieldName.contains("."))
+                  {
+                     String[] nameParts = fieldName.split("\\.", 2);
+                     for(ExposedJoin exposedJoin : CollectionUtils.nonNullList(table.getExposedJoins()))
+                     {
+                        if(exposedJoin.getJoinTable().equals(nameParts[0]))
+                        {
+                           QTableMetaData joinTable = QContext.getQInstance().getTable(nameParts[0]);
+                           fieldMap.put(fieldName, joinTable.getField(nameParts[1]));
+                        }
+                     }
+                  }
+                  else
+                  {
+                     fieldMap.put(fieldName, table.getField(fieldName));
+                  }
+               }
+               catch(Exception e)
+               {
+                  ///////////////////////////////////////////////////////////
+                  // put an empty field in - so no formatting will be done //
+                  ///////////////////////////////////////////////////////////
+                  LOG.info("Error getting field for setting display value", e, logPair("fieldName", fieldName), logPair("tableName", table.getName()));
+                  fieldMap.put(fieldName, new QFieldMetaData());
+               }
+            }
+         }
+
+         setDisplayValuesInRecord(fieldMap, record);
          record.setRecordLabel(formatRecordLabel(table, record));
       }
    }
@@ -319,6 +379,24 @@ public class QValueFormatter
 
 
 
+   /*******************************************************************************
+    ** For a list of records, set their recordLabels and display values
+    *******************************************************************************/
+   public static void setDisplayValuesInRecords(Map<String, QFieldMetaData> fields, List<QRecord> records)
+   {
+      if(records == null)
+      {
+         return;
+      }
+
+      for(QRecord record : records)
+      {
+         setDisplayValuesInRecord(fields, record);
+      }
+   }
+
+
+
    /*******************************************************************************
     ** For a list of records, set their display values
     *******************************************************************************/
@@ -336,6 +414,26 @@ public class QValueFormatter
 
 
 
+   /*******************************************************************************
+    ** For a list of records, set their display values
+    *******************************************************************************/
+   public static void setDisplayValuesInRecord(Map<String, QFieldMetaData> fields, QRecord record)
+   {
+      for(Map.Entry<String, QFieldMetaData> entry : fields.entrySet())
+      {
+         String         fieldName = entry.getKey();
+         QFieldMetaData field     = entry.getValue();
+
+         if(record.getDisplayValue(fieldName) == null)
+         {
+            String formattedValue = formatValue(field, record.getValue(fieldName));
+            record.setDisplayValue(fieldName, formattedValue);
+         }
+      }
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/
@@ -355,4 +453,127 @@ public class QValueFormatter
       }
    }
 
+
+
+   /*******************************************************************************
+    ** For any BLOB type fields in the list of records, change their value to
+    ** the URL where they can be downloaded, and set their display value to a file name.
+    *******************************************************************************/
+   public static void setBlobValuesToDownloadUrls(QTableMetaData table, List<QRecord> records)
+   {
+      for(QFieldMetaData field : table.getFields().values())
+      {
+         if(field.getType().equals(QFieldType.BLOB))
+         {
+            //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+            // file name comes from:                                                                                            //
+            // if there's a FILE_DOWNLOAD adornment, with a FILE_NAME_FIELD value, then the full filename comes from that field //
+            // - unless it was empty - then we do the "default thing":                                                          //
+            // else - the "default thing" is:                                                                                   //
+            // - tableLabel primaryKey fieldLabel                                                                               //
+            // - and - if the FILE_DOWNLOAD adornment had a DEFAULT_EXTENSION, then it gets added (preceded by a dot)           //
+            //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+            Optional<FieldAdornment>  fileDownloadAdornment = field.getAdornment(AdornmentType.FILE_DOWNLOAD);
+            Map<String, Serializable> adornmentValues       = Collections.emptyMap();
+
+            if(fileDownloadAdornment.isPresent())
+            {
+               adornmentValues = fileDownloadAdornment.get().getValues();
+            }
+
+            String fileNameField    = ValueUtils.getValueAsString(adornmentValues.get(AdornmentType.FileDownloadValues.FILE_NAME_FIELD));
+            String fileNameFormat   = ValueUtils.getValueAsString(adornmentValues.get(AdornmentType.FileDownloadValues.FILE_NAME_FORMAT));
+            String defaultExtension = ValueUtils.getValueAsString(adornmentValues.get(AdornmentType.FileDownloadValues.DEFAULT_EXTENSION));
+
+            for(QRecord record : records)
+            {
+               if(!doesFieldHaveValue(field, record))
+               {
+                  continue;
+               }
+
+               Serializable primaryKey = record.getValue(table.getPrimaryKeyField());
+               String       fileName   = null;
+
+               //////////////////////////////////////////////////
+               // try to make file name from the fileNameField //
+               //////////////////////////////////////////////////
+               if(StringUtils.hasContent(fileNameField))
+               {
+                  fileName = record.getValueString(fileNameField);
+               }
+
+               if(!StringUtils.hasContent(fileName))
+               {
+                  if(StringUtils.hasContent(fileNameFormat))
+                  {
+                     @SuppressWarnings("unchecked") // instance validation should make this safe!
+                     List<String> fileNameFormatFields = (List<String>) adornmentValues.get(AdornmentType.FileDownloadValues.FILE_NAME_FORMAT_FIELDS);
+                     List<String> values = fileNameFormatFields.stream().map(f -> ValueUtils.getValueAsString(record.getValue(f))).toList();
+                     fileName = QValueFormatter.formatStringWithValues(fileNameFormat, values);
+                  }
+               }
+
+               if(!StringUtils.hasContent(fileName))
+               {
+                  //////////////////////////////////
+                  // make default name if missing //
+                  //////////////////////////////////
+                  fileName = table.getLabel() + " " + primaryKey + " " + field.getLabel();
+
+                  if(StringUtils.hasContent(defaultExtension))
+                  {
+                     //////////////////////////////////////////
+                     // add default extension if we have one //
+                     //////////////////////////////////////////
+                     fileName += "." + defaultExtension;
+                  }
+               }
+
+               record.setValue(field.getName(), "/data/" + table.getName() + "/" + primaryKey + "/" + field.getName() + "/" + fileName);
+               record.setDisplayValue(field.getName(), fileName);
+            }
+         }
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private static boolean doesFieldHaveValue(QFieldMetaData field, QRecord record)
+   {
+      boolean fieldHasValue = false;
+
+      try
+      {
+         if(record.getValue(field.getName()) != null)
+         {
+            fieldHasValue = true;
+         }
+         else if(field.getIsHeavy())
+         {
+            /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+            // heavy fields that weren't fetched - they should have a backend-detail specifying their length (or null if null) //
+            /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+            Map<String, Serializable> heavyFieldLengths = (Map<String, Serializable>) record.getBackendDetail(QRecord.BACKEND_DETAILS_TYPE_HEAVY_FIELD_LENGTHS);
+            if(heavyFieldLengths != null)
+            {
+               Integer fieldLength = ValueUtils.getValueAsInteger(heavyFieldLengths.get(field.getName()));
+               if(fieldLength != null && fieldLength > 0)
+               {
+                  fieldHasValue = true;
+               }
+            }
+         }
+      }
+      catch(Exception e)
+      {
+         LOG.info("Error checking if field has value", e, logPair("fieldName", field.getName()), logPair("record", record));
+      }
+
+      return fieldHasValue;
+   }
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/SearchPossibleValueSourceAction.java

@@ -23,6 +23,8 @@ package com.kingsrook.qqq.backend.core.actions.values;
 
 
 import java.io.Serializable;
+import java.time.Instant;
+import java.time.LocalDate;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Objects;
@@ -208,46 +210,57 @@ public class SearchPossibleValueSourceAction
       }
       else
       {
-         if(StringUtils.hasContent(input.getSearchTerm()))
+         String searchTerm = input.getSearchTerm();
+         if(StringUtils.hasContent(searchTerm))
          {
             for(String valueField : possibleValueSource.getSearchFields())
             {
-               QFieldMetaData field = table.getField(valueField);
-               if(field.getType().equals(QFieldType.STRING))
+               try
                {
-                  queryFilter.addCriteria(new QFilterCriteria(valueField, QCriteriaOperator.STARTS_WITH, List.of(input.getSearchTerm())));
-               }
-               else if(field.getType().equals(QFieldType.DATE) || field.getType().equals(QFieldType.DATE_TIME))
-               {
-                  LOG.debug("Not querying PVS [" + possibleValueSource.getName() + "] on date field [" + field.getName() + "]");
-                  // todo - what? queryFilter.addCriteria(new QFilterCriteria(valueField, QCriteriaOperator.STARTS_WITH, List.of(input.getSearchTerm())));
-               }
-               else
-               {
-                  try
+                  QFieldMetaData field = table.getField(valueField);
+                  if(field.getType().equals(QFieldType.STRING))
                   {
-                     Integer valueAsInteger = ValueUtils.getValueAsInteger(input.getSearchTerm());
+                     queryFilter.addCriteria(new QFilterCriteria(valueField, QCriteriaOperator.STARTS_WITH, List.of(searchTerm)));
+                  }
+                  else if(field.getType().equals(QFieldType.DATE))
+                  {
+                     LocalDate searchDate = ValueUtils.getValueAsLocalDate(searchTerm);
+                     if(searchDate != null)
+                     {
+                        queryFilter.addCriteria(new QFilterCriteria(valueField, QCriteriaOperator.EQUALS, searchDate));
+                     }
+                  }
+                  else if(field.getType().equals(QFieldType.DATE_TIME))
+                  {
+                     Instant searchDate = ValueUtils.getValueAsInstant(searchTerm);
+                     if(searchDate != null)
+                     {
+                        queryFilter.addCriteria(new QFilterCriteria(valueField, QCriteriaOperator.EQUALS, searchDate));
+                     }
+                  }
+                  else
+                  {
+                     Integer valueAsInteger = ValueUtils.getValueAsInteger(searchTerm);
                      if(valueAsInteger != null)
                      {
                         queryFilter.addCriteria(new QFilterCriteria(valueField, QCriteriaOperator.EQUALS, List.of(valueAsInteger)));
                      }
                   }
-                  catch(Exception e)
-                  {
-                     ////////////////////////////////////////////////////////
-                     // write a FALSE criteria if the value isn't a number //
-                     ////////////////////////////////////////////////////////
-                     queryFilter.addCriteria(new QFilterCriteria(valueField, QCriteriaOperator.IN, List.of()));
-                  }
+               }
+               catch(Exception e)
+               {
+                  //////////////////////////////////////////////////////////////////////////////////////////
+                  // write a FALSE criteria upon exceptions (e.g., type conversion fails)                 //
+                  // Why are we doing this?  so a single-field query finds nothing instead of everything. //
+                  //////////////////////////////////////////////////////////////////////////////////////////
+                  queryFilter.addCriteria(new QFilterCriteria(valueField, QCriteriaOperator.IN, List.of()));
                }
             }
          }
       }
 
-      queryFilter.setOrderBys(possibleValueSource.getOrderByFields());
-
       // todo - skip & limit as params
-      queryInput.setLimit(250);
+      queryFilter.setLimit(250);
 
       ///////////////////////////////////////////////////////////////////////////////////////////////////////
       // if given a default filter, make it the 'top level' filter and the one we just created a subfilter //
@@ -257,6 +270,9 @@ public class SearchPossibleValueSourceAction
          input.getDefaultQueryFilter().addSubFilter(queryFilter);
          queryFilter = input.getDefaultQueryFilter();
       }
+
+      queryFilter.setOrderBys(possibleValueSource.getOrderByFields());
+
       queryInput.setFilter(queryFilter);
 
       QueryOutput             queryOutput     = new QueryAction().execute(queryInput);

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/ValueBehaviorApplier.java

@@ -29,6 +29,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.ValueTooLongBehavior;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+import com.kingsrook.qqq.backend.core.model.statusmessages.BadInputStatusMessage;
 import com.kingsrook.qqq.backend.core.utils.StringUtils;
 
 
@@ -76,7 +77,7 @@ public class ValueBehaviorApplier
                {
                   case TRUNCATE -> record.setValue(fieldName, StringUtils.safeTruncate(value, field.getMaxLength()));
                   case TRUNCATE_ELLIPSIS -> record.setValue(fieldName, StringUtils.safeTruncate(value, field.getMaxLength(), "..."));
-                  case ERROR -> record.addError("The value for " + field.getLabel() + " is too long (max allowed length=" + field.getMaxLength() + ")");
+                  case ERROR -> record.addError(new BadInputStatusMessage("The value for " + field.getLabel() + " is too long (max allowed length=" + field.getMaxLength() + ")"));
                   case PASS_THROUGH ->
                   {
                   }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/adapters/CsvToQRecordAdapter.java

@@ -31,11 +31,14 @@ import java.util.List;
 import java.util.Map;
 import java.util.function.Consumer;
 import com.kingsrook.qqq.backend.core.actions.reporting.RecordPipe;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.model.actions.shared.mapping.AbstractQFieldMapping;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+import com.kingsrook.qqq.backend.core.model.statusmessages.BadInputStatusMessage;
 import com.kingsrook.qqq.backend.core.utils.StringUtils;
+import com.kingsrook.qqq.backend.core.utils.ValueUtils;
 import org.apache.commons.csv.CSVFormat;
 import org.apache.commons.csv.CSVParser;
 import org.apache.commons.csv.CSVRecord;
@@ -61,7 +64,7 @@ public class CsvToQRecordAdapter
     ** using a given mapping.
     **
     *******************************************************************************/
-   public void buildRecordsFromCsv(RecordPipe recordPipe, String csv, QTableMetaData table, AbstractQFieldMapping<?> mapping, Consumer<QRecord> recordCustomizer)
+   public void buildRecordsFromCsv(RecordPipe recordPipe, String csv, QTableMetaData table, AbstractQFieldMapping<?> mapping, Consumer<QRecord> recordCustomizer) throws QException
    {
       buildRecordsFromCsv(new InputWrapper().withRecordPipe(recordPipe).withCsv(csv).withTable(table).withMapping(mapping).withRecordCustomizer(recordCustomizer));
    }
@@ -73,7 +76,7 @@ public class CsvToQRecordAdapter
     ** using a given mapping.
     **
     *******************************************************************************/
-   public List<QRecord> buildRecordsFromCsv(String csv, QTableMetaData table, AbstractQFieldMapping<?> mapping)
+   public List<QRecord> buildRecordsFromCsv(String csv, QTableMetaData table, AbstractQFieldMapping<?> mapping) throws QException
    {
       buildRecordsFromCsv(new InputWrapper().withCsv(csv).withTable(table).withMapping(mapping));
       return (recordList);
@@ -87,7 +90,7 @@ public class CsvToQRecordAdapter
     **
     ** todo - meta-data validation, type handling
     *******************************************************************************/
-   public void buildRecordsFromCsv(InputWrapper inputWrapper)
+   public void buildRecordsFromCsv(InputWrapper inputWrapper) throws QException
    {
       String                   csv              = inputWrapper.getCsv();
       AbstractQFieldMapping<?> mapping          = inputWrapper.getMapping();
@@ -155,14 +158,21 @@ public class CsvToQRecordAdapter
                // now move values into the QRecord, using the mapping to get the 'header' corresponding to each QField //
                //////////////////////////////////////////////////////////////////////////////////////////////////////////
                QRecord qRecord = new QRecord();
-               for(QFieldMetaData field : table.getFields().values())
+               try
                {
-                  String fieldSource = mapping == null ? field.getName() : String.valueOf(mapping.getFieldSource(field.getName()));
-                  fieldSource = adjustHeaderCase(fieldSource, inputWrapper);
-                  qRecord.setValue(field.getName(), csvValues.get(fieldSource));
-               }
+                  for(QFieldMetaData field : table.getFields().values())
+                  {
+                     String fieldSource = mapping == null ? field.getName() : String.valueOf(mapping.getFieldSource(field.getName()));
+                     fieldSource = adjustHeaderCase(fieldSource, inputWrapper);
+                     setValue(inputWrapper, qRecord, field, csvValues.get(fieldSource));
+                  }
 
-               runRecordCustomizer(recordCustomizer, qRecord);
+                  runRecordCustomizer(recordCustomizer, qRecord);
+               }
+               catch(Exception e)
+               {
+                  qRecord.addError(new BadInputStatusMessage("Error parsing line #" + (recordCount + 1) + ": " + e.getMessage()));
+               }
                addRecord(qRecord);
 
                recordCount++;
@@ -201,13 +211,20 @@ public class CsvToQRecordAdapter
                // now move values into the QRecord, using the mapping to get the 'header' corresponding to each QField //
                //////////////////////////////////////////////////////////////////////////////////////////////////////////
                QRecord qRecord = new QRecord();
-               for(QFieldMetaData field : table.getFields().values())
+               try
                {
-                  Integer fieldIndex = (Integer) mapping.getFieldSource(field.getName());
-                  qRecord.setValue(field.getName(), csvValues.get(fieldIndex));
-               }
+                  for(QFieldMetaData field : table.getFields().values())
+                  {
+                     Integer fieldIndex = (Integer) mapping.getFieldSource(field.getName());
+                     setValue(inputWrapper, qRecord, field, csvValues.get(fieldIndex));
+                  }
 
-               runRecordCustomizer(recordCustomizer, qRecord);
+                  runRecordCustomizer(recordCustomizer, qRecord);
+               }
+               catch(Exception e)
+               {
+                  qRecord.addError(new BadInputStatusMessage("Error parsing line #" + (recordCount + 1) + ": " + e.getMessage()));
+               }
                addRecord(qRecord);
 
                recordCount++;
@@ -230,6 +247,23 @@ public class CsvToQRecordAdapter
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private void setValue(InputWrapper inputWrapper, QRecord qRecord, QFieldMetaData field, String valueString)
+   {
+      if(inputWrapper.doCorrectValueTypes)
+      {
+         qRecord.setValue(field.getName(), ValueUtils.getValueAsFieldType(field.getType(), valueString));
+      }
+      else
+      {
+         qRecord.setValue(field.getName(), valueString);
+      }
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/
@@ -297,7 +331,7 @@ public class CsvToQRecordAdapter
    /*******************************************************************************
     ** Add a record - either to the pipe, or list, whichever we're building.
     *******************************************************************************/
-   private void addRecord(QRecord record)
+   private void addRecord(QRecord record) throws QException
    {
       if(recordPipe != null)
       {
@@ -340,6 +374,7 @@ public class CsvToQRecordAdapter
       private AbstractQFieldMapping<?> mapping;
       private Consumer<QRecord>        recordCustomizer;
       private Integer                  limit;
+      private boolean                  doCorrectValueTypes = false;
 
       private boolean caseSensitiveHeaders = false;
 
@@ -581,6 +616,40 @@ public class CsvToQRecordAdapter
          return (this);
       }
 
+
+
+      /*******************************************************************************
+       ** Getter for doCorrectValueTypes
+       **
+       *******************************************************************************/
+      public boolean getDoCorrectValueTypes()
+      {
+         return doCorrectValueTypes;
+      }
+
+
+
+      /*******************************************************************************
+       ** Setter for doCorrectValueTypes
+       **
+       *******************************************************************************/
+      public void setDoCorrectValueTypes(boolean doCorrectValueTypes)
+      {
+         this.doCorrectValueTypes = doCorrectValueTypes;
+      }
+
+
+
+      /*******************************************************************************
+       ** Fluent setter for doCorrectValueTypes
+       **
+       *******************************************************************************/
+      public InputWrapper withDoCorrectValueTypes(boolean doCorrectValueTypes)
+      {
+         this.doCorrectValueTypes = doCorrectValueTypes;
+         return (this);
+      }
+
    }
 
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/adapters/QRecordToCsvAdapter.java

@@ -80,8 +80,23 @@ public class QRecordToCsvAdapter
    /*******************************************************************************
     ** todo - kinda weak... can we find this in a CSV lib??
     *******************************************************************************/
-   private String sanitize(String value)
+   static String sanitize(String value)
    {
-      return (value.replaceAll("\"", "\"\"").replaceAll("\n", " "));
+      /////////////////////////////////////////////////////////////////////////////////////
+      // especially in big exports, we see a TON of memory allocated and CPU spent here, //
+      // if we just blindly replaceAll.  So, only do it if needed.                       //
+      /////////////////////////////////////////////////////////////////////////////////////
+      if(value.contains("\""))
+      {
+         value = value.replaceAll("\"", "\"\"");
+      }
+
+      if(value.contains("\n"))
+      {
+         value = value.replaceAll("\n", " ");
+      }
+
+      return (value);
    }
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/context/QContext.java

@@ -84,7 +84,7 @@ public class QContext
          actionStackThreadLocal.get().add(actionInput);
       }
 
-      if(!qInstance.getHasBeenValidated())
+      if(qInstance != null && !qInstance.getHasBeenValidated())
       {
          try
          {

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/exceptions/QBadRequestException.java

@@ -0,0 +1,53 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.exceptions;
+
+
+/*******************************************************************************
+ ** User-facing exception for when user provided bad or missing data in their request
+ **
+ *******************************************************************************/
+public class QBadRequestException extends QUserFacingException
+{
+
+
+   /*******************************************************************************
+    ** Constructor of message
+    **
+    *******************************************************************************/
+   public QBadRequestException(String message)
+   {
+      super(message);
+   }
+
+
+
+   /*******************************************************************************
+    ** Constructor of message & cause
+    **
+    *******************************************************************************/
+   public QBadRequestException(String message, Throwable cause)
+   {
+      super(message, cause);
+   }
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/exceptions/QException.java

@@ -22,12 +22,19 @@
 package com.kingsrook.qqq.backend.core.exceptions;
 
 
+import org.apache.logging.log4j.Level;
+
+
 /*******************************************************************************
  * Base class for checked exceptions thrown in qqq.
  *
  *******************************************************************************/
 public class QException extends Exception
 {
+   private boolean hasLoggedWarning;
+   private boolean hasLoggedError;
+
+
 
    /*******************************************************************************
     ** Constructor of message
@@ -59,4 +66,102 @@ public class QException extends Exception
    {
       super(message, cause);
    }
+
+
+
+   /*******************************************************************************
+    ** Getter for hasLoggedWarning
+    *******************************************************************************/
+   public boolean getHasLoggedWarning()
+   {
+      return (this.hasLoggedWarning);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for hasLoggedWarning
+    *******************************************************************************/
+   public void setHasLoggedWarning(boolean hasLoggedWarning)
+   {
+      this.hasLoggedWarning = hasLoggedWarning;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for hasLoggedWarning
+    *******************************************************************************/
+   public QException withHasLoggedWarning(boolean hasLoggedWarning)
+   {
+      this.hasLoggedWarning = hasLoggedWarning;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for hasLoggedError
+    *******************************************************************************/
+   public boolean getHasLoggedError()
+   {
+      return (this.hasLoggedError);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for hasLoggedError
+    *******************************************************************************/
+   public void setHasLoggedError(boolean hasLoggedError)
+   {
+      this.hasLoggedError = hasLoggedError;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for hasLoggedError
+    *******************************************************************************/
+   public QException withHasLoggedError(boolean hasLoggedError)
+   {
+      this.hasLoggedError = hasLoggedError;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** helper function for getting if level logged
+    *******************************************************************************/
+   public boolean hasLoggedLevel(Level level)
+   {
+      if(Level.WARN.equals(level))
+      {
+         return (hasLoggedWarning);
+      }
+      if(Level.ERROR.equals(level))
+      {
+         return (hasLoggedError);
+      }
+      return (false);
+   }
+
+
+
+   /*******************************************************************************
+    ** helper function for setting if level logged
+    *******************************************************************************/
+   public void setHasLoggedLevel(Level level)
+   {
+      if(Level.WARN.equals(level))
+      {
+         setHasLoggedWarning(true);
+      }
+      if(Level.ERROR.equals(level))
+      {
+         setHasLoggedError(true);
+      }
+   }
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java

@@ -27,17 +27,19 @@ import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
+import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
 import java.util.Set;
 import java.util.stream.Collectors;
+import com.kingsrook.qqq.backend.core.actions.metadata.JoinGraph;
 import com.kingsrook.qqq.backend.core.actions.permissions.BulkTableActionProcessPermissionChecker;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.logging.QLogger;
 import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
 import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
-import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeUsage;
 import com.kingsrook.qqq.backend.core.model.metadata.dashboard.QWidgetMetaDataInterface;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.AdornmentType;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.FieldAdornment;
@@ -56,23 +58,26 @@ import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendComponen
 import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendStepMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.processes.QStepMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.processes.QSupplementalProcessMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportDataSource;
 import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportView;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.ExposedJoin;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QFieldSection;
-import com.kingsrook.qqq.backend.core.model.metadata.tables.QMiddlewareTableMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QSupplementalTableMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.Tier;
+import com.kingsrook.qqq.backend.core.processes.implementations.bulk.delete.BulkDeleteLoadStep;
 import com.kingsrook.qqq.backend.core.processes.implementations.bulk.delete.BulkDeleteTransformStep;
+import com.kingsrook.qqq.backend.core.processes.implementations.bulk.edit.BulkEditLoadStep;
 import com.kingsrook.qqq.backend.core.processes.implementations.bulk.edit.BulkEditTransformStep;
 import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkInsertExtractStep;
+import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkInsertLoadStep;
 import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkInsertTransformStep;
 import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.ExtractViaQueryStep;
-import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.LoadViaDeleteStep;
-import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.LoadViaInsertStep;
-import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.LoadViaUpdateStep;
 import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess;
 import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
+import com.kingsrook.qqq.backend.core.utils.ListingHash;
 import com.kingsrook.qqq.backend.core.utils.StringUtils;
 
 
@@ -87,11 +92,20 @@ public class QInstanceEnricher
 
    private final QInstance qInstance;
 
+   private JoinGraph joinGraph;
+
    //////////////////////////////////////////////////////////
    // todo - come up w/ a way for app devs to set configs! //
    //////////////////////////////////////////////////////////
    private boolean configRemoveIdFromNameWhenCreatingPossibleValueFieldLabels = true;
 
+   //////////////////////////////////////////////////////////////////////////////////////////////////
+   // let an instance define mappings to be applied during name-to-label enrichments,              //
+   // e.g., to avoid ever incorrectly camel-casing an acronym (e.g., "Tla" should always be "TLA") //
+   // or to expand abbreviations in code (e.g., "Addr" should always be "Address"                  //
+   //////////////////////////////////////////////////////////////////////////////////////////////////
+   private static final Map<String, String> labelMappings = new LinkedHashMap<>();
+
 
 
    /*******************************************************************************
@@ -144,6 +158,81 @@ public class QInstanceEnricher
       {
          qInstance.getWidgets().values().forEach(this::enrichWidget);
       }
+
+      enrichJoins();
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private void enrichJoins()
+   {
+      try
+      {
+         joinGraph = new JoinGraph(qInstance);
+
+         for(QTableMetaData table : CollectionUtils.nonNullMap(qInstance.getTables()).values())
+         {
+            Set<JoinGraph.JoinConnectionList> joinConnections = joinGraph.getJoinConnections(table.getName());
+            for(ExposedJoin exposedJoin : CollectionUtils.nonNullList(table.getExposedJoins()))
+            {
+               /////////////////////////////////////////////////////////////////////////////////////////////////////
+               // proceed with caution - remember, validator will fail the instance if things are missing/invalid //
+               /////////////////////////////////////////////////////////////////////////////////////////////////////
+               if(exposedJoin.getJoinTable() != null)
+               {
+                  QTableMetaData joinTable = qInstance.getTable(exposedJoin.getJoinTable());
+                  if(joinTable != null)
+                  {
+                     //////////////////////////////////////////////////////////////////////////////////
+                     // default the exposed join's label to the join table's label, if it wasn't set //
+                     //////////////////////////////////////////////////////////////////////////////////
+                     if(!StringUtils.hasContent(exposedJoin.getLabel()))
+                     {
+                        exposedJoin.setLabel(joinTable.getLabel());
+                     }
+
+                     ///////////////////////////////////////////////////////////////////////////////
+                     // default the exposed join's join-path from the joinGraph, if it wasn't set //
+                     ///////////////////////////////////////////////////////////////////////////////
+                     if(CollectionUtils.nullSafeIsEmpty(exposedJoin.getJoinPath()))
+                     {
+                        List<JoinGraph.JoinConnectionList> eligibleJoinConnections = new ArrayList<>();
+                        for(JoinGraph.JoinConnectionList joinConnection : joinConnections)
+                        {
+                           if(joinTable.getName().equals(joinConnection.list().get(joinConnection.list().size() - 1).joinTable()))
+                           {
+                              eligibleJoinConnections.add(joinConnection);
+                           }
+                        }
+
+                        if(eligibleJoinConnections.isEmpty())
+                        {
+                           throw (new QException("Could not infer a joinPath for table [" + table.getName() + "], exposedJoin to [" + exposedJoin.getJoinTable() + "]:  No join connections between these tables exist in this instance."));
+                        }
+                        else if(eligibleJoinConnections.size() > 1)
+                        {
+                           throw (new QException("Could not infer a joinPath for table [" + table.getName() + "], exposedJoin to [" + exposedJoin.getJoinTable() + "]:  "
+                              + eligibleJoinConnections.size() + " join connections exist between these tables.  You need to specify one:\n"
+                              + StringUtils.join("\n", eligibleJoinConnections.stream().map(jcl -> jcl.getJoinNamesAsString()).toList()) + "."
+                           ));
+                        }
+                        else
+                        {
+                           exposedJoin.setJoinPath(eligibleJoinConnections.get(0).getJoinNamesAsList());
+                        }
+                     }
+                  }
+               }
+            }
+         }
+      }
+      catch(Exception e)
+      {
+         throw (new RuntimeException("Error enriching instance joins", e));
+      }
    }
 
 
@@ -182,9 +271,9 @@ public class QInstanceEnricher
       {
          table.getFields().values().forEach(this::enrichField);
 
-         for(QMiddlewareTableMetaData middlewareTableMetaData : CollectionUtils.nonNullMap(table.getMiddlewareMetaData()).values())
+         for(QSupplementalTableMetaData supplementalTableMetaData : CollectionUtils.nonNullMap(table.getSupplementalMetaData()).values())
          {
-            middlewareTableMetaData.enrich(table);
+            supplementalTableMetaData.enrich(qInstance, table);
          }
       }
 
@@ -288,6 +377,11 @@ public class QInstanceEnricher
          process.getStepList().forEach(this::enrichStep);
       }
 
+      for(QSupplementalProcessMetaData supplementalProcessMetaData : CollectionUtils.nonNullMap(process.getSupplementalMetaData()).values())
+      {
+         supplementalProcessMetaData.enrich(this, process);
+      }
+
       enrichPermissionRules(process);
    }
 
@@ -438,11 +532,70 @@ public class QInstanceEnricher
          enrichAppSection(section);
       }
 
+      ensureAppSectionMembersAreAppChildren(app);
+
       enrichPermissionRules(app);
    }
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private void ensureAppSectionMembersAreAppChildren(QAppMetaData app)
+   {
+      ListingHash<Class<? extends QAppChildMetaData>, String> childrenByType = new ListingHash<>();
+      childrenByType.put(QTableMetaData.class, new ArrayList<>());
+      childrenByType.put(QProcessMetaData.class, new ArrayList<>());
+      childrenByType.put(QReportMetaData.class, new ArrayList<>());
+
+      for(QAppChildMetaData qAppChildMetaData : CollectionUtils.nonNullList(app.getChildren()))
+      {
+         childrenByType.add(qAppChildMetaData.getClass(), qAppChildMetaData.getName());
+      }
+
+      for(QAppSection section : CollectionUtils.nonNullList(app.getSections()))
+      {
+         for(String tableName : CollectionUtils.nonNullList(section.getTables()))
+         {
+            if(!childrenByType.get(QTableMetaData.class).contains(tableName))
+            {
+               QTableMetaData table = qInstance.getTable(tableName);
+               if(table != null)
+               {
+                  app.withChild(table);
+               }
+            }
+         }
+
+         for(String processName : CollectionUtils.nonNullList(section.getProcesses()))
+         {
+            if(!childrenByType.get(QProcessMetaData.class).contains(processName))
+            {
+               QProcessMetaData process = qInstance.getProcess(processName);
+               if(process != null)
+               {
+                  app.withChild(process);
+               }
+            }
+         }
+
+         for(String reportName : CollectionUtils.nonNullList(section.getReports()))
+         {
+            if(!childrenByType.get(QReportMetaData.class).contains(reportName))
+            {
+               QReportMetaData report = qInstance.getReport(reportName);
+               if(report != null)
+               {
+                  app.withChild(report);
+               }
+            }
+         }
+      }
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/
@@ -562,7 +715,17 @@ public class QInstanceEnricher
          ////////////////////////////////////////////////////////////////
          .replaceAll("([0-9])([A-Za-z])", "$1 $2");
 
-      return (name.substring(0, 1).toUpperCase(Locale.ROOT) + suffix);
+      String label = name.substring(0, 1).toUpperCase(Locale.ROOT) + suffix;
+
+      /////////////////////////////////////////////////////////////////////////////////////////////
+      // apply any label mappings - e.g., to force app-specific acronyms/initialisms to all-caps //
+      /////////////////////////////////////////////////////////////////////////////////////////////
+      for(Map.Entry<String, String> entry : labelMappings.entrySet())
+      {
+         label = label.replaceAll(entry.getKey(), entry.getValue());
+      }
+
+      return (label);
    }
 
 
@@ -618,7 +781,7 @@ public class QInstanceEnricher
       QProcessMetaData process = StreamedETLWithFrontendProcess.defineProcessMetaData(
             BulkInsertExtractStep.class,
             BulkInsertTransformStep.class,
-            LoadViaInsertStep.class,
+            BulkInsertLoadStep.class,
             values
          )
          .withName(processName)
@@ -626,7 +789,7 @@ public class QInstanceEnricher
          .withTableName(table.getName())
          .withIsHidden(true)
          .withPermissionRules(qInstance.getDefaultPermissionRules().clone()
-            .withCustomPermissionChecker(new QCodeReference(BulkTableActionProcessPermissionChecker.class, QCodeUsage.CUSTOMIZER)));
+            .withCustomPermissionChecker(new QCodeReference(BulkTableActionProcessPermissionChecker.class)));
 
       List<QFieldMetaData> editableFields = new ArrayList<>();
       for(QFieldSection section : CollectionUtils.nonNullList(table.getSections()))
@@ -636,7 +799,7 @@ public class QInstanceEnricher
             try
             {
                QFieldMetaData field = table.getField(fieldName);
-               if(field.getIsEditable())
+               if(field.getIsEditable() && !field.getType().equals(QFieldType.BLOB))
                {
                   editableFields.add(field);
                }
@@ -655,7 +818,7 @@ public class QInstanceEnricher
       QFrontendStepMetaData uploadScreen = new QFrontendStepMetaData()
          .withName("upload")
          .withLabel("Upload File")
-         .withFormField(new QFieldMetaData("theFile", QFieldType.BLOB).withIsRequired(true))
+         .withFormField(new QFieldMetaData("theFile", QFieldType.BLOB).withLabel(table.getLabel() + " File").withIsRequired(true))
          .withComponent(new QFrontendComponentMetaData()
             .withType(QComponentType.HELP_TEXT)
             .withValue("previewText", "file upload instructions")
@@ -682,7 +845,7 @@ public class QInstanceEnricher
       QProcessMetaData process = StreamedETLWithFrontendProcess.defineProcessMetaData(
             ExtractViaQueryStep.class,
             BulkEditTransformStep.class,
-            LoadViaUpdateStep.class,
+            BulkEditLoadStep.class,
             values
          )
          .withName(processName)
@@ -690,10 +853,11 @@ public class QInstanceEnricher
          .withTableName(table.getName())
          .withIsHidden(true)
          .withPermissionRules(qInstance.getDefaultPermissionRules().clone()
-            .withCustomPermissionChecker(new QCodeReference(BulkTableActionProcessPermissionChecker.class, QCodeUsage.CUSTOMIZER)));
+            .withCustomPermissionChecker(new QCodeReference(BulkTableActionProcessPermissionChecker.class)));
 
       List<QFieldMetaData> editableFields = table.getFields().values().stream()
          .filter(QFieldMetaData::getIsEditable)
+         .filter(f -> !f.getType().equals(QFieldType.BLOB))
          .toList();
 
       QFrontendStepMetaData editScreen = new QFrontendStepMetaData()
@@ -729,7 +893,7 @@ public class QInstanceEnricher
       QProcessMetaData process = StreamedETLWithFrontendProcess.defineProcessMetaData(
             ExtractViaQueryStep.class,
             BulkDeleteTransformStep.class,
-            LoadViaDeleteStep.class,
+            BulkDeleteLoadStep.class,
             values
          )
          .withName(processName)
@@ -737,7 +901,7 @@ public class QInstanceEnricher
          .withTableName(table.getName())
          .withIsHidden(true)
          .withPermissionRules(qInstance.getDefaultPermissionRules().clone()
-            .withCustomPermissionChecker(new QCodeReference(BulkTableActionProcessPermissionChecker.class, QCodeUsage.CUSTOMIZER)));
+            .withCustomPermissionChecker(new QCodeReference(BulkTableActionProcessPermissionChecker.class)));
 
       List<QFieldMetaData> tableFields = table.getFields().values().stream().toList();
       process.getFrontendStep("review").setRecordListFields(tableFields);
@@ -955,7 +1119,8 @@ public class QInstanceEnricher
       {
          for(String fieldName : table.getFields().keySet())
          {
-            if(!usedFieldNames.contains(fieldName))
+            QFieldMetaData field = table.getField(fieldName);
+            if(!field.getIsHidden() && !usedFieldNames.contains(fieldName))
             {
                otherSection.getFieldNames().add(fieldName);
                usedFieldNames.add(fieldName);
@@ -1015,4 +1180,44 @@ public class QInstanceEnricher
       }
    }
 
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public JoinGraph getJoinGraph()
+   {
+      return (this.joinGraph);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public static void addLabelMapping(String from, String to)
+   {
+      labelMappings.put(from, to);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public static void removeLabelMapping(String from)
+   {
+      labelMappings.remove(from);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public static void clearLabelMappings()
+   {
+      labelMappings.clear();
+   }
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceHelpContentManager.java

@@ -0,0 +1,269 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2023.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.instances;
+
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.ListIterator;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
+import com.kingsrook.qqq.backend.core.logging.QLogger;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.helpcontent.HelpContent;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.help.HelpFormat;
+import com.kingsrook.qqq.backend.core.model.metadata.help.HelpRole;
+import com.kingsrook.qqq.backend.core.model.metadata.help.QHelpContent;
+import com.kingsrook.qqq.backend.core.model.metadata.help.QHelpRole;
+import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QFieldSection;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
+import com.kingsrook.qqq.backend.core.utils.StringUtils;
+import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
+
+
+/*******************************************************************************
+ ** Utility methods for working with (dynamic, from a table) HelpContent - and
+ ** putting it into meta-data in a QInstance.
+ *******************************************************************************/
+public class QInstanceHelpContentManager
+{
+   private static final QLogger LOG = QLogger.getLogger(QInstanceHelpContentManager.class);
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public static void loadHelpContent(QInstance qInstance)
+   {
+      try
+      {
+         if(qInstance.getTable(HelpContent.TABLE_NAME) == null)
+         {
+            return;
+         }
+
+         QueryInput queryInput = new QueryInput();
+         queryInput.setTableName(HelpContent.TABLE_NAME);
+         QueryOutput queryOutput = new QueryAction().execute(queryInput);
+         for(QRecord record : queryOutput.getRecords())
+         {
+            processHelpContentRecord(qInstance, record);
+         }
+      }
+      catch(Exception e)
+      {
+         LOG.error("Error loading help content", e);
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public static void processHelpContentRecord(QInstance qInstance, QRecord record)
+   {
+      try
+      {
+         /////////////////////////////////////////////////
+         // parse the key into its parts that we expect //
+         /////////////////////////////////////////////////
+         String              key            = record.getValueString("key");
+         Map<String, String> nameValuePairs = new HashMap<>();
+         for(String part : key.split(";"))
+         {
+            String[] parts = part.split(":");
+            nameValuePairs.put(parts[0], parts[1]);
+         }
+
+         String tableName   = nameValuePairs.get("table");
+         String processName = nameValuePairs.get("process");
+         String fieldName   = nameValuePairs.get("field");
+         String sectionName = nameValuePairs.get("section");
+
+         ///////////////////////////////////////////////////////////
+         // build a help content meta-data object from the record //
+         ///////////////////////////////////////////////////////////
+         QHelpContent helpContent = new QHelpContent()
+            .withContent(record.getValueString("content"))
+            .withRole(QHelpRole.valueOf(record.getValueString("role"))); // mmm, we could fall down a bit here w/ other app-defined roles...
+
+         if(StringUtils.hasContent(record.getValueString("format")))
+         {
+            helpContent.setFormat(HelpFormat.valueOf(record.getValueString("format")));
+         }
+         Set<HelpRole> roles = helpContent.getRoles();
+
+         ///////////////////////////////////////////////////////////////////////////////////////////////////
+         // check - if there are no contents, then let's remove this help content from the container      //
+         // (note pre-delete customizer will take advantage of this, passing in empty content on purpose) //
+         ///////////////////////////////////////////////////////////////////////////////////////////////////
+         if(!StringUtils.hasContent(helpContent.getContent()))
+         {
+            helpContent = null;
+         }
+
+         ///////////////////////////////////////////////////////////////////////////////////
+         // look at what parts of the key we got, and find the meta-data object to update //
+         ///////////////////////////////////////////////////////////////////////////////////
+         if(StringUtils.hasContent(tableName))
+         {
+            QTableMetaData table = qInstance.getTable(tableName);
+            if(table == null)
+            {
+               LOG.info("Unrecognized table in help content", logPair("key", key));
+               return;
+            }
+
+            if(StringUtils.hasContent(fieldName))
+            {
+               //////////////////////////
+               // handle a table field //
+               //////////////////////////
+               QFieldMetaData field = table.getFields().get(fieldName);
+               if(field == null)
+               {
+                  LOG.info("Unrecognized table field in help content", logPair("key", key));
+                  return;
+               }
+
+               if(helpContent != null)
+               {
+                  field.withHelpContent(helpContent);
+               }
+               else
+               {
+                  field.removeHelpContent(roles);
+               }
+            }
+            else if(StringUtils.hasContent(sectionName))
+            {
+               ////////////////////////////
+               // handle a table section //
+               ////////////////////////////
+               Optional<QFieldSection> optionalSection = table.getSections().stream().filter(s -> sectionName.equals(s.getName())).findFirst();
+               if(optionalSection.isEmpty())
+               {
+                  LOG.info("Unrecognized table section in help content", logPair("key", key));
+                  return;
+               }
+
+               if(helpContent != null)
+               {
+                  optionalSection.get().withHelpContent(helpContent);
+               }
+               else
+               {
+                  optionalSection.get().removeHelpContent(roles);
+               }
+            }
+         }
+         else if(StringUtils.hasContent(processName))
+         {
+            QProcessMetaData process = qInstance.getProcess(processName);
+            if(process == null)
+            {
+               LOG.info("Unrecognized process in help content", logPair("key", key));
+               return;
+            }
+
+            if(StringUtils.hasContent(fieldName))
+            {
+               ////////////////////////////
+               // handle a process field //
+               ////////////////////////////
+               Optional<QFieldMetaData> optionalField = CollectionUtils.mergeLists(process.getInputFields(), process.getOutputFields())
+                  .stream().filter(f -> fieldName.equals(f.getName()))
+                  .findFirst();
+
+               if(optionalField.isEmpty())
+               {
+                  LOG.info("Unrecognized process field in help content", logPair("key", key));
+                  return;
+               }
+
+               if(helpContent != null)
+               {
+                  optionalField.get().withHelpContent(helpContent);
+               }
+               else
+               {
+                  optionalField.get().removeHelpContent(roles);
+               }
+            }
+         }
+      }
+      catch(Exception e)
+      {
+         LOG.warn("Error processing a helpContent record", e, logPair("id", record.getValue("id")));
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** add a help content object to a list - replacing an entry in the list with the
+    ** same roles if one is found.
+    *******************************************************************************/
+   public static void putHelpContentInList(QHelpContent helpContent, List<QHelpContent> helpContents)
+   {
+      ListIterator<QHelpContent> iterator = helpContents.listIterator();
+      while(iterator.hasNext())
+      {
+         QHelpContent existingContent = iterator.next();
+         if(Objects.equals(existingContent.getRoles(), helpContent.getRoles()))
+         {
+            iterator.set(helpContent);
+            return;
+         }
+      }
+
+      helpContents.add(helpContent);
+   }
+
+
+
+   /*******************************************************************************
+    ** Remove any helpContent entries in a list if they have a set of roles that matches
+    ** the input set.
+    *******************************************************************************/
+   public static void removeHelpContentByRoleSetFromList(Set<HelpRole> roles, List<QHelpContent> helpContents)
+   {
+      if(helpContents == null)
+      {
+         return;
+      }
+
+      helpContents.removeIf(existingContent -> Objects.equals(existingContent.getRoles(), roles));
+   }
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidator.java

@@ -22,6 +22,7 @@
 package com.kingsrook.qqq.backend.core.instances;
 
 
+import java.io.Serializable;
 import java.lang.reflect.InvocationTargetException;
 import java.lang.reflect.Modifier;
 import java.util.ArrayList;
@@ -31,11 +32,11 @@ import java.util.List;
 import java.util.Map;
 import java.util.Objects;
 import java.util.Set;
-import java.util.function.Consumer;
 import java.util.function.Supplier;
 import java.util.stream.Stream;
 import com.kingsrook.qqq.backend.core.actions.automation.RecordAutomationHandler;
 import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizers;
+import com.kingsrook.qqq.backend.core.actions.metadata.JoinGraph;
 import com.kingsrook.qqq.backend.core.actions.processes.BackendStep;
 import com.kingsrook.qqq.backend.core.actions.scripts.TestScriptActionInterface;
 import com.kingsrook.qqq.backend.core.actions.values.QCustomPossibleValueProvider;
@@ -47,10 +48,11 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryJoin;
 import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
-import com.kingsrook.qqq.backend.core.model.metadata.QMiddlewareInstanceMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.QSupplementalInstanceMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
 import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeType;
-import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeUsage;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.AdornmentType;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.FieldAdornment;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.ValueTooLongBehavior;
 import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinOn;
@@ -61,14 +63,17 @@ import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppSection;
 import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.processes.QStepMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.processes.QSupplementalProcessMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.queues.SQSQueueProviderMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportDataSource;
 import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportField;
 import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportView;
+import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.QScheduleMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.security.FieldSecurityLock;
 import com.kingsrook.qqq.backend.core.model.metadata.security.RecordSecurityLock;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.AssociatedScript;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.Association;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.ExposedJoin;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QFieldSection;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.Tier;
@@ -80,6 +85,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.tables.cache.CacheOf;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.cache.CacheUseCase;
 import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
 import com.kingsrook.qqq.backend.core.utils.StringUtils;
+import com.kingsrook.qqq.backend.core.utils.ValueUtils;
 
 
 /*******************************************************************************
@@ -115,17 +121,26 @@ public class QInstanceValidator
          return;
       }
 
+      /////////////////////////////////////////////////////////////////////////////////////////////////////
+      // the enricher will build a join graph (if there are any joins).  we'd like to only do that       //
+      // once, during the enrichment/validation work, so, capture it, and store it back in the instance. //
+      /////////////////////////////////////////////////////////////////////////////////////////////////////
+      JoinGraph joinGraph = null;
       try
       {
          /////////////////////////////////////////////////////////////////////////////////////////////////
          // before validation, enrich the object (e.g., to fill in values that the user doesn't have to //
          /////////////////////////////////////////////////////////////////////////////////////////////////
          // TODO - possible point of customization (use a different enricher, or none, or pass it options).
-         new QInstanceEnricher(qInstance).enrich();
+         QInstanceEnricher qInstanceEnricher = new QInstanceEnricher(qInstance);
+         qInstanceEnricher.enrich();
+         joinGraph = qInstanceEnricher.getJoinGraph();
       }
       catch(Exception e)
       {
+         System.out.println();
          LOG.error("Error enriching instance prior to validation", e);
+         System.out.println();
          throw (new QInstanceValidationException("Error enriching qInstance prior to validation.", e));
       }
 
@@ -136,7 +151,7 @@ public class QInstanceValidator
       {
          validateBackends(qInstance);
          validateAutomationProviders(qInstance);
-         validateTables(qInstance);
+         validateTables(qInstance, joinGraph);
          validateProcesses(qInstance);
          validateReports(qInstance);
          validateApps(qInstance);
@@ -144,7 +159,7 @@ public class QInstanceValidator
          validateQueuesAndProviders(qInstance);
          validateJoins(qInstance);
          validateSecurityKeyTypes(qInstance);
-         validateMiddlewareMetaData(qInstance);
+         validateSupplementalMetaData(qInstance);
 
          validateUniqueTopLevelNames(qInstance);
       }
@@ -158,7 +173,9 @@ public class QInstanceValidator
          throw (new QInstanceValidationException(errors));
       }
 
-      qInstance.setHasBeenValidated(new QInstanceValidationKey());
+      QInstanceValidationKey validationKey = new QInstanceValidationKey();
+      qInstance.setHasBeenValidated(validationKey);
+      qInstance.setJoinGraph(validationKey, joinGraph);
    }
 
 
@@ -166,11 +183,11 @@ public class QInstanceValidator
    /*******************************************************************************
     **
     *******************************************************************************/
-   private void validateMiddlewareMetaData(QInstance qInstance)
+   private void validateSupplementalMetaData(QInstance qInstance)
    {
-      for(QMiddlewareInstanceMetaData middlewareInstanceMetaData : CollectionUtils.nonNullMap(qInstance.getMiddlewareMetaData()).values())
+      for(QSupplementalInstanceMetaData supplementalInstanceMetaData : CollectionUtils.nonNullMap(qInstance.getSupplementalMetaData()).values())
       {
-         middlewareInstanceMetaData.validate(qInstance, this);
+         supplementalInstanceMetaData.validate(qInstance, this);
       }
    }
 
@@ -366,7 +383,7 @@ public class QInstanceValidator
    /*******************************************************************************
     **
     *******************************************************************************/
-   private void validateTables(QInstance qInstance)
+   private void validateTables(QInstance qInstance, JoinGraph joinGraph)
    {
       if(assertCondition(CollectionUtils.nullSafeHasContents(qInstance.getTables()), "At least 1 table must be defined."))
       {
@@ -405,7 +422,7 @@ public class QInstanceValidator
             {
                table.getFields().forEach((fieldName, field) ->
                {
-                  validateTableField(qInstance, tableName, fieldName, field);
+                  validateTableField(qInstance, tableName, fieldName, table, field);
                });
             }
 
@@ -421,10 +438,13 @@ public class QInstanceValidator
                for(QFieldSection section : table.getSections())
                {
                   validateTableSection(qInstance, table, section, fieldNamesInSections);
-                  if(section.getTier().equals(Tier.T1))
+                  if(assertCondition(section.getTier() != null, "Table " + tableName + " " + section.getName() + " is missing its tier"))
                   {
-                     assertCondition(tier1Section == null, "Table " + tableName + " has more than 1 section listed as Tier 1");
-                     tier1Section = section;
+                     if(section.getTier().equals(Tier.T1))
+                     {
+                        assertCondition(tier1Section == null, "Table " + tableName + " has more than 1 section listed as Tier 1");
+                        tier1Section = section;
+                     }
                   }
 
                   assertCondition(!usedSectionNames.contains(section.getName()), "Table " + tableName + " has more than 1 section named " + section.getName());
@@ -437,7 +457,14 @@ public class QInstanceValidator
 
             for(String fieldName : CollectionUtils.nonNullMap(table.getFields()).keySet())
             {
-               assertCondition(fieldNamesInSections.contains(fieldName), "Table " + tableName + " field " + fieldName + " is not listed in any field sections.");
+               if(table.getField(fieldName).getIsHidden())
+               {
+                  assertCondition(!fieldNamesInSections.contains(fieldName), "Table " + tableName + " field " + fieldName + " is listed in a field section, but it is marked as hidden.");
+               }
+               else
+               {
+                  assertCondition(fieldNamesInSections.contains(fieldName), "Table " + tableName + " field " + fieldName + " is not listed in any field sections.");
+               }
             }
 
             if(table.getRecordLabelFields() != null && table.getFields() != null)
@@ -459,12 +486,61 @@ public class QInstanceValidator
             validateTableCacheOf(qInstance, table);
             validateTableRecordSecurityLocks(qInstance, table);
             validateTableAssociations(qInstance, table);
+            validateExposedJoins(qInstance, joinGraph, table);
          });
       }
    }
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private void validateExposedJoins(QInstance qInstance, JoinGraph joinGraph, QTableMetaData table)
+   {
+      Set<JoinGraph.JoinConnectionList> joinConnectionsForTable = null;
+      Set<String>                       usedLabels              = new HashSet<>();
+      Set<List<String>>                 usedJoinPaths           = new HashSet<>();
+
+      String tablePrefix = "Table " + table.getName() + " ";
+      for(ExposedJoin exposedJoin : CollectionUtils.nonNullList(table.getExposedJoins()))
+      {
+         String joinPrefix = tablePrefix + "exposedJoin [missingJoinTableName] ";
+         if(assertCondition(StringUtils.hasContent(exposedJoin.getJoinTable()), tablePrefix + "has an exposedJoin that is missing a joinTable name."))
+         {
+            joinPrefix = tablePrefix + "exposedJoin " + exposedJoin.getJoinTable() + " ";
+            if(assertCondition(qInstance.getTable(exposedJoin.getJoinTable()) != null, joinPrefix + "is referencing an unrecognized table"))
+            {
+               if(assertCondition(CollectionUtils.nullSafeHasContents(exposedJoin.getJoinPath()), joinPrefix + "is missing a joinPath."))
+               {
+                  joinConnectionsForTable = Objects.requireNonNullElseGet(joinConnectionsForTable, () -> joinGraph.getJoinConnections(table.getName()));
+
+                  boolean foundJoinConnection = false;
+                  for(JoinGraph.JoinConnectionList joinConnectionList : joinConnectionsForTable)
+                  {
+                     if(joinConnectionList.matchesJoinPath(exposedJoin.getJoinPath()))
+                     {
+                        foundJoinConnection = true;
+                     }
+                  }
+                  assertCondition(foundJoinConnection, joinPrefix + "specified a joinPath [" + exposedJoin.getJoinPath() + "] which does not match a valid join connection in the instance.");
+
+                  assertCondition(!usedJoinPaths.contains(exposedJoin.getJoinPath()), tablePrefix + "has more than one join with the joinPath: " + exposedJoin.getJoinPath());
+                  usedJoinPaths.add(exposedJoin.getJoinPath());
+               }
+            }
+         }
+
+         if(assertCondition(StringUtils.hasContent(exposedJoin.getLabel()), joinPrefix + "is missing a label."))
+         {
+            assertCondition(!usedLabels.contains(exposedJoin.getLabel()), tablePrefix + "has more than one join labeled: " + exposedJoin.getLabel());
+            usedLabels.add(exposedJoin.getLabel());
+         }
+      }
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/
@@ -500,6 +576,11 @@ public class QInstanceValidator
       RECORD_SECURITY_LOCKS_LOOP:
       for(RecordSecurityLock recordSecurityLock : CollectionUtils.nonNullList(table.getRecordSecurityLocks()))
       {
+         if(!assertCondition(recordSecurityLock != null, prefix + "has a null recordSecurityLock (did you mean to give it a null list of locks?)"))
+         {
+            continue;
+         }
+
          String securityKeyTypeName = recordSecurityLock.getSecurityKeyType();
          if(assertCondition(StringUtils.hasContent(securityKeyTypeName), prefix + "has a recordSecurityLock that is missing a securityKeyType"))
          {
@@ -508,6 +589,8 @@ public class QInstanceValidator
 
          prefix = "Table " + table.getName() + " recordSecurityLock (of key type " + securityKeyTypeName + ") ";
 
+         assertCondition(recordSecurityLock.getLockScope() != null, prefix + " is missing its lockScope");
+
          boolean hasAnyBadJoins = false;
          for(String joinName : CollectionUtils.nonNullList(recordSecurityLock.getJoinNameChain()))
          {
@@ -590,7 +673,7 @@ public class QInstanceValidator
    /*******************************************************************************
     **
     *******************************************************************************/
-   private void validateTableField(QInstance qInstance, String tableName, String fieldName, QFieldMetaData field)
+   private void validateTableField(QInstance qInstance, String tableName, String fieldName, QTableMetaData table, QFieldMetaData field)
    {
       assertCondition(Objects.equals(fieldName, field.getName()),
          "Inconsistent naming in table " + tableName + " for field " + fieldName + "/" + field.getName() + ".");
@@ -632,6 +715,55 @@ public class QInstanceValidator
          assertCondition(fieldSecurityLock.getDefaultBehavior() != null, prefix + "has a fieldSecurityLock that is missing a defaultBehavior");
          assertCondition(CollectionUtils.nullSafeHasContents(fieldSecurityLock.getOverrideValues()), prefix + "has a fieldSecurityLock that is missing overrideValues");
       }
+
+      for(FieldAdornment adornment : CollectionUtils.nonNullList(field.getAdornments()))
+      {
+         Map<String, Serializable> adornmentValues = CollectionUtils.nonNullMap(adornment.getValues());
+         if(assertCondition(adornment.getType() != null, prefix + "has an adornment that is missing a type"))
+         {
+            String adornmentPrefix = prefix.trim() + ", " + adornment.getType() + " adornment ";
+            switch(adornment.getType())
+            {
+               case SIZE ->
+               {
+                  String width = ValueUtils.getValueAsString(adornmentValues.get("width"));
+                  if(assertCondition(StringUtils.hasContent(width), adornmentPrefix + "is missing a width value"))
+                  {
+                     assertNoException(() -> AdornmentType.Size.valueOf(width.toUpperCase()), adornmentPrefix + "has an unrecognized width value [" + width + "]");
+                  }
+               }
+               case FILE_DOWNLOAD ->
+               {
+                  String fileNameField = ValueUtils.getValueAsString(adornmentValues.get(AdornmentType.FileDownloadValues.FILE_NAME_FIELD));
+                  if(StringUtils.hasContent(fileNameField)) // file name isn't required - but if given, must be a field on the table.
+                  {
+                     assertNoException(() -> table.getField(fileNameField), adornmentPrefix + "specifies an unrecognized fileNameField [" + fileNameField + "]");
+                  }
+
+                  if(adornmentValues.containsKey(AdornmentType.FileDownloadValues.FILE_NAME_FORMAT_FIELDS))
+                  {
+                     try
+                     {
+                        @SuppressWarnings("unchecked")
+                        List<String> formatFieldNames = (List<String>) adornmentValues.get(AdornmentType.FileDownloadValues.FILE_NAME_FORMAT_FIELDS);
+                        for(String formatFieldName : CollectionUtils.nonNullList(formatFieldNames))
+                        {
+                           assertNoException(() -> table.getField(formatFieldName), adornmentPrefix + "specifies an unrecognized field name in fileNameFormatFields [" + formatFieldName + "]");
+                        }
+                     }
+                     catch(Exception e)
+                     {
+                        errors.add(adornmentPrefix + "fileNameFormatFields could not be accessed (is it a List<String>?)");
+                     }
+                  }
+               }
+               default ->
+               {
+                  // no validations by default
+               }
+            }
+         }
+      }
    }
 
 
@@ -840,9 +972,9 @@ public class QInstanceValidator
    /*******************************************************************************
     **
     *******************************************************************************/
-   private void validateTableCustomizer(String tableName, String customizerName, QCodeReference codeReference)
+   private void validateTableCustomizer(String tableName, String roleName, QCodeReference codeReference)
    {
-      String prefix = "Table " + tableName + ", customizer " + customizerName + ": ";
+      String prefix = "Table " + tableName + ", customizer " + roleName + ": ";
 
       if(!preAssertionsForCodeReference(codeReference, prefix))
       {
@@ -865,7 +997,7 @@ public class QInstanceValidator
             //////////////////////////////////////////////////
             Object customizerInstance = getInstanceOfCodeReference(prefix, customizerClass);
 
-            TableCustomizers tableCustomizer = TableCustomizers.forRole(customizerName);
+            TableCustomizers tableCustomizer = TableCustomizers.forRole(roleName);
             if(tableCustomizer == null)
             {
                ////////////////////////////////////////////////////////////////////////////////////////////////////
@@ -878,29 +1010,9 @@ public class QInstanceValidator
                ////////////////////////////////////////////////////////////////////////
                // make sure the customizer instance can be cast to the expected type //
                ////////////////////////////////////////////////////////////////////////
-               if(customizerInstance != null && tableCustomizer.getTableCustomizer().getExpectedType() != null)
+               if(customizerInstance != null && tableCustomizer.getExpectedType() != null)
                {
-                  Object castedObject = getCastedObject(prefix, tableCustomizer.getTableCustomizer().getExpectedType(), customizerInstance);
-
-                  Consumer<Object> validationFunction = tableCustomizer.getTableCustomizer().getValidationFunction();
-                  if(castedObject != null && validationFunction != null)
-                  {
-                     try
-                     {
-                        validationFunction.accept(castedObject);
-                     }
-                     catch(ClassCastException e)
-                     {
-                        errors.add(prefix + "Error validating customizer type parameters: " + e.getMessage());
-                     }
-                     catch(Exception e)
-                     {
-                        ///////////////////////////////////////////////////////////////////////////////////////////////////////////
-                        // mmm, calling customizers w/ random data is expected to often throw, so, this check is iffy at best... //
-                        // if we run into more trouble here, we might consider disabling the whole "validation function" check.  //
-                        ///////////////////////////////////////////////////////////////////////////////////////////////////////////
-                     }
-                  }
+                  assertObjectCanBeCasted(prefix, tableCustomizer.getExpectedType(), customizerInstance);
                }
             }
          }
@@ -910,18 +1022,18 @@ public class QInstanceValidator
 
 
    /*******************************************************************************
-    **
+    ** Make sure that a given object can be casted to an expected type.
     *******************************************************************************/
-   private <T> T getCastedObject(String prefix, Class<T> expectedType, Object customizerInstance)
+   private <T> T assertObjectCanBeCasted(String errorPrefix, Class<T> expectedType, Object object)
    {
       T castedObject = null;
       try
       {
-         castedObject = expectedType.cast(customizerInstance);
+         castedObject = expectedType.cast(object);
       }
       catch(ClassCastException e)
       {
-         errors.add(prefix + "CodeReference is not of the expected type: " + expectedType);
+         errors.add(errorPrefix + "CodeReference is not of the expected type: " + expectedType);
       }
       return castedObject;
    }
@@ -992,13 +1104,39 @@ public class QInstanceValidator
       boolean hasFields = CollectionUtils.nullSafeHasContents(section.getFieldNames());
       boolean hasWidget = StringUtils.hasContent(section.getWidgetName());
 
-      if(assertCondition(hasFields || hasWidget, "Table " + table.getName() + " section " + section.getName() + " does not have any fields or a widget."))
+      String sectionPrefix = "Table " + table.getName() + " section " + section.getName() + " ";
+      if(assertCondition(hasFields || hasWidget, sectionPrefix + "does not have any fields or a widget."))
       {
          if(table.getFields() != null && hasFields)
          {
             for(String fieldName : section.getFieldNames())
             {
-               assertCondition(table.getFields().containsKey(fieldName), "Table " + table.getName() + " section " + section.getName() + " specifies fieldName " + fieldName + ", which is not a field on this table.");
+               ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+               // note - this was originally written as an assertion:                                                                                                                                       //
+               // if(assertCondition(qInstance.getTable(otherTableName) != null, sectionPrefix + "join-field " + fieldName + ", which is referencing an unrecognized table name [" + otherTableName + "]")) //
+               // but... then a field name with dots gives us a bad time here, so...                                                                                                                        //
+               ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+               if(fieldName.contains(".") && qInstance.getTable(fieldName.split("\\.")[0]) != null)
+               {
+                  String[] parts            = fieldName.split("\\.");
+                  String   otherTableName   = parts[0];
+                  String   foreignFieldName = parts[1];
+
+                  if(assertCondition(qInstance.getTable(otherTableName) != null, sectionPrefix + "join-field " + fieldName + ", which is referencing an unrecognized table name [" + otherTableName + "]"))
+                  {
+                     List<ExposedJoin> matchedExposedJoins = CollectionUtils.nonNullList(table.getExposedJoins()).stream().filter(ej -> otherTableName.equals(ej.getJoinTable())).toList();
+                     if(assertCondition(CollectionUtils.nullSafeHasContents(matchedExposedJoins), sectionPrefix + "join-field " + fieldName + ", referencing table [" + otherTableName + "] which is not an exposed join on this table."))
+                     {
+                        assertCondition(!matchedExposedJoins.get(0).getIsMany(qInstance), sectionPrefix + "join-field " + fieldName + " references an is-many join, which is not supported.");
+                     }
+                     assertCondition(qInstance.getTable(otherTableName).getFields().containsKey(foreignFieldName), sectionPrefix + "join-field " + fieldName + " specifies a fieldName [" + foreignFieldName + "] which does not exist in that table [" + otherTableName + "].");
+                  }
+               }
+               else
+               {
+                  assertCondition(table.getFields().containsKey(fieldName), sectionPrefix + "specifies fieldName " + fieldName + ", which is not a field on this table.");
+               }
+
                assertCondition(!fieldNamesInSections.contains(fieldName), "Table " + table.getName() + " has field " + fieldName + " listed more than once in its field sections.");
 
                fieldNamesInSections.add(fieldName);
@@ -1006,7 +1144,7 @@ public class QInstanceValidator
          }
          else if(hasWidget)
          {
-            assertCondition(qInstance.getWidget(section.getWidgetName()) != null, "Table " + table.getName() + " section " + section.getName() + " specifies widget " + section.getWidgetName() + ", which is not a widget in this instance.");
+            assertCondition(qInstance.getWidget(section.getWidgetName()) != null, sectionPrefix + "specifies widget " + section.getWidgetName() + ", which is not a widget in this instance.");
          }
       }
    }
@@ -1108,6 +1246,27 @@ public class QInstanceValidator
                   }
                }
             }
+
+            ///////////////////////////////////////////////////////////////////////////////
+            // if the process has a schedule, make sure required schedule data populated //
+            ///////////////////////////////////////////////////////////////////////////////
+            if(process.getSchedule() != null)
+            {
+               QScheduleMetaData schedule = process.getSchedule();
+               assertCondition(schedule.getRepeatMillis() != null || schedule.getRepeatSeconds() != null, "Either repeat millis or repeat seconds must be set on schedule in process " + processName);
+
+               if(schedule.getVariantBackend() != null)
+               {
+                  assertCondition(qInstance.getBackend(schedule.getVariantBackend()) != null, "A variant backend was not found for " + schedule.getVariantBackend());
+                  assertCondition(schedule.getVariantRunStrategy() != null, "A variant run strategy was not set for " + schedule.getVariantBackend() + " on schedule in process " + processName);
+               }
+            }
+
+            for(QSupplementalProcessMetaData supplementalProcessMetaData : CollectionUtils.nonNullMap(process.getSupplementalMetaData()).values())
+            {
+               supplementalProcessMetaData.validate(qInstance, process, this);
+            }
+
          });
       }
    }
@@ -1435,7 +1594,6 @@ public class QInstanceValidator
 
                      if(assertCondition(possibleValueSource.getCustomCodeReference() != null, "custom-type possibleValueSource " + pvsName + " is missing a customCodeReference."))
                      {
-                        assertCondition(QCodeUsage.POSSIBLE_VALUE_PROVIDER.equals(possibleValueSource.getCustomCodeReference().getCodeUsage()), "customCodeReference for possibleValueSource " + pvsName + " is not a possibleValueProvider.");
                         validateSimpleCodeReference("PossibleValueSource " + pvsName + " custom code reference: ", possibleValueSource.getCustomCodeReference(), QCustomPossibleValueProvider.class);
                      }
                   }
@@ -1479,7 +1637,7 @@ public class QInstanceValidator
             ////////////////////////////////////////////////////////////////////////
             if(classInstance != null)
             {
-               getCastedObject(prefix, expectedClass, classInstance);
+               assertObjectCanBeCasted(prefix, expectedClass, classInstance);
             }
          }
       }
@@ -1586,7 +1744,7 @@ public class QInstanceValidator
     ** But if it throws, add the provided message to the list of errors (and return false,
     ** e.g., in case you need to stop evaluating rules to avoid exceptions).
     *******************************************************************************/
-   private boolean assertNoException(UnsafeLambda unsafeLambda, String message)
+   public boolean assertNoException(UnsafeLambda unsafeLambda, String message)
    {
       try
       {
@@ -1619,7 +1777,7 @@ public class QInstanceValidator
    /*******************************************************************************
     **
     *******************************************************************************/
-   private void warn(String message)
+   public void warn(String message)
    {
       if(printWarnings)
       {