Merged dev into integration/sprint-35

.circleci/config.yml

@@ -1,7 +1,7 @@
 version: 2.1
 
 orbs:
-  localstack: localstack/platform@2.1
+  localstack: localstack/platform@1.0
 
 commands:
   store_jacoco_site:
@@ -98,31 +98,6 @@ commands:
             - ~/.m2
           key: v1-dependencies-{{ checksum "pom.xml" }}
 
-  install_asciidoctor:
-    steps:
-      - checkout
-      - run:
-          name: Install asciidoctor
-          command: |
-            sudo apt-get update
-            sudo apt install -y asciidoctor
-
-  run_asciidoctor:
-    steps:
-      - run:
-          name: Run asciidoctor
-          command: |
-            cd docs
-            asciidoctor -a docinfo=shared index.adoc
-
-  upload_docs_site:
-    steps:
-      - run:
-          name: scp html to justinsgotskinnylegs.com
-          command: |
-            cd docs
-            scp index.html dkelkhoff@45.79.44.221:/mnt/first-volume/dkelkhoff/nginx/html/justinsgotskinnylegs.com/qqq-docs.html
-
 jobs:
   mvn_test:
     executor: localstack/default
@@ -139,13 +114,6 @@ jobs:
       - mvn_verify
       - mvn_jar_deploy
 
-  publish_asciidoc:
-    executor: localstack/default
-    steps:
-      - install_asciidoctor
-      - run_asciidoctor
-      - upload_docs_site
-
 workflows:
   test_only:
     jobs:
@@ -153,7 +121,7 @@ workflows:
           context: [ qqq-maven-registry-credentials, build-qqq-sample-app ]
           filters:
             branches:
-              ignore: /(dev|integration.*)/
+              ignore: /dev/
             tags:
               ignore: /(version|snapshot)-.*/
 
@@ -163,10 +131,7 @@ workflows:
           context: [ qqq-maven-registry-credentials, build-qqq-sample-app ]
           filters:
             branches:
-              only: /(dev|integration.*)/
+              only: /dev/
             tags:
               only: /(version|snapshot)-.*/
-      - publish_asciidoc:
-          filters:
-            branches:
-              only: /dev/
+

docs/.gitignore

@@ -1 +0,0 @@
-*.html

docs/actions/GetAction.adoc

@@ -1,64 +0,0 @@
-== GetAction
-include::../variables.adoc[]
-
-The `*GetAction*` is essentially a subset of the <<QueryAction>>, only specifically meant to get just a single record from a {link-table}.
-In SQL/RDBMS terms, it is analogous to a `SELECT` statement, where a single record may be found - or - it may not be found.
-
-For all tables, `GetAction` can do a lookup by primary key.
-In addition, for tables that define a `UniqueKey`, name/value pairs (in the form of a `Map`) can be used as input for `GetAction`.
-
-=== Examples
-[source,java]
-.Basic form - by primary key
-----
-GetInput input = new GetInput();
-input.setTableName("orders");
-input.setPrimaryKey(1);
-GetOutput output = new GetAction.execute(input);
-QRecord record = output.getRecord();
-----
-
-[source,java]
-.Secondary form - by Unique Key
-----
-GetInput input = new GetInput();
-input.setTableName("products");
-input.setUniqueKey(Map.of("storeId", 1701, "sku", "ABCD"));
-GetOutput output = new GetAction.execute(input);
-QRecord record = output.getRecord();
-----
-
-=== GetInput
-* `table` - *String, Required* - Name of the table being queried against.
-* `primaryKey` - *Serializable, Conditional* - Value for the primary key field of the table being queried.
-Type should match the table's primary key field's type.
-If a `primaryKey` is not given, then a `uniqueKey` must be given.
-* `uniqueKey` - *Map of String -> Serializable, Conditional* - Map of name-value pairs that define the record to be fetcheed.
-Keys in the map must be field names from the table being queried.
-Values in the map should should be of types that correspond to the fields.
-If a `primaryKey` is not given, then a `uniqueKey` must be given.
-If both `primaryKey` and `uniqueKey` are given, then `uniqueKey` is ignored.
-* `transaction` - *QBackendTransaction object* - Optional transaction object.
-** Behavior for this object is backend-dependant.
-In an RDBMS backend, this object is generally needed if you want your query to see data that may have been modified within the same transaction.
-
-* `shouldTranslatePossibleValues` - *boolean, default: false* - Controls whether any fields in the table with a *possibleValueSource* assigned to them should have those possible values looked up
-(e.g., to provide text translations in the generated records' `displayValues` map).
-** For example, if getting a record to present to a user, this would generally need to be *true*.
-But if getting a record as part of a process, then this can generally be left as *false*.
-* `shouldGenerateDisplayValues` - *boolean, default: false* - Controls whether field level *displayFormats* should be used to populate the generated records' `displayValues` map.
-** For example, if getting a record to present to a user, this would generally need to be *true*.
-But if getting a record as part of a process, then this can generally be left as *false*.
-* `shouldFetchHeavyFields` - *boolean, default: true* - Controls whether or not fields marked as `isHeavy` should be fetched & returned or not.
-* `shouldOmitHiddenFields` - *boolean, default: true* - Controls whether or not fields marked as `isHidden` should be included in the result or not.
-* `shouldMaskPassword` - *boolean, default: true* - Controls whether or not fields with `type` = `PASSWORD` should be masked, or if their actual values should be returned.
-* `queryJoins` - *List of <<QueryJoin>> objects* - Optional list of tables to be joined with the main table being queried.
-See QueryJoin under <<QueryAction>> for further details.
-* `includeAssociations` - *boolean, default: false* - Control whether or not records from tables defined as `associations` under the table being queried should be included in the result or not.
-* `associationNamesToInclude` - *Collection of String* - If `includeAssociations` is true, then this field can be used to limit which associated tables are included.
-If this field is null, then all associated tables are included.
-Otherwise, a table is only included if its name is in this collection.
-
-=== GetOutput
-* `record` - *QRecord* - The record that was specified by the input `primaryKey` or `uniqueKey`.
-Will be null if the record was not found.

docs/actions/InsertAction.adoc

@@ -1,86 +0,0 @@
-== InsertAction
-include::../variables.adoc[]
-
-To insert (add, create) new records into any {link-table}, the `*InsertAction*` is used.
-In SQL/RDBMS terms, it is analogous to a `INSERT` statement, where one or more records can be provided as input.
-
-=== Examples
-[source,java]
-.Canonical InsertAction invocation
-----
-InsertInput insertInput = new InsertInput();
-insertInput.setTableName("person");
-insertInput.setRecords(personRecordList);
-InsertOutput insertOutput = new InsertAction().execute(insertInput);
-List<QRecord> insertedPersonRecords = insertOutput.getRecords();
-----
-
-=== Details
-`InsertAction` does several things beyond just inserting records into the specified table.
-A high-level flow of its internal logic is:
-
-. For tables using an automation status field, set its value to `PENDING_INSERT_AUTOMATIONS` for all `records` that are going to be inserted.
-. Perform the following validations, which include running the table's `PRE_INSERT_CUSTOMIZER`, if one is defined, at the time that is specified by the customizer's `WhenToRun` property (default is `AFTER_ALL_VALIDATIONS`):
-.. Ensure that default values specified in the table's fields are present if needed.
-.. Apply per-field behaviors, as defined in {link-field} meta-data, such as truncating strings that are longer than their specified max.
-.. Check for unique key violations (done here instead of in the backend, to provide better error messaging, and to allow a subset of records to be stored while some fail).
-_We might want to make an input control in the future to specify that either the full input set should succeed or fail..._
-.. Validate that required fields (again, per {link-field} meta-data) are set, generating per-record errors if they are not.
-.. Validate any security fields in the records - e.g., ensure that the user has permission to insert records with the values they are attempting to insert.
-. Send the records to the table's backend module to actually insert them into the backend storage.
-. If the table has any associations defined, and if associated records are present, then recursively run `InsertAction` on the associated records.
-.. In particular, before these recursive `InsertAction` calls are made, values that were generated by the original insert may need to be propagated down into the associated records.
-*** For example, if inserting `order` and `lineItem` records, where a {link-join} exists between the two tables on `order.id` and `lineItem.orderId`, and `order.id` values were generated in the first `InsertAction`, then those values are propagated down into the associated `lineItem.orderId` fields.
-. If the {link-instance} has an `audit` table, then based on the {link-table}'s audit rules, audits about the inserted records are created.
-. If the table has a `POST_INSERT_CUSTOMIZER`, it is executed.
-
-=== Overloads
-
-`InsertAction` can be called in a few alternate forms, mostly just for convenience:
-
-[source,java]
-.If inserting a single record, get that record back instead of the InsertOutput:
-----
-InsertInput insertInput = new InsertInput();
-insertInput.setTableName("person");
-insertInput.setRecords(List.of(personRecord));
-QRecord insertedRecord = new InsertAction().executeForRecord(insertInput);
-
-// or more compactly, using InsertInput.withRecord (instead of withRecords)
-QRecord insertedRecord = new InsertAction()
-   .executeForRecord(new InsertInput("person").withRecord(personRecord));
-----
-
-[source,java]
-.Taking QRecordEntity objects as inputs instead of QRecords:
-----
-// insert a list of person entities:
-InsertInput insertInput = new InsertInput("person").withRecordEntities(personList);
-InsertOutput insertOutput = new InsertAction().execute(insertInput);
-
-// or for a single person entity (also mapping the output record back to an entity):
-Person insertedPerson = new Person(new InsertAction()
-   .executeForRecord(new InsertInput("person").withRecordEntity(person)));
-
-----
-
-=== InsertInput
-* `table` - *String, Required* - Name of the table that records are to be inserted into.
-* `records` - *List of QRecord, Required* - List of records to be inserted into the table.
-If the list is empty, the insert action does a `noop`.
-
-.Less common options
-* `inputSource` - *InputSource object, default: QInputSource.SYSTEM* - an indicator of what the source of the action is - generally, a `SYSTEM` triggered action, or a `USER` triggered action.
-** `InsertAction` will call the `shouldValidateRequiredFields()` method on this object to determine if it should validate that required fields on the records have values.
-Both `QInputSource.SYSTEM` and `QInputSource.USER` return `true` from this method, but application can define their own `InputSource` objects with different behavior.
-** In addition, this field can be used in pre- and post-insert customizers to drive further custom logic.
-* `skipUniqueKeyCheck` - *boolean, default: false* - control whether or not `InsertAction` should check for unique key violations before attempting to insert its records.
-** In a context where one has previously done this validation, or is okay letting the backend provide such checks, they may wish to avoid re-doing this work, and thus may set this property to `true`.
-* `omitDmlAudit` - *boolean, default: false* - control if the automatic DML audit that `InsertAction` generally performs should be omitted.
-* `auditContext` - *String* - optional message which can be included in DML audit messages, to give users more context about why the insert occurred.
-
-
-=== InsertOutput
-* `records` - *List of QRecord* - Copy of the input list of records, with details added based on the results of the input action.
-** If there were warnings or errors, the corresponding field (`warnings` or `errors`) will be set in the records.
-** If the insert action generated any values (such as a serial id or a default value), those values will be in the record's `fields` map.

docs/index.adoc

@@ -26,22 +26,12 @@ include::metaData/Reports.adoc[leveloffset=+1]
 include::metaData/Icons.adoc[leveloffset=+1]
 include::metaData/PermissionRules.adoc[leveloffset=+1]
 
-== Services
-
-include::misc/ScheduledJobs.adoc[leveloffset=+1]
-
-=== Web server (Javalin)
-#todo#
-
-=== API server (OpenAPI)
-#todo#
 
 == Custom Application Code
 include::misc/QContext.adoc[leveloffset=+1]
 include::misc/QRecords.adoc[leveloffset=+1]
 include::misc/QRecordEntities.adoc[leveloffset=+1]
 include::misc/ProcessBackendSteps.adoc[leveloffset=+1]
-include::misc/RenderingWidgets.adoc[leveloffset=+1]
 
 === Table Customizers
 #todo#
@@ -71,6 +61,3 @@ include::actions/InsertAction.adoc[leveloffset=+1]
 == QQQ Default Implementations
 include::implementations/TableSync.adoc[leveloffset=+1]
 // later... include::actions/RenderTemplateAction.adoc[leveloffset=+1]
-
-== QQQ Utility Classes
-include::utilities/RecordLookupHelper.adoc[leveloffset=+1]

docs/justfile

@@ -1,12 +0,0 @@
-## https://github.com/casey/just
-
-default:
-   just --list
-
-build-index-html:
-   asciidoctor -a docinfo=shared index.adoc
-   line=$(grep 'Last updated' index.html) && sed -i "s/id=\"content\">/&$line/" index.html
-
-build-and-publish-index-html: build-index-html
-   scp index.html first-node:/mnt/first-volume/dkelkhoff/nginx/html/justinsgotskinnylegs.com/qqq-docs.html
-   @echo "Updated: https://justinsgotskinnylegs.com/qqq-docs.html"

docs/metaData/Fields.adoc

@@ -21,99 +21,5 @@ Used to set values in the `displayValues` map within a `QRecord`.
 * `possibleValueSourceName` - *String* - Reference to a {link-pvs} to be used for this field.
 Values in this field should correspond to ids from the referenced Possible Value Source.
 * `maxLength` - *Integer* - Maximum length (number of characters) allowed for values in this field.
-Only applicable for fields with `type=STRING`.  Needs to be used with a `FieldBehavior` of type `ValueTooLongBehavior`.
-
-==== Field Behaviors
-Additional behaviors can be attached to fields through the use of the `behaviors` attribute,
-which is a `Set` of 0 or more instances of implementations of the `FieldBehavior` interface.
-Note that in some cases, these instances may be `enum` constants,
-but other times may be regular Objects.
-
-QQQ provides a set of common field behaviors.
-Applications can also define their own field behaviors by implementing the `FieldBehavior` interface,
-and attaching instances of their custom behavior classes to fields.
-
-===== ValueTooLongBehavior
-Used on String fields.  Requires the field to have a `maxLength` set.
-Depending on the chosen instance of this enum, before a record is Inserted or Updated,
-if the value in the field is longer than the `maxLength`, then one of the following actions can occur:
-
-* `TRUNCATE` - The value will be simply truncated to the `maxLength`.
-* `TRUNCATE_ELLIPSIS` - The value will be truncated to 3 characters less than the `maxLength`, and three periods (an ellipsis) will be placed at the end.
-* `ERROR` - An error will be reported, and the record will not be inserted or updated.
-* `PASS_THROUGH` - Nothing will happen.  This is the same as not having a `ValueTooLongBehavior` on the field.
-
-[source,java]
-.Examples of using ValueTooLongBehavior
-----
-   new QFieldMetaData("sku", QFieldType.STRING)
-      .withMaxLength(40),
-      .withBehavior(ValueTooLongBehavior.ERROR),
-
-   new QFieldMetaData("reason", QFieldType.STRING)
-      .withMaxLength(250),
-      .withBehavior(ValueTooLongBehavior.TRUNCATE_ELLIPSIS),
-
-----
-
-===== DynamicDefaultValueBehavior
-Used to set a dynamic default value to a field when it is being inserted or updated.
-For example, instead of having a hard-coded `defaultValue` specified in the field meta-data,
-and instead of having to add, for example, a pre-insert custom action.
-
-* `CREATE_DATE` - On inserts, sets the field's value to the current time.
-* `MODIFY_DATE` - On inserts and updates, sets the field's value to the current time.
-* `USER_ID` - On inserts and updates, sets the field's value to the current user's id (but only if the value is currently null).
-
-_Note that the `QInstanceEnricher` will, by default, add the `CREATE_DATE` and `MODIFY_DATE` `DynamicDefaultValueBehavior`
-options to any fields named `"createDate"` or `"modifyDate"`.
-This behavior can be disabled by setting the `configAddDynamicDefaultValuesToFieldsNamedCreateDateAndModifyDate` property
-on the `QInstanceEnricher` instance used by the application to `false`._
-
-[source,java]
-.Examples of using DynamicDefaultValueBehavior
-----
-   new QFieldMetaData("createDate", QFieldType.DATE_TIME)
-      .withBehavior(DynamicDefaultValueBehavior.CREATE_DATE),
-
-   new QFieldMetaData("modifyDate", QFieldType.DATE_TIME)
-      .withBehavior(DynamicDefaultValueBehavior.MODIFY_DATE),
-
-   new QFieldMetaData("createdByUserId", QFieldType.STRING)
-      .withBehavior(DynamicDefaultValueBehavior.USER_ID),
-----
-
-===== DateTimeDisplayValueBehavior
-By default, in QQQ, fields of type `DATE_TIME` are stored in UTC,
-and their values in a QRecord is a java `Instant` instance, which is always UTC.
-However, frontends will prefer to display date-time values in the user's local Time Zone whenever possible.
-
-Using `DateTimeDisplayValueBehavior` allows a `DATE_TIME` field to be displayed in a different Time Zone.
-An example use-case for this would be displaying airplane flight times,
-where you would want a flight from California to New York to display Pacific Time for its departure time,
-and Eastern Time for its arrival.
-
-An instance of `DateTimeDisplayValueBehavior` can be configured to either use a hard-coded time `ZoneId`
-(for example, to always show users UTC, or a business's home-office time zone).
-Or, it can be set up to get the time zone to use from another field in the table.
-
-[source,java]
-.Examples of using DateTimeDisplayValueBehavior
-----
-new QTableMetaData().withName("flights").withFields(List.of(
-   ...
-   new QFieldMetaData("departureTimeZoneId", QFieldType.STRING),
-   new QFieldMetaData("arrivaTimeZoneId", QFieldType.STRING),
-
-   new QFieldMetaData("departureTime", QFieldType.DATE_TIME)
-      .withBehavior(new DateTimeDisplayValueBehavior()
-         .withZoneIdFromFieldName("departureTimeZoneId")),
-
-   new QFieldMetaData("arrivalTime", QFieldType.DATE_TIME)
-      .withBehavior(new DateTimeDisplayValueBehavior()
-         .withZoneIdFromFieldName("arrivalTimeZoneId"))
-
-   new QFieldMetaData("ticketSaleStartDateTime", QFieldType.DATE_TIME)
-      .withBehavior(new DateTimeDisplayValueBehavior()
-         .withDefaultZoneId("UTC"))
-----
+Only applicable for fields with `type=STRING`.
+* `

docs/metaData/Icons.adoc

@@ -1,20 +0,0 @@
-[#Icons]
-== Icons
-include::../variables.adoc[]
-
-#TODO#
-
-=== QIcon
-Icons are defined in a QQQ Instance in a `*QIcon*` object.
-
-#TODO#
-
-*QIcon Properties:*
-
-* `name` - *String* - Name of an icon from the https://mui.com/material-ui/material-icons/[Material UI Icon set]
-** Note that icon names from the link above need to be converted from _CamelCase_ to _underscore_case_...
-* `path` - *String* - Path to a file served under the application's web root, to be used as the icon.
-
-_Either `name` or `path` must be specified. If both are given, then name is used._
-
-

docs/metaData/Joins.adoc

@@ -2,57 +2,16 @@
 == Joins
 include::../variables.adoc[]
 
-A `QJoinMetaData` is a meta-data object that tells QQQ, essentially “it is possible for these 2 tables to join, here’s how to do it”.
-
-Joins can be used then, in an application, in a number of possible ways:
-
-* In a {link-table}, we can specify joins to be “exposed”, e.g., made available to users on a query screen
-* Also in a Table, as part of an “Association”, which sets up one table as a “parent” of another,
-such that you can store (and fetch) the child-records at the same time as the parent
-** A common use-case here may be an order & lineItem table -
-such that QQQ can generate an API uses to allow you to post an order and its lines in a single request, and they get stored all together
-* In defining the security field (record lock) on a table,
-sometimes, it isn’t a field directly on the table, but instead comes from a joined table (possibly even more than just 1 table away).
-** For example, maybe a lineItem table, doesn't have a clientId, but needs secured by that field
-- so its recordLock can specify a “joinNameChain” that describes how to get from lineItem to order.clientId
-* The `QueryAction` can take (through its QueryInput object) zero or more QueryJoin objects,
-which must make a reference (implicitly or explicitly) to a QJoinMetaData.
-See the section on <<QueryJoin,QueryJoins>> for more details.
+#TODO#
 
 === QJoinMetaData
 Joins are defined in a QQQ Instance in a `*QJoinMetaData*` object.
 
-In this object, we have the concept of a "leftTable" and a "rightTable".
-There isn't generally anything special about which table is on the "left" and which is on the "right".
-But the remaining pieces of the QJoinMetaData do all need to line-up with these sides.
-
-For example:
-
-* The Type (one-to-one, one-to-many, many-to-one) - where the leftTable comes first, and rightTable comes second
-(e.g., a one-to-many means 1-row in leftTable has many-rows in rightTable associated with it)
-* In a JoinOn object, the 1st field name given is from the leftTable;
-the second fieldName from the rightTable.
+#TODO#
 
 *QJoinMetaData Properties:*
 
-* `name` - *String, Required* - Unique name for the join within the QQQ Instance.
-** One convention is to name joins based on (leftTable + "Join" + rightTable).
-** If you do not wish to define join names yourself, the method `withInferredName()`
-can be called (which defers to
-`public static String makeInferredJoinName(String leftTable, String rightTable)`),
-to create a name for the join following the (leftTable + "Join" + rightTable) convention.
-* `leftTable` - *String, Required* - Name of a {link-table} in the {link-instance}.
-* `rightTable` - *String, Required* - Name of a {link-table} in the {link-instance}.
-* `type` - *enum, Required* - cardinality between the two tables in the join.
-** e.g., `ONE_TO_ONE`, `ONE_TO_MANY` (indicating 1 record in the left table may join
-to many records in the right table), or `MANY_TO_ONE` (vice-versa).
-** Note that there is no MANY_TO_MANY option, as a many-to-many is built as multiple QJoinMetaData's
-going through the intermediary (intersection) table.
-* `joinOns` - *List<JoinOn>, Required* - fields used to join the tables.
-Note: In the 2-arg JoinOn constructor, the leftTable's field comes first.
-Alternatively, the no-arg constructor can be used along with `.withLeftField().withRightField()`
-* `orderBys` - *List<QFilterOrderBy>* - Optional list of order-by objects,
-used in some framework-generated queries using the join.
-The field names are assumed to come from the rightTable.
+* `name` - *String, Required* - Unique name for the join within the QQQ Instance. #todo infererences or conventions?#
+
+#TODO#
 
-#TODO# what else do we need here?

docs/metaData/PermissionRules.adoc

@@ -1,13 +0,0 @@
-[#PermissionRules]
-== Permission Rules
-include::../variables.adoc[]
-
-#TODO#
-
-=== PermissionRule
-#TODO#
-
-*PermissionRule Properties:*
-
-#TODO#
-

docs/metaData/Processes.adoc

@@ -16,7 +16,6 @@ Processes are defined in a QQQ Instance in a `*QProcessMetaData*` object.
 In addition to directly building a `QProcessMetaData` object setting its properties directly, there are a few common process patterns that provide *Builder* objects for ease-of-use.
 See StreamedETLWithFrontendProcess below for a common example
 
-[#_QProcessMetaData_Properties]
 *QProcessMetaData Properties:*
 
 * `name` - *String, Required* - Unique name for the process within the QQQ Instance.
@@ -31,13 +30,12 @@ See below for details.
 * `permissionRules` - *QPermissionRules object* - define the permission/access rules for the process.
 See {link-permissionRules} for details.
 * `steps` and `stepList` - *Map of String → <<QStepMetaData>>* and *List of QStepMetaData* - Defines the <<QFrontendStepMetaData,screens>> and <<QBackendStepMetaData,backend code>> that makes up the process.
-** `stepList` is the list of steps in the order that they will be executed
-(that is to say - this is the _default_ order of execution - but it can be customized - see <<_custom_process_flow>> for details).
-** `steps` is a map, including all steps from `stepList`, but which may also include steps which can used by the process if its backend steps make the decision to do so, at run-time (e.g., using <<_custom_process_flow>>).
+** `stepList` is the list of steps in the order that they will by default be executed.
+** `steps` is a map, including all steps from `stepList`, but which may also include steps which can used by the process if its backend steps make the decision to do so, at run-time.
 ** A process's steps are normally defined in one of two was:
 *** 1) by a single call to `.withStepList(List<QStepMetaData>)`, which internally adds each step into the `steps` map.
 *** 2) by multiple calls to `.addStep(QStepMetaData)`, which adds a step to both the `stepList` and `steps` map.
-** If a process also needs optional steps (for a <<_custom_process_flow>>), they should be added by a call to `.addOptionalStep(QStepMetaData)`, which only places them in the `steps` map.
+** If a process also needs optional steps, they should be added by a call to `.addOptionalStep(QStepMetaData)`, which only places them in the `steps` map.
 * `schedule` - *<<QScheduleMetaData>>* - set up the process to run automatically on the specified schedule.
 See below for details.
 * `minInputRecords` - *Integer* - #not used...#
@@ -216,112 +214,3 @@ But for some cases, doing page-level transactions can reduce long-transactions a
 * `withFields(List<QFieldMetaData> fieldList)` - Adds additional input fields to the preview step of the process.
 * `withBasepullConfiguration(BasepullConfiguration basepullConfiguration)` - Add a <<BasepullConfiguration>> to the process.
 * `withSchedule(QScheduleMetaData schedule)` - Add a <<QScheduleMetaData>> to the process.
-
-[#_custom_process_flow]
-==== Custom Process Flow
-As referenced in the definition of the <<_QProcessMetaData_Properties,QProcessMetaData Properties>>, by default, a process
-will execute each of its steps in-order, as defined in the `stepList` property.
-However, a Backend Step can customize this flow #todo - write more clearly here...
-
-There are generally 2 method to call (in a `BackendStep`) to do a dynamic flow:
-
-* `RunBackendStepOutput.setOverrideLastStepName(String stepName)`
-** QQQ's `RunProcessAction` keeps track of which step it "last" ran, e.g., to tell it which one to run next.
-However, if a step sets the `OverrideLastStepName` property in its output object,
-then the step named in that property becomes the effective "last" step,
-thus determining which step comes next.
-
-* `RunBackendStepOutput.updateStepList(List<String> stepNameList)`
-** Calling this method changes the process's runtime definition of steps to be executed.
-Thus allowing a completely custom flow.
-It should be noted, that the "last" step name (as tracked by QQQ within `RunProcessAction`)
-does need to be found in the new `stepNameList` - otherwise, the framework will not know where you were,
-for figuring out where to go next.
-
-[source,java]
-.Example of a defining process that can use a flexible flow:
-----
-// for a case like this, it would be recommended to define all step names in constants:
-public final static String STEP_START = "start";
-public final static String STEP_A     = "a";
-public final static String STEP_B     = "b";
-public final static String STEP_C     = "c";
-public final static String STEP_1     = "1";
-public final static String STEP_2     = "2";
-public final static String STEP_3     = "3";
-public final static String STEP_END   = "end";
-
-// also, to define the possible flows (lists of steps) in constants as well:
-public final static List<String> LETTERS_STEP_LIST = List.of(
-  STEP_START, STEP_A, STEP_B, STEP_C, STEP_END);
-
-public final static List<String> NUMBERS_STEP_LIST = List.of(
-  STEP_START, STEP_1, STEP_2, STEP_3, STEP_END);
-
-// when we define the process's meta-data, we only give a "skeleton" stepList -
-// we must at least have our starting step, and we may want at least one frontend step
-// for the UI to show some placeholder(s):
-QProcessMetaData process = new QProcessMetaData()
-   .withName(PROCESS_NAME)
-   .withStepList(List.of(
-      new QBackendStepMetaData().withName(STEP_START)
-         .withCode(new QCodeReference(/*...*/)),
-      new QFrontendStepMetaData()
-         .withName(STEP_END)
-   ));
-
-// the additional steps get added via `addOptionalStep`, which only puts them in
-// the process's stepMap, not its stepList!
-process.addOptionalStep(new QFrontendStepMetaData().withName(STEP_A));
-process.addOptionalStep(new QBackendStepMetaData().withName(STEP_B)
-   .withCode(new QCodeReference(/*...*/)));
-process.addOptionalStep(new QFrontendStepMetaData().withName(STEP_C));
-
-process.addOptionalStep(new QBackendStepMetaData().withName(STEP_1)
-   .withCode(new QCodeReference(/*...*/)));
-process.addOptionalStep(new QFrontendStepMetaData().withName(STEP_2));
-process.addOptionalStep(new QBackendStepMetaData().withName(STEP_3)
-   .withCode(new QCodeReference(/*...*/)));
-
-----
-
-[source,java]
-.Example of a process backend step adjusting the process's runtime flow:
-----
-/***************************************************************************
-** look at the value named "which". if it's "letters", then make the process
-** go through the stepList consisting of letters; else, update the step list
-** to be the "numbers" steps.
-**
-** Also - if the "skipSomeSteps" value is give as true, then set the
-** overrideLastStepName to skip again (in the letters case, skip past A, B
-** and C; in the numbers case, skip past 1 and 2).
-***************************************************************************/
-public static class StartStep implements BackendStep
-{
-  @Override
-  public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException
-  {
-     Boolean skipSomeSteps = runBackendStepInput.getValueBoolean("skipSomeSteps");
-
-     if(runBackendStepInput.getValueString("which").equals("letters"))
-     {
-        runBackendStepOutput.updateStepList(LETTERS_STEP_LIST);
-        if(BooleanUtils.isTrue(skipSomeSteps))
-        {
-           runBackendStepOutput.setOverrideLastStepName(STEP_C);
-        }
-     }
-     else
-     {
-        runBackendStepOutput.updateStepList(NUMBERS_STEP_LIST);
-        if(BooleanUtils.isTrue(skipSomeSteps))
-        {
-           runBackendStepOutput.setOverrideLastStepName(STEP_2);
-        }
-     }
-  }
-}
-----
-
-

docs/metaData/QInstance.adoc

@@ -1,39 +0,0 @@
-[#QInstance]
-== QInstance
-include::../variables.adoc[]
-
-An application in QQQ is defined as a set of Meta Data objects.  These objects are all stored together in an object called a `*QInstance*`.
-
-Currently, a `QInstance` must be programmatically constructed in java code - e.g., by constructing objects which get added to the QInstance, for example:
-
-[source,java]
-.Adding meta-data for two tables and one process to a QInstance
-----
-QInstance qInstance = new QInstance();
-qInstance.addTable(definePersonTable());
-qInstance.addTable(defineHomeTable());
-qInstance.addProcess(defineSendPersonHomeProcess());
-----
-
-It is on the QQQ roadmap to allow meta-data to be defined in a non-programmatic way, for example, in YAML or JSON files, or even from a dynamic data source (e.g. a document or relational database).
-
-The middleware and/or frontends being used in an application will drive how the `QInstance` is connected to the running server/process.
-For example, using the `qqq-middleware-javalin` module, a the `QJavalinImplementation` class () has a constructor which takes a `QInstance` as an argument:
-
-[source,java]
-.Starting a QQQ Javalin middleware server - passing a QInstance as a parameter to the new QJavalinImplementation
-----
-QJavalinImplementation qJavalinImplementation = new QJavalinImplementation(qInstance);
-Javalin service = Javalin.create();
-service.routes(qJavalinImplementation.getRoutes());
-service.start();
-----
-
-*QBackendMetaData Setup Methods:*
-
-These are the methods that one is most likely to use when setting up (defining) a `QInstance` object:
-
-* asdf
-
-*QBackendMetaData Usage Methods:*
-

docs/metaData/SecurtiyKeyTypes.adoc

@@ -2,79 +2,16 @@
 == Security Key Types
 include::../variables.adoc[]
 
-In QQQ, record-level security is provided by using a lock & key metaphor.
-
-The use-case being handled here is:
-
-* A user has full permission on a table (query, insert, update, and delete).
-* However, they should only be allowed to read a sub-set of the rows in the table.
-** e.g., maybe it's a multi-tenant system, or the table has user-specific records.
-
-The lock & key metaphor is realized by the user being associated with one or more "Keys"
-(as values in their session), and records in tables being associated with one or more "Locks"
-(as values in fields).
-A user is only allowed to access records where the user's key(s) match the record's security lock(s).
-
-For a practical example, picture a multi-tenant Order Management System,where all orders are assigned to a "client".
-Users (customers) should only be able to see orders associated with the client which that user works for.
-
-In this scenario, the `order` table would have a "lock" on its `clientId` field.
-Customer-users would have a `clientId` key in their session.
-When the QQQ backend did a search for records (e.g., an SQL query) it would implicitly
-(without any code being written by the application developer) filter the table to only
-allow the user to see records with their `clientId`.
-
-To implement this scenario, the application would define the following pieces of meta-data:
-
-* At the QQQ-Instance level, a `SecurityKeyType`,
-to define a domain of possible locks & keys within an application.
-** An application can define multiple Security Key Types.
-For example, maybe `clientId` and `userId` as key types.
-* At the per-table level, a `RecordSecurityLock`,
-which references a security key type, and how that key type should be applied to the table.
-** For example, what field stores the `clientId` value in the `order` table.
-* Finally, when a user's session is constructed via a QQQ Authentication provider,
-security key values are set, based on data from the authentication backend.
-
-=== Additional Scenarios
-
-==== All Access Key
-A "super-user" may be allowed to access all records in a table regardless of their record locks,
-if the Security Key Type specifies an `allAccessKeyName`,
-and if the user has a key in their session with that key name, and a value of `true`.
-Going back to the lock & key metaphor, this can be thought of as a "skeleton key",
-that can unlock any record lock (of the security key's type).
-
-==== Null Value Behaviors
-In a record security lock, different behaviors can be defined for handling rows with a null key value.
-
-For example:
-
-* Sometimes orders may be loaded into the OMS system described above, where the application doesn't yet know what client the order belongs to.
-In this case, the application may need to ensure that such records, with a `null` value in `clientId` are hidden from customer-users,
-to avoid potentially leaking a different client's data.
-** This can be accomplished with a record security lock on the `order` table, with a `nullValueBehavior` of `DENY`.
-** Furthermore, if internal/admin users _should_ be given access to such records, then the security key type can be
-configured with a `nullValueBehaviorKeyName` (e.g., `"clientIdNullValueBehavior"`), which can be set per-user to allow
-access to records, overriding the table lock's specified `nullValueBehavior`.
-*** This could also be done by giving internal/admin users an `allAccessKey`, but sometimes that is not what is required.
-
-* Maybe a warehouse locations table is assigned a `clientId` once inventory for a client is placed in the location,
-at which point in time, only the client's users should be allowed to see the record.
-But, if no client has been assigned to the location, and `clientId` is `null`,
-then you may want to allow any user to see such records.
-** This can be accomplished with a record security lock on the Warehouse Locations table, with a `nullValueBehavior` of `ALLOW`.
+#TODO#
 
 === QSecurityKeyType
 A Security Key Type is defined in a QQQ Instance in a `*QSecurityKeyType*` object.
 
+#TODO#
+
 *QSecurityKeyType Properties:*
 
-* `name` - *String, Required* - Unique name for this security key type within the QQQ Instance.
-* `allAccessKeyName` - *String* - Optional name of the all-access security key associated with this key type.
-* `nullValueBehaviorKeyName` - *String* - Optional name of the null-value-behavior overriding security key associated with this key type.
-** Note, `name`, `allAccessKeyName`, and `nullValueBehaviorKeyName` are all checked against each other for uniqueness.
-A `QInstanceValidationException` will be thrown if any name collisions occur.
-* `possibleValueSourceName` - *String* - Optional reference to a possible value source from which value for the key can come.
+* `name` - *String, Required* - Unique name for the security key type within the QQQ Instance.
 
+#TODO#
 

docs/metaData/Tables.adoc

@@ -244,41 +244,3 @@ QQQ provides the mechanism for UI's to present and manage such scripts (e.g., th
 * `scriptTypeId` - *Serializable (typically Integer), Required* - primary key value from the `"scriptType"` table in the instance, to designate the type of the Script.
 * `scriptTester` - *QCodeReference* - reference to a class which implements `TestScriptActionInterface`, that can be used by UI's for running an associated script to test it.
 
-
-
-=== Supplemental Meta Data
-==== QQQ Frontend Material Dashboard
-When running a QQQ application with the QQQ Frontend Material Dashboard module (QFMD),
-there are various pieces of supplemental meta-data which can be assigned to a Table,
-to modify some behaviors for the table in this UI.
-
-===== Default Quick Filter Field Names
-QFMD's table query has a "Basic" mode, which will always display a subset of the table's fields as quick-access filters.
-By default, the "Tier 1" fields on a table (e.g., fields in a Section that is marked as T1) will be used for this purpose.
-
-However, you can customize which fields are shown as the default quick-filter fields, by providing a list of field names in a
-`MaterialDashboardTableMetaData` object, placed in the table's `supplementalMetaData`.
-
-[source,java]
-----
-table.withSupplementalMetaData(new MaterialDashboardTableMetaData()
-    .withDefaultQuickFilterFieldNames(List.of("id", "warehouseId", "statusId", "orderDate")));
-----
-
-===== Go To Field Names
-QFMD has a feature where a table's query screen can include a "Go To" button,
-which a user can hit to open a modal popup, into which the user can enter a record's identifier,
-to be brought directly to the record matching that identifier.
-
-To use this feature, the table must have a List of `GotoFieldNames` set in its
-`MaterialDashboardTableMetaData` object in the table's `supplementalMetaData`.
-
-Each entry in this list is actually a list of fields, e.g., to account for a multi-value unique-key.
-
-[source,java]
-----
-table.withSupplementalMetaData(new MaterialDashboardTableMetaData()
-    .withGotoFieldNames(List.of(
-        List.of("id"),
-        List.of("partnerName", "partnerOrderId"))));
-----

docs/metaData/Tables.html

@@ -0,0 +1,553 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+<meta charset="UTF-8">
+<meta http-equiv="X-UA-Compatible" content="IE=edge">
+<meta name="viewport" content="width=device-width, initial-scale=1.0">
+<meta name="generator" content="Asciidoctor 2.0.18">
+<title>QQQ Tables</title>
+<link rel="stylesheet" href="https://fonts.googleapis.com/css?family=Open+Sans:300,300italic,400,400italic,600,600italic%7CNoto+Serif:400,400italic,700,700italic%7CDroid+Sans+Mono:400,700">
+<style>
+/*! Asciidoctor default stylesheet | MIT License | https://asciidoctor.org */
+/* Uncomment the following line when using as a custom stylesheet */
+/* @import "https://fonts.googleapis.com/css?family=Open+Sans:300,300italic,400,400italic,600,600italic%7CNoto+Serif:400,400italic,700,700italic%7CDroid+Sans+Mono:400,700"; */
+html{font-family:sans-serif;-webkit-text-size-adjust:100%}
+a{background:none}
+a:focus{outline:thin dotted}
+a:active,a:hover{outline:0}
+h1{font-size:2em;margin:.67em 0}
+b,strong{font-weight:bold}
+abbr{font-size:.9em}
+abbr[title]{cursor:help;border-bottom:1px dotted #dddddf;text-decoration:none}
+dfn{font-style:italic}
+hr{height:0}
+mark{background:#ff0;color:#000}
+code,kbd,pre,samp{font-family:monospace;font-size:1em}
+pre{white-space:pre-wrap}
+q{quotes:"\201C" "\201D" "\2018" "\2019"}
+small{font-size:80%}
+sub,sup{font-size:75%;line-height:0;position:relative;vertical-align:baseline}
+sup{top:-.5em}
+sub{bottom:-.25em}
+img{border:0}
+svg:not(:root){overflow:hidden}
+figure{margin:0}
+audio,video{display:inline-block}
+audio:not([controls]){display:none;height:0}
+fieldset{border:1px solid silver;margin:0 2px;padding:.35em .625em .75em}
+legend{border:0;padding:0}
+button,input,select,textarea{font-family:inherit;font-size:100%;margin:0}
+button,input{line-height:normal}
+button,select{text-transform:none}
+button,html input[type=button],input[type=reset],input[type=submit]{-webkit-appearance:button;cursor:pointer}
+button[disabled],html input[disabled]{cursor:default}
+input[type=checkbox],input[type=radio]{padding:0}
+button::-moz-focus-inner,input::-moz-focus-inner{border:0;padding:0}
+textarea{overflow:auto;vertical-align:top}
+table{border-collapse:collapse;border-spacing:0}
+*,::before,::after{box-sizing:border-box}
+html,body{font-size:100%}
+body{background:#fff;color:rgba(0,0,0,.8);padding:0;margin:0;font-family:"Noto Serif","DejaVu Serif",serif;line-height:1;position:relative;cursor:auto;-moz-tab-size:4;-o-tab-size:4;tab-size:4;word-wrap:anywhere;-moz-osx-font-smoothing:grayscale;-webkit-font-smoothing:antialiased}
+a:hover{cursor:pointer}
+img,object,embed{max-width:100%;height:auto}
+object,embed{height:100%}
+img{-ms-interpolation-mode:bicubic}
+.left{float:left!important}
+.right{float:right!important}
+.text-left{text-align:left!important}
+.text-right{text-align:right!important}
+.text-center{text-align:center!important}
+.text-justify{text-align:justify!important}
+.hide{display:none}
+img,object,svg{display:inline-block;vertical-align:middle}
+textarea{height:auto;min-height:50px}
+select{width:100%}
+.subheader,.admonitionblock td.content>.title,.audioblock>.title,.exampleblock>.title,.imageblock>.title,.listingblock>.title,.literalblock>.title,.stemblock>.title,.openblock>.title,.paragraph>.title,.quoteblock>.title,table.tableblock>.title,.verseblock>.title,.videoblock>.title,.dlist>.title,.olist>.title,.ulist>.title,.qlist>.title,.hdlist>.title{line-height:1.45;color:#7a2518;font-weight:400;margin-top:0;margin-bottom:.25em}
+div,dl,dt,dd,ul,ol,li,h1,h2,h3,#toctitle,.sidebarblock>.content>.title,h4,h5,h6,pre,form,p,blockquote,th,td{margin:0;padding:0}
+a{color:#2156a5;text-decoration:underline;line-height:inherit}
+a:hover,a:focus{color:#1d4b8f}
+a img{border:0}
+p{line-height:1.6;margin-bottom:1.25em;text-rendering:optimizeLegibility}
+p aside{font-size:.875em;line-height:1.35;font-style:italic}
+h1,h2,h3,#toctitle,.sidebarblock>.content>.title,h4,h5,h6{font-family:"Open Sans","DejaVu Sans",sans-serif;font-weight:300;font-style:normal;color:#ba3925;text-rendering:optimizeLegibility;margin-top:1em;margin-bottom:.5em;line-height:1.0125em}
+h1 small,h2 small,h3 small,#toctitle small,.sidebarblock>.content>.title small,h4 small,h5 small,h6 small{font-size:60%;color:#e99b8f;line-height:0}
+h1{font-size:2.125em}
+h2{font-size:1.6875em}
+h3,#toctitle,.sidebarblock>.content>.title{font-size:1.375em}
+h4,h5{font-size:1.125em}
+h6{font-size:1em}
+hr{border:solid #dddddf;border-width:1px 0 0;clear:both;margin:1.25em 0 1.1875em}
+em,i{font-style:italic;line-height:inherit}
+strong,b{font-weight:bold;line-height:inherit}
+small{font-size:60%;line-height:inherit}
+code{font-family:"Droid Sans Mono","DejaVu Sans Mono",monospace;font-weight:400;color:rgba(0,0,0,.9)}
+ul,ol,dl{line-height:1.6;margin-bottom:1.25em;list-style-position:outside;font-family:inherit}
+ul,ol{margin-left:1.5em}
+ul li ul,ul li ol{margin-left:1.25em;margin-bottom:0}
+ul.circle{list-style-type:circle}
+ul.disc{list-style-type:disc}
+ul.square{list-style-type:square}
+ul.circle ul:not([class]),ul.disc ul:not([class]),ul.square ul:not([class]){list-style:inherit}
+ol li ul,ol li ol{margin-left:1.25em;margin-bottom:0}
+dl dt{margin-bottom:.3125em;font-weight:bold}
+dl dd{margin-bottom:1.25em}
+blockquote{margin:0 0 1.25em;padding:.5625em 1.25em 0 1.1875em;border-left:1px solid #ddd}
+blockquote,blockquote p{line-height:1.6;color:rgba(0,0,0,.85)}
+@media screen and (min-width:768px){h1,h2,h3,#toctitle,.sidebarblock>.content>.title,h4,h5,h6{line-height:1.2}
+h1{font-size:2.75em}
+h2{font-size:2.3125em}
+h3,#toctitle,.sidebarblock>.content>.title{font-size:1.6875em}
+h4{font-size:1.4375em}}
+table{background:#fff;margin-bottom:1.25em;border:1px solid #dedede;word-wrap:normal}
+table thead,table tfoot{background:#f7f8f7}
+table thead tr th,table thead tr td,table tfoot tr th,table tfoot tr td{padding:.5em .625em .625em;font-size:inherit;color:rgba(0,0,0,.8);text-align:left}
+table tr th,table tr td{padding:.5625em .625em;font-size:inherit;color:rgba(0,0,0,.8)}
+table tr.even,table tr.alt{background:#f8f8f7}
+table thead tr th,table tfoot tr th,table tbody tr td,table tr td,table tfoot tr td{line-height:1.6}
+h1,h2,h3,#toctitle,.sidebarblock>.content>.title,h4,h5,h6{line-height:1.2;word-spacing:-.05em}
+h1 strong,h2 strong,h3 strong,#toctitle strong,.sidebarblock>.content>.title strong,h4 strong,h5 strong,h6 strong{font-weight:400}
+.center{margin-left:auto;margin-right:auto}
+.stretch{width:100%}
+.clearfix::before,.clearfix::after,.float-group::before,.float-group::after{content:" ";display:table}
+.clearfix::after,.float-group::after{clear:both}
+:not(pre).nobreak{word-wrap:normal}
+:not(pre).nowrap{white-space:nowrap}
+:not(pre).pre-wrap{white-space:pre-wrap}
+:not(pre):not([class^=L])>code{font-size:.9375em;font-style:normal!important;letter-spacing:0;padding:.1em .5ex;word-spacing:-.15em;background:#f7f7f8;border-radius:4px;line-height:1.45;text-rendering:optimizeSpeed}
+pre{color:rgba(0,0,0,.9);font-family:"Droid Sans Mono","DejaVu Sans Mono",monospace;line-height:1.45;text-rendering:optimizeSpeed}
+pre code,pre pre{color:inherit;font-size:inherit;line-height:inherit}
+pre>code{display:block}
+pre.nowrap,pre.nowrap pre{white-space:pre;word-wrap:normal}
+em em{font-style:normal}
+strong strong{font-weight:400}
+.keyseq{color:rgba(51,51,51,.8)}
+kbd{font-family:"Droid Sans Mono","DejaVu Sans Mono",monospace;display:inline-block;color:rgba(0,0,0,.8);font-size:.65em;line-height:1.45;background:#f7f7f7;border:1px solid #ccc;border-radius:3px;box-shadow:0 1px 0 rgba(0,0,0,.2),inset 0 0 0 .1em #fff;margin:0 .15em;padding:.2em .5em;vertical-align:middle;position:relative;top:-.1em;white-space:nowrap}
+.keyseq kbd:first-child{margin-left:0}
+.keyseq kbd:last-child{margin-right:0}
+.menuseq,.menuref{color:#000}
+.menuseq b:not(.caret),.menuref{font-weight:inherit}
+.menuseq{word-spacing:-.02em}
+.menuseq b.caret{font-size:1.25em;line-height:.8}
+.menuseq i.caret{font-weight:bold;text-align:center;width:.45em}
+b.button::before,b.button::after{position:relative;top:-1px;font-weight:400}
+b.button::before{content:"[";padding:0 3px 0 2px}
+b.button::after{content:"]";padding:0 2px 0 3px}
+p a>code:hover{color:rgba(0,0,0,.9)}
+#header,#content,#footnotes,#footer{width:100%;margin:0 auto;max-width:62.5em;*zoom:1;position:relative;padding-left:.9375em;padding-right:.9375em}
+#header::before,#header::after,#content::before,#content::after,#footnotes::before,#footnotes::after,#footer::before,#footer::after{content:" ";display:table}
+#header::after,#content::after,#footnotes::after,#footer::after{clear:both}
+#content{margin-top:1.25em}
+#content::before{content:none}
+#header>h1:first-child{color:rgba(0,0,0,.85);margin-top:2.25rem;margin-bottom:0}
+#header>h1:first-child+#toc{margin-top:8px;border-top:1px solid #dddddf}
+#header>h1:only-child,body.toc2 #header>h1:nth-last-child(2){border-bottom:1px solid #dddddf;padding-bottom:8px}
+#header .details{border-bottom:1px solid #dddddf;line-height:1.45;padding-top:.25em;padding-bottom:.25em;padding-left:.25em;color:rgba(0,0,0,.6);display:flex;flex-flow:row wrap}
+#header .details span:first-child{margin-left:-.125em}
+#header .details span.email a{color:rgba(0,0,0,.85)}
+#header .details br{display:none}
+#header .details br+span::before{content:"\00a0\2013\00a0"}
+#header .details br+span.author::before{content:"\00a0\22c5\00a0";color:rgba(0,0,0,.85)}
+#header .details br+span#revremark::before{content:"\00a0|\00a0"}
+#header #revnumber{text-transform:capitalize}
+#header #revnumber::after{content:"\00a0"}
+#content>h1:first-child:not([class]){color:rgba(0,0,0,.85);border-bottom:1px solid #dddddf;padding-bottom:8px;margin-top:0;padding-top:1rem;margin-bottom:1.25rem}
+#toc{border-bottom:1px solid #e7e7e9;padding-bottom:.5em}
+#toc>ul{margin-left:.125em}
+#toc ul.sectlevel0>li>a{font-style:italic}
+#toc ul.sectlevel0 ul.sectlevel1{margin:.5em 0}
+#toc ul{font-family:"Open Sans","DejaVu Sans",sans-serif;list-style-type:none}
+#toc li{line-height:1.3334;margin-top:.3334em}
+#toc a{text-decoration:none}
+#toc a:active{text-decoration:underline}
+#toctitle{color:#7a2518;font-size:1.2em}
+@media screen and (min-width:768px){#toctitle{font-size:1.375em}
+body.toc2{padding-left:15em;padding-right:0}
+#toc.toc2{margin-top:0!important;background:#f8f8f7;position:fixed;width:15em;left:0;top:0;border-right:1px solid #e7e7e9;border-top-width:0!important;border-bottom-width:0!important;z-index:1000;padding:1.25em 1em;height:100%;overflow:auto}
+#toc.toc2 #toctitle{margin-top:0;margin-bottom:.8rem;font-size:1.2em}
+#toc.toc2>ul{font-size:.9em;margin-bottom:0}
+#toc.toc2 ul ul{margin-left:0;padding-left:1em}
+#toc.toc2 ul.sectlevel0 ul.sectlevel1{padding-left:0;margin-top:.5em;margin-bottom:.5em}
+body.toc2.toc-right{padding-left:0;padding-right:15em}
+body.toc2.toc-right #toc.toc2{border-right-width:0;border-left:1px solid #e7e7e9;left:auto;right:0}}
+@media screen and (min-width:1280px){body.toc2{padding-left:20em;padding-right:0}
+#toc.toc2{width:20em}
+#toc.toc2 #toctitle{font-size:1.375em}
+#toc.toc2>ul{font-size:.95em}
+#toc.toc2 ul ul{padding-left:1.25em}
+body.toc2.toc-right{padding-left:0;padding-right:20em}}
+#content #toc{border:1px solid #e0e0dc;margin-bottom:1.25em;padding:1.25em;background:#f8f8f7;border-radius:4px}
+#content #toc>:first-child{margin-top:0}
+#content #toc>:last-child{margin-bottom:0}
+#footer{max-width:none;background:rgba(0,0,0,.8);padding:1.25em}
+#footer-text{color:hsla(0,0%,100%,.8);line-height:1.44}
+#content{margin-bottom:.625em}
+.sect1{padding-bottom:.625em}
+@media screen and (min-width:768px){#content{margin-bottom:1.25em}
+.sect1{padding-bottom:1.25em}}
+.sect1:last-child{padding-bottom:0}
+.sect1+.sect1{border-top:1px solid #e7e7e9}
+#content h1>a.anchor,h2>a.anchor,h3>a.anchor,#toctitle>a.anchor,.sidebarblock>.content>.title>a.anchor,h4>a.anchor,h5>a.anchor,h6>a.anchor{position:absolute;z-index:1001;width:1.5ex;margin-left:-1.5ex;display:block;text-decoration:none!important;visibility:hidden;text-align:center;font-weight:400}
+#content h1>a.anchor::before,h2>a.anchor::before,h3>a.anchor::before,#toctitle>a.anchor::before,.sidebarblock>.content>.title>a.anchor::before,h4>a.anchor::before,h5>a.anchor::before,h6>a.anchor::before{content:"\00A7";font-size:.85em;display:block;padding-top:.1em}
+#content h1:hover>a.anchor,#content h1>a.anchor:hover,h2:hover>a.anchor,h2>a.anchor:hover,h3:hover>a.anchor,#toctitle:hover>a.anchor,.sidebarblock>.content>.title:hover>a.anchor,h3>a.anchor:hover,#toctitle>a.anchor:hover,.sidebarblock>.content>.title>a.anchor:hover,h4:hover>a.anchor,h4>a.anchor:hover,h5:hover>a.anchor,h5>a.anchor:hover,h6:hover>a.anchor,h6>a.anchor:hover{visibility:visible}
+#content h1>a.link,h2>a.link,h3>a.link,#toctitle>a.link,.sidebarblock>.content>.title>a.link,h4>a.link,h5>a.link,h6>a.link{color:#ba3925;text-decoration:none}
+#content h1>a.link:hover,h2>a.link:hover,h3>a.link:hover,#toctitle>a.link:hover,.sidebarblock>.content>.title>a.link:hover,h4>a.link:hover,h5>a.link:hover,h6>a.link:hover{color:#a53221}
+details,.audioblock,.imageblock,.literalblock,.listingblock,.stemblock,.videoblock{margin-bottom:1.25em}
+details{margin-left:1.25rem}
+details>summary{cursor:pointer;display:block;position:relative;line-height:1.6;margin-bottom:.625rem;outline:none;-webkit-tap-highlight-color:transparent}
+details>summary::-webkit-details-marker{display:none}
+details>summary::before{content:"";border:solid transparent;border-left:solid;border-width:.3em 0 .3em .5em;position:absolute;top:.5em;left:-1.25rem;transform:translateX(15%)}
+details[open]>summary::before{border:solid transparent;border-top:solid;border-width:.5em .3em 0;transform:translateY(15%)}
+details>summary::after{content:"";width:1.25rem;height:1em;position:absolute;top:.3em;left:-1.25rem}
+.admonitionblock td.content>.title,.audioblock>.title,.exampleblock>.title,.imageblock>.title,.listingblock>.title,.literalblock>.title,.stemblock>.title,.openblock>.title,.paragraph>.title,.quoteblock>.title,table.tableblock>.title,.verseblock>.title,.videoblock>.title,.dlist>.title,.olist>.title,.ulist>.title,.qlist>.title,.hdlist>.title{text-rendering:optimizeLegibility;text-align:left;font-family:"Noto Serif","DejaVu Serif",serif;font-size:1rem;font-style:italic}
+table.tableblock.fit-content>caption.title{white-space:nowrap;width:0}
+.paragraph.lead>p,#preamble>.sectionbody>[class=paragraph]:first-of-type p{font-size:1.21875em;line-height:1.6;color:rgba(0,0,0,.85)}
+.admonitionblock>table{border-collapse:separate;border:0;background:none;width:100%}
+.admonitionblock>table td.icon{text-align:center;width:80px}
+.admonitionblock>table td.icon img{max-width:none}
+.admonitionblock>table td.icon .title{font-weight:bold;font-family:"Open Sans","DejaVu Sans",sans-serif;text-transform:uppercase}
+.admonitionblock>table td.content{padding-left:1.125em;padding-right:1.25em;border-left:1px solid #dddddf;color:rgba(0,0,0,.6);word-wrap:anywhere}
+.admonitionblock>table td.content>:last-child>:last-child{margin-bottom:0}
+.exampleblock>.content{border:1px solid #e6e6e6;margin-bottom:1.25em;padding:1.25em;background:#fff;border-radius:4px}
+.exampleblock>.content>:first-child{margin-top:0}
+.exampleblock>.content>:last-child{margin-bottom:0}
+.sidebarblock{border:1px solid #dbdbd6;margin-bottom:1.25em;padding:1.25em;background:#f3f3f2;border-radius:4px}
+.sidebarblock>:first-child{margin-top:0}
+.sidebarblock>:last-child{margin-bottom:0}
+.sidebarblock>.content>.title{color:#7a2518;margin-top:0;text-align:center}
+.exampleblock>.content>:last-child>:last-child,.exampleblock>.content .olist>ol>li:last-child>:last-child,.exampleblock>.content .ulist>ul>li:last-child>:last-child,.exampleblock>.content .qlist>ol>li:last-child>:last-child,.sidebarblock>.content>:last-child>:last-child,.sidebarblock>.content .olist>ol>li:last-child>:last-child,.sidebarblock>.content .ulist>ul>li:last-child>:last-child,.sidebarblock>.content .qlist>ol>li:last-child>:last-child{margin-bottom:0}
+.literalblock pre,.listingblock>.content>pre{border-radius:4px;overflow-x:auto;padding:1em;font-size:.8125em}
+@media screen and (min-width:768px){.literalblock pre,.listingblock>.content>pre{font-size:.90625em}}
+@media screen and (min-width:1280px){.literalblock pre,.listingblock>.content>pre{font-size:1em}}
+.literalblock pre,.listingblock>.content>pre:not(.highlight),.listingblock>.content>pre[class=highlight],.listingblock>.content>pre[class^="highlight "]{background:#f7f7f8}
+.literalblock.output pre{color:#f7f7f8;background:rgba(0,0,0,.9)}
+.listingblock>.content{position:relative}
+.listingblock code[data-lang]::before{display:none;content:attr(data-lang);position:absolute;font-size:.75em;top:.425rem;right:.5rem;line-height:1;text-transform:uppercase;color:inherit;opacity:.5}
+.listingblock:hover code[data-lang]::before{display:block}
+.listingblock.terminal pre .command::before{content:attr(data-prompt);padding-right:.5em;color:inherit;opacity:.5}
+.listingblock.terminal pre .command:not([data-prompt])::before{content:"$"}
+.listingblock pre.highlightjs{padding:0}
+.listingblock pre.highlightjs>code{padding:1em;border-radius:4px}
+.listingblock pre.prettyprint{border-width:0}
+.prettyprint{background:#f7f7f8}
+pre.prettyprint .linenums{line-height:1.45;margin-left:2em}
+pre.prettyprint li{background:none;list-style-type:inherit;padding-left:0}
+pre.prettyprint li code[data-lang]::before{opacity:1}
+pre.prettyprint li:not(:first-child) code[data-lang]::before{display:none}
+table.linenotable{border-collapse:separate;border:0;margin-bottom:0;background:none}
+table.linenotable td[class]{color:inherit;vertical-align:top;padding:0;line-height:inherit;white-space:normal}
+table.linenotable td.code{padding-left:.75em}
+table.linenotable td.linenos,pre.pygments .linenos{border-right:1px solid;opacity:.35;padding-right:.5em;-webkit-user-select:none;-moz-user-select:none;-ms-user-select:none;user-select:none}
+pre.pygments span.linenos{display:inline-block;margin-right:.75em}
+.quoteblock{margin:0 1em 1.25em 1.5em;display:table}
+.quoteblock:not(.excerpt)>.title{margin-left:-1.5em;margin-bottom:.75em}
+.quoteblock blockquote,.quoteblock p{color:rgba(0,0,0,.85);font-size:1.15rem;line-height:1.75;word-spacing:.1em;letter-spacing:0;font-style:italic;text-align:justify}
+.quoteblock blockquote{margin:0;padding:0;border:0}
+.quoteblock blockquote::before{content:"\201c";float:left;font-size:2.75em;font-weight:bold;line-height:.6em;margin-left:-.6em;color:#7a2518;text-shadow:0 1px 2px rgba(0,0,0,.1)}
+.quoteblock blockquote>.paragraph:last-child p{margin-bottom:0}
+.quoteblock .attribution{margin-top:.75em;margin-right:.5ex;text-align:right}
+.verseblock{margin:0 1em 1.25em}
+.verseblock pre{font-family:"Open Sans","DejaVu Sans",sans-serif;font-size:1.15rem;color:rgba(0,0,0,.85);font-weight:300;text-rendering:optimizeLegibility}
+.verseblock pre strong{font-weight:400}
+.verseblock .attribution{margin-top:1.25rem;margin-left:.5ex}
+.quoteblock .attribution,.verseblock .attribution{font-size:.9375em;line-height:1.45;font-style:italic}
+.quoteblock .attribution br,.verseblock .attribution br{display:none}
+.quoteblock .attribution cite,.verseblock .attribution cite{display:block;letter-spacing:-.025em;color:rgba(0,0,0,.6)}
+.quoteblock.abstract blockquote::before,.quoteblock.excerpt blockquote::before,.quoteblock .quoteblock blockquote::before{display:none}
+.quoteblock.abstract blockquote,.quoteblock.abstract p,.quoteblock.excerpt blockquote,.quoteblock.excerpt p,.quoteblock .quoteblock blockquote,.quoteblock .quoteblock p{line-height:1.6;word-spacing:0}
+.quoteblock.abstract{margin:0 1em 1.25em;display:block}
+.quoteblock.abstract>.title{margin:0 0 .375em;font-size:1.15em;text-align:center}
+.quoteblock.excerpt>blockquote,.quoteblock .quoteblock{padding:0 0 .25em 1em;border-left:.25em solid #dddddf}
+.quoteblock.excerpt,.quoteblock .quoteblock{margin-left:0}
+.quoteblock.excerpt blockquote,.quoteblock.excerpt p,.quoteblock .quoteblock blockquote,.quoteblock .quoteblock p{color:inherit;font-size:1.0625rem}
+.quoteblock.excerpt .attribution,.quoteblock .quoteblock .attribution{color:inherit;font-size:.85rem;text-align:left;margin-right:0}
+p.tableblock:last-child{margin-bottom:0}
+td.tableblock>.content{margin-bottom:1.25em;word-wrap:anywhere}
+td.tableblock>.content>:last-child{margin-bottom:-1.25em}
+table.tableblock,th.tableblock,td.tableblock{border:0 solid #dedede}
+table.grid-all>*>tr>*{border-width:1px}
+table.grid-cols>*>tr>*{border-width:0 1px}
+table.grid-rows>*>tr>*{border-width:1px 0}
+table.frame-all{border-width:1px}
+table.frame-ends{border-width:1px 0}
+table.frame-sides{border-width:0 1px}
+table.frame-none>colgroup+*>:first-child>*,table.frame-sides>colgroup+*>:first-child>*{border-top-width:0}
+table.frame-none>:last-child>:last-child>*,table.frame-sides>:last-child>:last-child>*{border-bottom-width:0}
+table.frame-none>*>tr>:first-child,table.frame-ends>*>tr>:first-child{border-left-width:0}
+table.frame-none>*>tr>:last-child,table.frame-ends>*>tr>:last-child{border-right-width:0}
+table.stripes-all>*>tr,table.stripes-odd>*>tr:nth-of-type(odd),table.stripes-even>*>tr:nth-of-type(even),table.stripes-hover>*>tr:hover{background:#f8f8f7}
+th.halign-left,td.halign-left{text-align:left}
+th.halign-right,td.halign-right{text-align:right}
+th.halign-center,td.halign-center{text-align:center}
+th.valign-top,td.valign-top{vertical-align:top}
+th.valign-bottom,td.valign-bottom{vertical-align:bottom}
+th.valign-middle,td.valign-middle{vertical-align:middle}
+table thead th,table tfoot th{font-weight:bold}
+tbody tr th{background:#f7f8f7}
+tbody tr th,tbody tr th p,tfoot tr th,tfoot tr th p{color:rgba(0,0,0,.8);font-weight:bold}
+p.tableblock>code:only-child{background:none;padding:0}
+p.tableblock{font-size:1em}
+ol{margin-left:1.75em}
+ul li ol{margin-left:1.5em}
+dl dd{margin-left:1.125em}
+dl dd:last-child,dl dd:last-child>:last-child{margin-bottom:0}
+li p,ul dd,ol dd,.olist .olist,.ulist .ulist,.ulist .olist,.olist .ulist{margin-bottom:.625em}
+ul.checklist,ul.none,ol.none,ul.no-bullet,ol.no-bullet,ol.unnumbered,ul.unstyled,ol.unstyled{list-style-type:none}
+ul.no-bullet,ol.no-bullet,ol.unnumbered{margin-left:.625em}
+ul.unstyled,ol.unstyled{margin-left:0}
+li>p:empty:only-child::before{content:"";display:inline-block}
+ul.checklist>li>p:first-child{margin-left:-1em}
+ul.checklist>li>p:first-child>.fa-square-o:first-child,ul.checklist>li>p:first-child>.fa-check-square-o:first-child{width:1.25em;font-size:.8em;position:relative;bottom:.125em}
+ul.checklist>li>p:first-child>input[type=checkbox]:first-child{margin-right:.25em}
+ul.inline{display:flex;flex-flow:row wrap;list-style:none;margin:0 0 .625em -1.25em}
+ul.inline>li{margin-left:1.25em}
+.unstyled dl dt{font-weight:400;font-style:normal}
+ol.arabic{list-style-type:decimal}
+ol.decimal{list-style-type:decimal-leading-zero}
+ol.loweralpha{list-style-type:lower-alpha}
+ol.upperalpha{list-style-type:upper-alpha}
+ol.lowerroman{list-style-type:lower-roman}
+ol.upperroman{list-style-type:upper-roman}
+ol.lowergreek{list-style-type:lower-greek}
+.hdlist>table,.colist>table{border:0;background:none}
+.hdlist>table>tbody>tr,.colist>table>tbody>tr{background:none}
+td.hdlist1,td.hdlist2{vertical-align:top;padding:0 .625em}
+td.hdlist1{font-weight:bold;padding-bottom:1.25em}
+td.hdlist2{word-wrap:anywhere}
+.literalblock+.colist,.listingblock+.colist{margin-top:-.5em}
+.colist td:not([class]):first-child{padding:.4em .75em 0;line-height:1;vertical-align:top}
+.colist td:not([class]):first-child img{max-width:none}
+.colist td:not([class]):last-child{padding:.25em 0}
+.thumb,.th{line-height:0;display:inline-block;border:4px solid #fff;box-shadow:0 0 0 1px #ddd}
+.imageblock.left{margin:.25em .625em 1.25em 0}
+.imageblock.right{margin:.25em 0 1.25em .625em}
+.imageblock>.title{margin-bottom:0}
+.imageblock.thumb,.imageblock.th{border-width:6px}
+.imageblock.thumb>.title,.imageblock.th>.title{padding:0 .125em}
+.image.left,.image.right{margin-top:.25em;margin-bottom:.25em;display:inline-block;line-height:0}
+.image.left{margin-right:.625em}
+.image.right{margin-left:.625em}
+a.image{text-decoration:none;display:inline-block}
+a.image object{pointer-events:none}
+sup.footnote,sup.footnoteref{font-size:.875em;position:static;vertical-align:super}
+sup.footnote a,sup.footnoteref a{text-decoration:none}
+sup.footnote a:active,sup.footnoteref a:active{text-decoration:underline}
+#footnotes{padding-top:.75em;padding-bottom:.75em;margin-bottom:.625em}
+#footnotes hr{width:20%;min-width:6.25em;margin:-.25em 0 .75em;border-width:1px 0 0}
+#footnotes .footnote{padding:0 .375em 0 .225em;line-height:1.3334;font-size:.875em;margin-left:1.2em;margin-bottom:.2em}
+#footnotes .footnote a:first-of-type{font-weight:bold;text-decoration:none;margin-left:-1.05em}
+#footnotes .footnote:last-of-type{margin-bottom:0}
+#content #footnotes{margin-top:-.625em;margin-bottom:0;padding:.75em 0}
+div.unbreakable{page-break-inside:avoid}
+.big{font-size:larger}
+.small{font-size:smaller}
+.underline{text-decoration:underline}
+.overline{text-decoration:overline}
+.line-through{text-decoration:line-through}
+.aqua{color:#00bfbf}
+.aqua-background{background:#00fafa}
+.black{color:#000}
+.black-background{background:#000}
+.blue{color:#0000bf}
+.blue-background{background:#0000fa}
+.fuchsia{color:#bf00bf}
+.fuchsia-background{background:#fa00fa}
+.gray{color:#606060}
+.gray-background{background:#7d7d7d}
+.green{color:#006000}
+.green-background{background:#007d00}
+.lime{color:#00bf00}
+.lime-background{background:#00fa00}
+.maroon{color:#600000}
+.maroon-background{background:#7d0000}
+.navy{color:#000060}
+.navy-background{background:#00007d}
+.olive{color:#606000}
+.olive-background{background:#7d7d00}
+.purple{color:#600060}
+.purple-background{background:#7d007d}
+.red{color:#bf0000}
+.red-background{background:#fa0000}
+.silver{color:#909090}
+.silver-background{background:#bcbcbc}
+.teal{color:#006060}
+.teal-background{background:#007d7d}
+.white{color:#bfbfbf}
+.white-background{background:#fafafa}
+.yellow{color:#bfbf00}
+.yellow-background{background:#fafa00}
+span.icon>.fa{cursor:default}
+a span.icon>.fa{cursor:inherit}
+.admonitionblock td.icon [class^="fa icon-"]{font-size:2.5em;text-shadow:1px 1px 2px rgba(0,0,0,.5);cursor:default}
+.admonitionblock td.icon .icon-note::before{content:"\f05a";color:#19407c}
+.admonitionblock td.icon .icon-tip::before{content:"\f0eb";text-shadow:1px 1px 2px rgba(155,155,0,.8);color:#111}
+.admonitionblock td.icon .icon-warning::before{content:"\f071";color:#bf6900}
+.admonitionblock td.icon .icon-caution::before{content:"\f06d";color:#bf3400}
+.admonitionblock td.icon .icon-important::before{content:"\f06a";color:#bf0000}
+.conum[data-value]{display:inline-block;color:#fff!important;background:rgba(0,0,0,.8);border-radius:50%;text-align:center;font-size:.75em;width:1.67em;height:1.67em;line-height:1.67em;font-family:"Open Sans","DejaVu Sans",sans-serif;font-style:normal;font-weight:bold}
+.conum[data-value] *{color:#fff!important}
+.conum[data-value]+b{display:none}
+.conum[data-value]::after{content:attr(data-value)}
+pre .conum[data-value]{position:relative;top:-.125em}
+b.conum *{color:inherit!important}
+.conum:not([data-value]):empty{display:none}
+dt,th.tableblock,td.content,div.footnote{text-rendering:optimizeLegibility}
+h1,h2,p,td.content,span.alt,summary{letter-spacing:-.01em}
+p strong,td.content strong,div.footnote strong{letter-spacing:-.005em}
+p,blockquote,dt,td.content,span.alt,summary{font-size:1.0625rem}
+p{margin-bottom:1.25rem}
+.sidebarblock p,.sidebarblock dt,.sidebarblock td.content,p.tableblock{font-size:1em}
+.exampleblock>.content{background:#fffef7;border-color:#e0e0dc;box-shadow:0 1px 4px #e0e0dc}
+.print-only{display:none!important}
+@page{margin:1.25cm .75cm}
+@media print{*{box-shadow:none!important;text-shadow:none!important}
+html{font-size:80%}
+a{color:inherit!important;text-decoration:underline!important}
+a.bare,a[href^="#"],a[href^="mailto:"]{text-decoration:none!important}
+a[href^="http:"]:not(.bare)::after,a[href^="https:"]:not(.bare)::after{content:"(" attr(href) ")";display:inline-block;font-size:.875em;padding-left:.25em}
+abbr[title]{border-bottom:1px dotted}
+abbr[title]::after{content:" (" attr(title) ")"}
+pre,blockquote,tr,img,object,svg{page-break-inside:avoid}
+thead{display:table-header-group}
+svg{max-width:100%}
+p,blockquote,dt,td.content{font-size:1em;orphans:3;widows:3}
+h2,h3,#toctitle,.sidebarblock>.content>.title{page-break-after:avoid}
+#header,#content,#footnotes,#footer{max-width:none}
+#toc,.sidebarblock,.exampleblock>.content{background:none!important}
+#toc{border-bottom:1px solid #dddddf!important;padding-bottom:0!important}
+body.book #header{text-align:center}
+body.book #header>h1:first-child{border:0!important;margin:2.5em 0 1em}
+body.book #header .details{border:0!important;display:block;padding:0!important}
+body.book #header .details span:first-child{margin-left:0!important}
+body.book #header .details br{display:block}
+body.book #header .details br+span::before{content:none!important}
+body.book #toc{border:0!important;text-align:left!important;padding:0!important;margin:0!important}
+body.book #toc,body.book #preamble,body.book h1.sect0,body.book .sect1>h2{page-break-before:always}
+.listingblock code[data-lang]::before{display:block}
+#footer{padding:0 .9375em}
+.hide-on-print{display:none!important}
+.print-only{display:block!important}
+.hide-for-print{display:none!important}
+.show-for-print{display:inherit!important}}
+@media amzn-kf8,print{#header>h1:first-child{margin-top:1.25rem}
+.sect1{padding:0!important}
+.sect1+.sect1{border:0}
+#footer{background:none}
+#footer-text{color:rgba(0,0,0,.6);font-size:.9em}}
+@media amzn-kf8{#header,#content,#footnotes,#footer{padding:0}}
+</style>
+</head>
+<body class="article">
+<div id="header">
+</div>
+<div id="content">
+<div class="sect1">
+<h2 id="_qqq_tables">QQQ Tables</h2>
+<div class="sectionbody">
+<div class="paragraph">
+<p>The core type of object in a QQQ Instance is the Table.
+In the most common use-case, a QQQ Table may be the in-app representation of a Database table.
+That is, it is a collection of records (or rows) of data, each of which has a set of fields (or columns).</p>
+</div>
+<div class="paragraph">
+<p>QQQ also allows other types of data sources (<a href="Backends{relfilesuffix}">QQQ Backends</a>) to be used as tables, such as File systems, API&#8217;s, Java enums or objects, etc.
+All of these backend types present the same interfaces (both user-interfaces, and application programming interfaces), regardless of their backend type.</p>
+</div>
+<div class="sect2">
+<h3 id="_qtablemetadata">QTableMetaData</h3>
+<div class="paragraph">
+<p>Tables are defined in a QQQ Instance in a <code><strong>QTableMetaData</strong></code> object.
+All tables must reference a <a href="Backends{relfilesuffix}">QQQ Backend</a>, a list of fields that define the shape of records in the table, and additional data to describe how to work with the table within its backend.</p>
+</div>
+<div class="paragraph">
+<p><strong>QTableMetaData Properties:</strong></p>
+</div>
+<div class="ulist">
+<ul>
+<li>
+<p><code>name</code> - <strong>String, Required</strong> - Unique name for the table within the QQQ Instance.</p>
+</li>
+<li>
+<p><code>label</code> - <strong>String</strong> - User-facing label for the table, presented in User Interfaces.
+Inferred from <code>name</code> if not set.</p>
+</li>
+<li>
+<p><code>backendName</code> - <strong>String, Required</strong> - Name of a <a href="Backends{relfilesuffix}">QQQ Backend</a> in which this table&#8217;s data is managed.</p>
+</li>
+<li>
+<p><code>fields</code> - <strong>Map of String → <a href="Fields{relfilesuffix}">QQQ Field</a>, Required</strong> - The columns of data that make up all records in this table.</p>
+</li>
+<li>
+<p><code>primaryKeyField</code> - <strong>String, Conditional</strong> - Name of a <a href="Fields{relfilesuffix}">QQQ Field</a> that serves as the primary key (e.g., unique identifier) for records in this table.</p>
+</li>
+<li>
+<p><code>uniqueKeys</code> - <strong>List of UniqueKey</strong> - Definition of additional unique constraints (from an RDBMS point of view) from the table.
+e.g., sets of columns which must have unique values for each record in the table.</p>
+</li>
+<li>
+<p><code>backendDetails</code> - <strong>QTableBackendDetails or subclass</strong> - Additional data to configure the table within its <a href="Backends{relfilesuffix}">QQQ Backend</a>.</p>
+</li>
+<li>
+<p><code>automationDetails</code> - <strong>QTableAutomationDetails</strong> - Configuration of automated jobs that run against records in the table, e.g., upon insert or update.</p>
+</li>
+<li>
+<p><code>customizers</code> - <strong>Map of String → QCodeReference</strong> - References to custom code that are injected into standard table actions, that allow applications to customize certain parts of how the table works.</p>
+</li>
+<li>
+<p><code>parentAppName</code> - <strong>String</strong> - Name of a <a href="Apps{relfilesuffix}">QQQ App</a> that this table exists within.</p>
+</li>
+<li>
+<p><code>icon</code> - <strong>QIcon</strong> - Icon associated with this table in certain user interfaces.</p>
+</li>
+<li>
+<p><code>recordLabelFormat</code> - <strong>String</strong> - Java Format String, used with <code>recordLabelFields</code> to produce a label shown for records from the table.</p>
+</li>
+<li>
+<p><code>recordLabelFields</code> - <strong>List of String, Conditional</strong> - Used with <code>recordLabelFormat</code> to provide values for any format specifiers in the format string.
+These strings must be field names within the table.</p>
+<div class="ulist">
+<ul>
+<li>
+<p>Example of using <code>recordLabelFormat</code> and <code>recordLabelFields</code>:</p>
+</li>
+</ul>
+</div>
+</li>
+</ul>
+</div>
+<div class="listingblock">
+<div class="content">
+<pre class="CodeRay highlight"><code data-lang="java"><span style="color:#777">// given these fields in the table:</span>
+<span style="color:#080;font-weight:bold">new</span> QFieldMetaData(<span style="background-color:hsla(0,100%,50%,0.05)"><span style="color:#710">&quot;</span><span style="color:#D20">name</span><span style="color:#710">&quot;</span></span>, QFieldType.STRING)
+<span style="color:#080;font-weight:bold">new</span> QFieldMetaData(<span style="background-color:hsla(0,100%,50%,0.05)"><span style="color:#710">&quot;</span><span style="color:#D20">birthDate</span><span style="color:#710">&quot;</span></span>, QFieldType.DATE)
+
+<span style="color:#777">// We can produce a record label such as &quot;Darin Kelkhoff (1980-05-31)&quot; via:</span>
+.withRecordLabelFormat(<span style="background-color:hsla(0,100%,50%,0.05)"><span style="color:#710">&quot;</span><span style="color:#D20">%s (%s)</span><span style="color:#710">&quot;</span></span>)
+.withRecordLabelFields(<span style="color:#0a8;font-weight:bold">List</span>.of(<span style="background-color:hsla(0,100%,50%,0.05)"><span style="color:#710">&quot;</span><span style="color:#D20">name</span><span style="color:#710">&quot;</span></span>, <span style="background-color:hsla(0,100%,50%,0.05)"><span style="color:#710">&quot;</span><span style="color:#D20">birthDate</span><span style="color:#710">&quot;</span></span>))</code></pre>
+</div>
+</div>
+<div class="ulist">
+<ul>
+<li>
+<p><code>sections</code> - <strong>List of QFieldSection</strong> - Mechanism to organize fields within user interfaces, into logical sections.
+If any sections are present in the table meta data, then all fields in the table must be listed in exactly 1 section.
+If no sections are defined, then instance enrichment will define default sections.</p>
+</li>
+<li>
+<p><code>associatedScripts</code> - <strong>List of AssociatedScript</strong> - Definition of user-defined scripts that can be associated with records within the table.</p>
+</li>
+<li>
+<p><code>enabledCapabilities</code> and <code>disabledCapabilities</code> - <strong>Set of Capability enum values</strong> - Overrides from the backend level, for capabilities that this table does or does not possess.</p>
+</li>
+</ul>
+</div>
+</div>
+</div>
+</div>
+</div>
+<div id="footer">
+<div id="footer-text">
+Last updated 2022-11-21 09:02:56 -0600
+</div>
+</div>
+</body>
+</html>

docs/metaData/Widgets.adoc

@@ -2,50 +2,16 @@
 == Widgets
 include::../variables.adoc[]
 
-Widgets are the most customizable UI components in QQQ.
-They can be used either on App Home Screens (e.g., as Dashboard screens),
-or they can be included into Record View screens.
-
-QQQ defines several types of widgets, such as charts (pie, bar, line),
-numeric displays, application-populated tables, or even fully custom HTML.
+#TODO#
 
 === QWidgetMetaData
 A Widget is defined in a QQQ Instance in a `*QWidgetMetaData*` object.
 
+#TODO#
+
 *QWidgetMetaData Properties:*
 
 * `name` - *String, Required* - Unique name for the widget within the QQQ Instance.
-* `type` - *String, Required* - Specifies the UI & data type for the widget.
-* `label` - *String* - User-facing header or title for a widget.
-* `tooltip` - *String* - Text contents to be placed in a tooltip associated with the widget's label in the UI.
-** Values should come from the `WidgetType` enum's `getType()` method (e.g., `WidgetType.BAR_CHART.getType()`)
-* `gridColumns` - *Integer* - for a desktop-sized screen, in a 12-based grid,
-how many columns the widget should consume.
-* `codeReference` - *QCodeReference, Required* - Reference to the custom code,
-a subclass of `AbstractWidgetRenderer`, which is responsible for loading data to render the widget.
-* `footerHTML` - *String* - HTML String, which, if present, will be displayed in the
-footer of the widget (not supported by all widget types).
-* `isCard` - *boolean, default false* #TODO#
-* `showReloadButton` - *boolean, default true* #TODO#
-* `showExportButton` - *boolean, default false* #TODO#
-* `dropdowns` - #TODO#
-* `storeDropdownSelections` - *boolean* #TODO#
-* `icons` - *Map<String, QIcon>* #TODO#
-* `defaultValues` - *Map<String, Serializable>* #TODO#
 
-There are also some subclasses of `QWidgetMetaData`, for some specific widget types:
-
-*ParentWidgetMetaData Properties:*
-
-* `title` - *String* #TODO - how does this differ from label?#
-* `childWidgetNameList` - *List<String>, Required*
-* `childProcessNameList` - *List<String>* #TODO appears unused - check, and delete#
-* `laytoutType` - *enum of GRID or TABS, default GRID*
-
-*QNoCodeWidgetMetaData Properties:*
-
-* `values` - *List<AbstractWidgetValueSource>* #TODO#
-* `outputs` - *List<AbstractWidgetOutput>* #TODO#
-
-#TODO - Examples#
+#TODO#
 

docs/misc/ProcessBackendSteps.adoc

@@ -1,155 +0,0 @@
-== Process Backend Steps
-include::../variables.adoc[]
-
-In many QQQ applications, much of the code that engineers write will take the form of Backend Steps for {link-processes}.
-Such code is defined in classes which implement the interface `BackendStep`.  
-This interface defines only a single method:
-[source,java]
-.BackendStep.java
-----
-public interface BackendStep
-{
-   /*******************************************************************************
-    ** Execute the backend step - using the request as input, and the result as output.
-    **
-    *******************************************************************************/
-   void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException;
-}
-
-----
-
-Process backend steps have access to state information - specifically, a list of records, and a map of name=value pairs - in the input & output objects.
-This state data is persisted by QQQ between steps (e.g., if a frontend step is presented to a user between backend steps).
-
-
-=== RunBackendStepInput
-All input data to the step is available in the `RunBackendStepInput` object.
-Key methods in this class are:
-
-* `getRecords()` - Returns the List of <<QRecords>> that are currently being acted on in the process.
-* `getValues()` - Returns a Map of String -> Serializable; name=value pairs that are the state-data of the process.
-** Values can be added to this state from a process's meta-data, from a screen, or from another backend step.
-* `getValue(String fieldName)` - Returns a specific value, by name, from the process's state.
-** This method has several variations that return the value as a specific type, such as `getValueString`, `getValueInteger`, `getValueBoolean`...
-* `getAsyncJobCallback()` - Accessor for an `AsyncJobCallback` object, which provides a way for the process backend step to communicate about its status or progress with a user running the process in a frontend.
-Provides methods:
-** `updateStatus(String message)` - Where general status messages can be given.
-For example, `"Loading census data"`
-** `updateStatus(int current, int total)` - For updating a progress meter.
-e.g., "47 of 1701" would be display by calling `.updateStatus(47, 1701)`
-* `getFrontendStepBehavior()` - Enum, indicating what should happen when a frontend step is encountered as the process's next step to run.
-Possible values are:
-** `BREAK` - Indicates that the process's execution should be suspended, so that the screen represented by the frontend step can be presented to a user.
-This would be the expected behavior if a process is being run by a user from a UI.
-** `SKIP` - Indicates that frontend steps should be skipped.
-This would be the expected behavior if a process is running from a scheduled job (without a user present to drive it), for example.
-** `FAIL` - Indicates that the process should end with an exception if a frontend step is encountered.
-** A backend step may want to act differently based on its frontendStepBehavior.
-For example, additional data may be looked up for displaying to a user if the behavior is `BREAK`.
-* `getBasepullLastRunTime()` - For <<BasepullConfiguration,Basepull>> processes, this is the `Instant` stored in the basepull table as the process's last run time.
-
-=== RunBackendStepOutput
-All output from a process step should be placed in its `RunBackendStepOutput` object (and/or stored to a backend, as appropriate).
-Key methods in this class are:
-
-* `addValue(String fieldName, Serializable value)` - Adds a single named value to the process's state, overwriting it the value if it already exists.
-* `addRecord(QRecord record)` - Add a `<<QRecord>>` to the process's output.
-* `addAuditSingleInput(AuditSingleInput auditSingleInput)` - Add a new entry to the process's list of audit inputs, to be stored at the completion of the process.
-** An `AuditSingleInput` object can most easily be built with the constructor: `AuditSingleInput(String tableName, QRecord record, String auditMessage)`.
-** Additional audit details messages (sub-bullets that accompany the primary `auditMessage`) can be added to an `AuditSingleInput` via the `addDetail(String message)` method.
-** _Note that at this time, the automatic storing of these audits is only provided by the execute step of a StreamedETLWithFrontendProcesses._
-
-=== Example
-
-[source,java]
-.Example of a BackendStep
-----
-/*******************************************************************************
- ** For the "person" table's "Add Age" process -
- ** For each input person record, add the specified yearsToAdd to their age.
- *******************************************************************************/
-public class AddAge implements BackendStep
-{
-   /*******************************************************************************
-    **
-    *******************************************************************************/
-   @Override
-   public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput)
-   {
-      /////////////////////////////////////////////////////////////////
-      // get the yearsToAdd input field value from the process input //
-      /////////////////////////////////////////////////////////////////
-      Integer yearsToAdd = runBackendStepInput.getValueInteger("yearsToAdd");
-      int totalYearsAdded = 0;
-
-      ///////////////////////////////////////////////////
-      // loop over the records passed into the process //
-      ///////////////////////////////////////////////////
-      for(QRecord record : runBackendStepInput.getRecords())
-      {
-         Integer age = record.getValueInteger("age");
-         age += yearsToAdd;
-         totalYearsAdded += yearsToAdd;
-
-         ////////////////////////////////////////////////////////////////////////////////////////////
-         // update the record with the new "age" value.                                            //
-         // note that this update record object will implicitly be available to the process's next //
-         // backend step, via the sharing of the processState object.                              //
-         ////////////////////////////////////////////////////////////////////////////////////////////
-         record.setValue("age", age);
-      }
-
-      /////////////////////////////////////////
-      // set an output value for the process //
-      /////////////////////////////////////////
-      runBackendStepOutput.addValue("totalYearsAdded", totalYearsAdded);
-   }
-}
-
-----
-
-
-=== Backend Steps for StreamedETLWithFrontendProcesses
-
-For <<StreamedETLWithFrontendProcess>> type processes, backend steps are defined a little bit differently than they are for other process types.
-In this type of process, the process meta-data defines 3 backend steps which are built-in to QQQ, and which do not have any custom application logic.
-These steps are:
-
-* `StreamedETLPreviewStep`
-* `StreamedETLValidateStep`
-* `StreamedETLExecuteStep`
-
-For custom application logic to be implemented in a StreamedETLWithFrontendProcesses, an application engineer must define (up to) 3 backend step classes which are loaded by the steps listed above.
-These application-defined steps must extend specific classes (which themselves implement the `BackendStep` interface), to provide the needed logic of this style of process.
-These steps are:
-
-* *Extract* - a subclass of `AbstractExtractStep` - is responsible for Extracting records from the source table.
-** For this step, we can often use the QQQ-provided class `ExtractViaQueryStep`, or sometimes a subclass of it.
-** The Extract step is called before the Preview, Validate, and Result screens, though for the Preview screen, it is set to only extract a small number of records (10).
-* *Transform* - a subclass of `AbstractTransformStep` - is responsible for applying the majority of the business logic of the process.
-In ETL terminology, this is the "Transform" action - which means applying some type of logical transformation an input record (found by the Extract step) to generate an output record (stored by the Load step).
-** A Transform step's `run` method will be called, potentially, multiple times, each time with a page of records in the `runBackendStepInput` parameter.
-** This method is responsible for adding records to the `runBackendStepOutput`, which will then be passed to the *Load* step.
-** This class is also responsible for implementing the method `getProcessSummary`, which provides the data to the *Validate* screen.
-** The run method will generally update ProcessSummaryLine objects to facilitate this functionality.
-** The Transform step is called before the Preview, Validate, and Result screens, consuming all records selected by the Extract step.
-* *Load* - a subclass of `AbstractLoadStep` - is responsible for the Load function of the ETL job.
-_A quick word on terminology - this step is actually doing what we are more likely to think of as storing data - which feels like the opposite of “loading” - but we use the name Load to keep in line with the ETL naming convention…_
-** The Load step is ONLY called before the Result screen is presented (possibly after Preview, if the user chose to skip validation, otherwise, after validation).
-** Similar to the Transform step, the Load step's `run` method will be called potentially multiple times, with pages of records in its input.
-** As such, the Load step is generally the only step where data writes should occur.
-*** e.g., a Transform step should not do any writes, as it will be called when the user is going to the Preview & Validate screens - e.g., before the user confirmed that they want to execute the action!
-** A common pattern is that the Load step just needs to insert or update the list of records output by the Transform step, in which case the QQQ-provided `LoadViaInsertStep` or `LoadViaUpdateStep` can be used, but custom use-cases can be built as well.
-
-Another distinction between StreamedELTWithFrontendProcess steps and general QQQ process backend steps, is that the list of records in the input & output objects is NOT shared for StreamedELTWithFrontendProcess steps.
-The direct implication of this is, that a Transform step MUST explicitly call `output.addRecord()` for any records that it wants to pass along to the Load step.
-
-==== Example
-
-[source,java]
-.Examples of a Transform and Load step for a StreamedELTWithFrontendProcess
-----
-   // todo!
-----
-
-#todo: more details on these 3 specialized types of process steps (e.g., method to overload, when stuff like pre-action is called; how summaries work).#

docs/misc/QContext.adoc

@@ -1,30 +0,0 @@
-== QContext
-include::../variables.adoc[]
-
-The class `QContext` contains a collection of thread-local variables, to define the current context of the QQQ code that is currently running.
-For example, what `QInstance` (meta-data container) is being used, what `QSession` (user attributes) is active, etc.
-
-Most of the time, main-line application code does not need to worry about setting up the `QContext` - although unit-test code can is a common exception to that rule.
-This is because all of QQQ's entry-points into execution (e.g., web context handlers, CLI startup methods, schedulers, and multi-threaded executors) take care of initializing the context.
-
-It is more common though, for application code to need to get data from the context, such as the current session or any piece of meta-data from the QInstance.
-The methods to access data from the `QContext` are fairly straightforward:
-
-=== Examples
-==== Get a QTableMetaData from the active QInstance
-[source,java]
-----
-QTableMeataData myTable = QContext.getQInstance().getTable("myTable");
-for(QFieldMeataData field : myTable.getFields().values())
-{
-   // ...
-}
-----
-
-==== Get a security key value for the current user session
-[source,java]
-----
-QSession           session   = QContext.getQSession();
-List<Serializable> clientIds = session.getSecurityKeyValues("clientId");
-----
-

docs/misc/QRecordEntities.adoc

@@ -1,116 +0,0 @@
-== QRecordEntities
-include::../variables.adoc[]
-
-While `<<QRecords>>` provide a flexible mechanism for passing around record data in a QQQ Application, they have one big disadvantage from the point-of-view of a Java Application:
-They do not provide a mechanism to ensure compile-time checks of field names or field types.
-
-As such, an alternative mechanism exists, which allows records in a QQQ application to be worked with following a more familiar Java Bean (Entity Bean) like pattern.
-This mechanism is known as a `QRecordEntity`.
-
-Specifically speaking, `QRecordEntity` is an abstract base class, whose purpose is to be the base class for entity-bean classes.
-Using reflection, `QRecordEntity` is able to convert objects back and forth from `QRecord` to specific entity-bean subtypes.
-
-For example, the method `QRecordEntity::toQRecord()` converts a record entity object into a `QRecord`.
-
-Inversely, `QRecordEntity::populateFromQRecord(QRecord record)` sets fields in a record entity object, based on the values in the supplied `QRecord`.
-It is conventional for a subclass of `QRecordEntity` to have both a no-arg (default) constructor, and a constructor that takes a `QRecord` as a parameter, and calls `populateFromQRecord`.
-
-In addition to these constructors, a `QRecordEntity` subclass will generally contain:
-
-* A `public static final String TABLE_NAME`, used throughout the application as a constant reference to the name for the {link-table}.
-* A series of `private` fields, corresponding to the fields in the table that the entity represents.
-** If these fields are annotated as `@QField()`, then the {link-table} meta-data for the table that the entity represents can in part be inferred by QQQ, by using the method `QTableMetaData::withFieldsFromEntity`.
-* `getX()`, `setX()`, and `withX()` methods for all of the entity's fields.
-
-=== Examples
-[source,java]
-.Example Definition of a QRecordEntity subclass:  Person.java
-----
-/*******************************************************************************
- ** QRecordEntity for the person table.
- *******************************************************************************/
-public class Person extends QRecordEntity
-{
-   public static final String TABLE_NAME = "person";
-
-   @QField(isEditable = false)
-   private Integer id;
-
-   @QField()
-   private String firstName;
-
-   @QField()
-   private String lastName;
-
-   @QField()
-   private Integer age;
-
-   // all other fields
-
-   /*******************************************************************************
-    ** Default constructor
-    *******************************************************************************/
-   public Person()
-   {
-   }
-
-
-   /*******************************************************************************
-    ** Constructor that takes a QRecord
-    *******************************************************************************/
-   public Person(QRecord record)
-   {
-      populateFromQRecord(record);
-   }
-
-
-   /*******************************************************************************
-    ** Custom method to concatenate firstName and lastName
-    *******************************************************************************/
-   public String getFullName()
-   {
-      //////////////////////////////////////////////////////////////////////
-      // if there were more than an example, we could do some null-checks //
-      // here to avoid silly output like "Bobby null" :)                  //
-      //////////////////////////////////////////////////////////////////////
-      return (firstName + " " + lastName);
-   }
-
-   // all getter, setter, and fluent setter (wither) methods
-}
-----
-
-The core ORM actions and process-execution actions of QQQ work with `QRecords`.
-However, application engineers may want to apply patterns like the following example, to gain the compile-time safety of `QRecordEntities`:
-
-[source,java]
-.Example Usage of a QRecordEntity
-----
-//////////////////////////////////////////////////////////////////////
-// assume we're working with the "person" table & entity from above //
-//////////////////////////////////////////////////////////////////////
-List<QRecord> recordsToUpdate = new ArrayList<>();
-for(QRecord record : inputRecordList)
-{
-   /////////////////////////////////////////////////////////////////////////////
-   // call the constructor that copies values from the record into the entity //
-   /////////////////////////////////////////////////////////////////////////////
-   Person person = new Person(record);
-
-   ////////////////////////////////////////////////
-   // call a custom method defined in the entity //
-   ////////////////////////////////////////////////
-   LOG.info("Processing: " + person.getFullName());
-
-   /////////////////////////////////////////////////////////////
-   // age is an Integer, so no type-conversion is needed here //
-   /////////////////////////////////////////////////////////////
-   person.setAge(person.getAge() + 1);
-
-   ///////////////////////////////////////////////////////////////////////////
-   // to pass the updated records to UpdateAction, convert them to QRecords //
-   ///////////////////////////////////////////////////////////////////////////
-   recordsToUpdate.add(person.toQRecord());
-}
-----
-

docs/misc/QRecords.adoc

@@ -1,68 +0,0 @@
-== QRecords
-include::../variables.adoc[]
-
-Almost all code inside a QQQ application will be dealing with *Records* (aka Tuples or Rows).
-That is: a collection of named values, representing a single Entity, Fact, or, Row from a {link-table}.
-
-The class that QQQ uses to work with records is called:  `QRecord`.
-
-=== Values
-At its core, a `QRecord` is a wrapper around a `Map<String, Serializable> values`.
-These are the *actual* values for the fields in the table for the record.
-That is, direct representations of the values as they are stored in the {link-backend}.
-
-The keys in the `values` map are names from the {link-fields} in the {link-table}.
-
-The values in `values` map are declared as `Serializable` (to help ensure the serializability of the `QRecord` as a whole).
-In practice, their types will be based on the `QFieldType` of the {link-field} that they correspond to.
-That will typically be one of: `String`, `Integer`, `Boolean`, `BigDecimal`, `Instant`, `LocalDate`, `LocalTime`, or `byte[]`.
-Be aware that `null` values may be in the `values` map, especially/per if the backend/table support `null`.
-
-To work with the `values` map, the following methods are provided:
-
-* `setValue(String fieldName, Serializable value)` - Sets a value for the specified field in the record.
-** Overloaded as `setValue(String fieldName, Object value)` - For some cases where the value may not be known to be `Serializable`.
-In this overload, if the value is `null` or `Serializable`, the primary version of `setValue` is called.
-Otherwise, the `value` is passed through `String::valueOf`, and the result is stored.
-** Overloaded as `setValue(QFieldMetaData field, Serializable value)` - which simply defers to the primary version of `setValue`, passing `field.getName()` as the first parameter.
-* `removeValue(String fieldName)` - Remove the given field from the `values` map.
-** Note that in some situations this is an important distinction from having a `null` value in the map (See <<UpdateAction)>>).
-* `setValues(Map<String, Serializable> values)` - Sets the full map of `values`.
-* `getValues()` - Returns the full map of `values`.
-* `getValue(String fieldName)` - Returns the value for the named field - possibly `null` - as a `Serializable`.
-* Several type-specific variations of `getValueX(String fieldName)`, where internally, values will be not exactly type-cast, but effectively converted (if possible) to the requested type.
-These conversions are done using the `ValueUtils.getValueAsX(Object)` methods.
-These methods are generally the preferred/cleanest way to get record values in application code, when it is needed in a type-specific way .
-** `getValueString(String fieldName)`
-** `getValueInteger(String fieldName)`
-** `getValueBoolean(String fieldName)`
-** `getValueBigDecimal(String fieldName)`
-** `getValueInstant(String fieldName)`
-** `getValueLocalDate(String fieldName)`
-** `getValueLocalTime(String fieldName)`
-** `getValueByteArray(String fieldName)`
-
-=== Display Values
-In addition to the `values` map, a `QRecord` contains another map called `displayValues`, which only stores `String` values.
-That is to say, values for other types are stringified, based on their {link-field}'s type and `displayFormat` property.
-In addition, fields which have a `possibleValueSource` property will have their translated values set in the `displayValues` map.
-
-By default, a `QRecord` will not have its `displayValues` populated.
-To populate `displayValues`, the <<QueryAction>> and <<GetAction>> classes take a property in their inputs called `shouldGenerateDisplayValues`, which must be set to `true` to generate `displayValues`.
-In addition, these two actions also have a property `shouldTranslatePossibleValues` in their inputs, which needs to be set to `true` if possible-value lookups are to be performed.
-
-As an alternative to the `shouldGenerateDisplayValues` and `shouldTranslatePossibleValues` inputs to <<QueryAction>> and <<GetAction>>, one can directly call the `QValueFormatter.setDisplayValuesInRecords` and/or `qPossibleValueTranslator.translatePossibleValuesInRecords` methods.
-Or, for special cases, `setDisplayValue(String fieldName, String displayValue)` or `setDisplayValues(Map<String, String> displayValues)` can be called directly.
-
-=== Backend Details
-Sometimes a backend may want to place additional data in a `QRecord` that doesn't exactly correspond to a field.
-To do this, the `Map<String, Serializable> backendDetails` member is used.
-
-For example, an API backend may store the full JSON `String` that came from the API as a backend detail in a `QRecord`.
-Or fields that are marked as `isHeavy`, if the full (heavy) value of the field hasn't been fetched, then the lengths of any such heavy fields may be stored in `backendDetails`.
-
-=== Errors and Warnings
-#todo#
-
-=== Associated Records
-#todo#

docs/misc/RenderingWidgets.adoc

@@ -1,224 +0,0 @@
-== Rendering Widgets
-include::../variables.adoc[]
-
-=== WidgetRenderer classes
-In general, to fully implement a Widget, you must define its `QWidgetMetaData`,
-and supply a subclass of `AbstractWidgetRenderer`, to provide the data to the widget.
-(Note the "No Code" category of widgets, which are an exception to this generalization).
-
-The only method required in a subclass of `AbstractWidgetRenderer` is:
-
-   public RenderWidgetOutput render(RenderWidgetInput input) throws QException
-
-The fields available in `RenderWidgetInput` are:
-
-- `Map<String, String> queryParams` - These are parameters supplied by the frontend, for example,
-if a user selected values from dropdowns to control a dimension of your widget, those name/value
-pairs would be in this map.  Similarly, if your widget is being included on a record view screen, then
-the record's primary key will be in this map.
-- `QWidgetMetaDataInterface widgetMetaData` - This is the meta-data for the widget being
-rendered.  This can be useful in case you are using the same renderer class for multiple widgets.
-
-The only field in `RenderWidgetOutput` is:
-
-- `QWidgetData widgetData` - This is a base class, with several attributes, and more importantly,
-several subclasses, specific to the type of widget that is being rendered.
-
-==== Widget-Type Specific Rendering Details
-Different widget types expect & require different types of values to be set in the `RenderWidgetOutput` by their renderers.
-
-===== Pie Chart
-The `WidgetType.PIE_CHART` requires an object of type `ChartData`.
-The fields on this type are:
-
-* `chartData` an instance of `ChartData.Data`, which has the following fields:
-** `labels` - *List<String>, required* - the labels for the slices of the pie.
-** `datasets` - *List<Dataset> required* - the data for each slice of the pie.
-For a Pie chart, only a single entry in this list is used
-(other chart types using `ChartData` may support more than 1 entry in this list).
-Fields in this object are:
-*** `label` - *String, required* - a label to describe the dataset as a whole.
-e.g., "Orders" for a pie showing orders of different statuses.
-*** `data` - *List<Number>, required* - the data points for each slice of the pie.
-*** `color` - *String* - HTML color for the slice
-*** `urls` - *List<String>* - Optional URLs for slices of the pie to link to when clicked.
-*** `backgroundColors` - *List<String>* - Optional HTML color codes for each slice of the pie.
-
-[source,java]
-.Pie chart widget example
-----
-// meta data
-new QWidgetMetaData()
-    .withName("pieChartExample")
-    .withType(WidgetType.PIE_CHART.getType())
-    .withGridColumns(4)
-    .withIsCard(true)
-    .withLabel("Pie Chart Example")
-    .withCodeReference(new QCodeReference(PieChartExampleRenderer.class));
-
-// renderer
-private List<String> labels = new ArrayList<>();
-private List<String> colors = new ArrayList<>();
-private List<Number> data   = new ArrayList<>();
-
-/*******************************************************************************
-** helper method - to add values for a slice to the lists
-*******************************************************************************/
-private void addSlice(String label, String color, Number datum)
-{
-    labels.add(label);
-    colors.add(color);
-    data.add(datum);
-}
-
-/*******************************************************************************
-** main method of the widget renderer
-*******************************************************************************/
-@Override
-public RenderWidgetOutput render(RenderWidgetInput input) throws QException
-{
-   addSlice("Apple", "#FF0000", 100);
-   addSlice("Orange", "#FF8000", 150);
-   addSlice("Banana", "#FFFF00", 75);
-   addSlice("Lime", "#00FF00", 100);
-   addSlice("Blueberry", "#0000FF", 200);
-
-   ChartData chartData = new ChartData()
-      .withChartData(new ChartData.Data()
-         .withLabels(labels)
-         .withDatasets(List.of(
-            new ChartData.Data.Dataset()
-               .withLabel("Flavor")
-               .withData(data)
-               .withBackgroundColors(colors)
-               .withUrls(urls))));
-
-   return (new RenderWidgetOutput(chartData));
-}
-
-----
-
-===== Bar Chart
-#todo#
-
-===== Stacked Bar Chart
-#todo#
-
-===== Horizontal Bar Chart
-#todo#
-
-===== Child Record List
-#todo#
-
-===== Line Chart
-#todo#
-
-===== Small Line Chart
-#todo#
-
-===== Statistics
-#todo#
-
-===== Parent Widget
-#todo#
-
-===== Composite
-A `WidgetType.COMPOSITE` is built by using one or more smaller elements, known as `Blocks`.
-Note that `Blocks` can also be used in the data of some other widget types
-(specifically, within a cell of a Table-type widget, or (in the future?) as a header above a pie or bar chart).
-
-A composite widget renderer must return data of type `CompositeWidgetData`,
-which has the following fields:
-
-* `blocks` - *List<AbstractBlockWidgetData>, required* - The blocks (1 or more) being composited together to make the widget.
-See below for details on the specific Block types.
-* `styleOverrides` - *Map<String, Serializable>* - Optional map of CSS attributes
-(named following javascriptStyleCamelCase) to apply to the `<div>` element that wraps the rendered blocks.
-* `layout` - *Layout enum* - Optional specifier for how the blocks should be laid out.
-e.g., predefined sets of CSS attributes to achieve specific layouts.
-** Note that some blocks are designed to work within composites with specific layouts.
-Look for matching names, such as `Layout.BADGES_WRAPPER` to go with `NumberIconBadgeBlock`.
-
-[source,java]
-.Composite widget example - consisting of 3 Progress Bar Blocks, and one Divider Block
-----
-// meta data
-new QWidgetMetaData()
-    .withName("compositeExample")
-    .withType(WidgetType.COMPOSITE.getType())
-    .withGridColumns(4)
-    .withIsCard(true)
-    .withLabel("Composite Example")
-    .withCodeReference(new QCodeReference(CompositeExampleRenderer.class));
-
-// renderer
-public RenderWidgetOutput render(RenderWidgetInput input) throws QException
-{
-   CompositeWidgetData data = new CompositeWidgetData();
-
-   data.addBlock(new ProgressBarBlockData()
-      .withValues(new ProgressBarValues()
-         .withHeading("Blocks")
-         .withPercent(new BigDecimal("78.5"))));
-
-   data.addBlock(new ProgressBarBlockData()
-      .withValues(new ProgressBarValues()
-         .withHeading("Progress")
-         .withPercent(new BigDecimal(0))));
-
-   data.addBlock(new DividerBlockData());
-
-   data.addBlock(new ProgressBarBlockData()
-      .withStyles(new ProgressBarStyles().withBarColor("#C0C000"))
-      .withValues(new ProgressBarValues()
-         .withHeading("Custom Color")
-         .withPercent(new BigDecimal("75.3"))));
-
-   return (new RenderWidgetOutput(data));
-}
-----
-
-
-===== Table
-#todo#
-
-===== HTML
-#todo#
-
-===== Divider
-#todo#
-
-===== Process
-#todo#
-
-===== Stepper
-#todo#
-
-===== Data Bag Viewer
-#todo#
-
-===== Script Viewer
-#todo#
-
-=== Block-type Specific Rendering Details
-For Composite-type widgets (or other widgets which can include blocks),
-there are specific data classes required to be returned by the widget renderer.
-
-Each block type defines a subclass of `AbstractBlockWidgetData`,
-which is a generic class with 3 type parameters:
-
-* `V` - an implementation of `BlockValuesInterface` - to define the type of values that the block uses.
-* `S` - an implementation of `BlockSlotsInterface` (expected to be an `enum`) - to define the "slots" in the block,
-that can have Tooltips and/or Links applied to them.
-* `SX` - an implementation of `BlockStylesInterface` - to define the types of style customizations that the block supports.
-
-These type parameters are designed to ensure type-safety for the application developer,
-to ensure that only
-
-=== Additional Tips
-
-* To make a Dashboard page (e.g., an App consisting of Widgets) with a parent widget use the parent widget's label as the page's label:
-** On the `QAppMetaData` that contains the Parent widget, call
-`.withSupplementalMetaData(new MaterialDashboardAppMetaData().withShowAppLabelOnHomeScreen(false))`.
-** In the Parent widget's renderer, on the `ParentWidgetData`, call `setLabel("My Label")` and
-`setIsLabelPageTitle(true)`.

docs/misc/ScheduledJobs.adoc

@@ -1,141 +0,0 @@
-== Schedulers and Scheduled Jobs
-include::../variables.adoc[]
-
-QQQ has the ability to automatically run various types of jobs on schedules,
-either defined in your instance's meta-data,
-or optionally via data in your application, in a `scheduledJob` table.
-
-=== Schedulers and QSchedulerMetaData
-2 types of schedulers are included in QQQ by default (though an application can define its own schedulers):
-
-* `SimpleScheduler` - is (as its name suggests) a simple class which uses java's `ScheduledExecutorService`
-to run jobs on repeating intervals.
-** Cannot run cron schedules - only repeating intervals.
-** If multiple servers are running, each will potentially run the same job concurrently
-** Has no configurations, e.g., to limit the number of threads.
-
-* `QuartzScheduler` - uses the 3rd party https://www.quartz-scheduler.org/[Quartz Scheduler] library to provide
-a much more capable, though admittedly more complex, scheduling solution.
-** Can run both cron schedules and repeating intervals.
-** By default, will not allow concurrent executions of the same job.
-** Supports multiple configurations, e.g., to limit the number of threads.
-
-An application can define its own scheduler by providing a class which implements the `QSchedulerInterface`.
-
-A `QInstance` can work with 0 or more schedulers, as defined by adding `QSchedulerMetaData` objects
-to the instance.
-
-This meta-data class is `abstract`, and is extended by the 2 built-in schedulers
-(e.g., `SimpleSchedulerMetaData` and `QuartzSchedulerMetaData`).  As such,
-these concrete subclasses are what you need to instantiate and add to your instance.
-
-To configure a QuartzScheduler, you can add a `Properties` object to the `QuartzSchedulerMetaData` object.
-See https://www.quartz-scheduler.org/documentation/[Quartz's documentation] for available configuration properties.
-
-[source,java]
-.Defining SchedulerMetaData
-----
-qInstance.addScheduler(new SimpleSchedulerMetaData().withName("mySimpleScheduler"));
-
-qInstance.addScheduler(new QuartzSchedulerMetaData()
-   .withName("myQuartzScheduler")
-   .withProperties(myQuartzProperties);
-----
-
-=== SchedulableTypes
-The types of jobs which can be scheduled in a QQQ application are defined in the `QInstance` by
-instances of the `SchedulableType` meta-data class.
-These objects contain a name, along with a `QCodeReference` to the `runner`,
-which must be a class that implements the `SchedulableRunner` interface.
-
-By default, (in the `QInstanceEnricher`), QQQ will make 3 `SchedulableType` options available:
-
-* `PROCESS` - Any Process defined in the `QInstance` can be scheduled.
-* `QUEUE_PROCESSOR` - A Queue defined in the `QInstance`, which requires polling (e.g., SQS), can be scheduled.
-* `TABLE_AUTOMATIONS` - A Table in the `QInstance`, with `AutomationDetails` referring to an
-AutomationProvider which requires polling, can be scheduled.
-
-If an application only wants to use a subset of these `SchedulableType` options,
-or to add custom `SchedulableType` options,
-the `QInstance` will need to have 1 or more `SchedulableType` objects in it before the `QInstanceEnricher` runs.
-
-=== User-defined Scheduled Jobs
-To allow users to schedule jobs (rather than using programmer-defined schedules (in meta-data)),
-you can add a set of tables to your `QInstance`, using the `ScheduledJobsMetaDataProvider` class:
-
-[source,java]
-.Adding the ScheduledJob tables and related meta-data to a QInstance
-----
-new ScheduledJobsMetaDataProvider().defineAll(
-   qInstance, backendName, table -> tableEnricher(table));
-----
-
-This meta-data provider adds a "scheduledJob" and "scheduledJobParameter" table, along with
-some PossibleValueSources.
-These tables include post-action customizers, which manage (re-, un-) scheduling jobs based on
-changes made to records in this these tables.
-
-Also, when `QScheduleManager` is started, it will query these tables,and will schedule jobs as defined therein.
-
-_You can use a mix of user-defined and meta-data defined scheduled jobs in your instance.
-However, if a ScheduledJob record references a process, queue, or table automation with a
-meta-data defined schedule, the ScheduledJob will NOT be started by ScheduleManager --
-rather, the meta-data definition will "win"._
-
-[source,sql]
-.Example of inserting scheduled jobs records directly into an SQL backend
-----
--- A process:
-INSERT INTO scheduled_job (label, scheduler_name, cron_expression, cron_time_zone_id, repeat_seconds, type, is_active) VALUES
-   ('myProcess', 'QuartzScheduler', null, null, 300, 'PROCESS', 1);
-INSERT INTO scheduled_job_parameter (scheduled_job_id, `key`, value) VALUES
-   ((SELECT id FROM scheduled_job WHERE label = 'myProcess'), 'processName', 'myProcess');
-
--- A table's insert & update automations:
-INSERT INTO scheduled_job (label, scheduler_name, cron_expression, cron_time_zone_id, repeat_seconds, type, is_active) VALUES
-   ('myTable.PENDING_INSERT_AUTOMATIONS', 'QuartzScheduler', null, null, 15, 'TABLE_AUTOMATIONS', 1),
-   ('myTable.PENDING_UPDATE_AUTOMATIONS', 'QuartzScheduler', null, null, 15, 'TABLE_AUTOMATIONS', 1);
-INSERT INTO scheduled_job_parameter (scheduled_job_id, `key`, value) VALUES
-   ((SELECT id FROM scheduled_job WHERE label = 'myTable.PENDING_INSERT_AUTOMATIONS'), 'tableName', 'myTable'),
-   ((SELECT id FROM scheduled_job WHERE label = 'myTable.PENDING_INSERT_AUTOMATIONS'), 'automationStatus', 'PENDING_INSERT_AUTOMATIONS'),
-   ((SELECT id FROM scheduled_job WHERE label = 'myTable.PENDING_UPDATE_AUTOMATIONS'), 'tableName', 'myTable'),
-   ((SELECT id FROM scheduled_job WHERE label = 'myTable.PENDING_UPDATE_AUTOMATIONS'), 'automationStatus', 'PENDING_UPDATE_AUTOMATIONS');
-
--- A queue processor:
-INSERT INTO scheduled_job (label, scheduler_name, cron_expression, cron_time_zone_id, repeat_seconds, type, is_active) VALUES
-   ('mySqsQueue', 'QuartzScheduler', null, null, 60, 'QUEUE_PROCESSOR', 1);
-INSERT INTO scheduled_job_parameter (scheduled_job_id, `key`, value) VALUES
-   ((SELECT id FROM scheduled_job WHERE label = 'mySqsQueue'), 'queueName', 'mySqsQueue');
-----
-
-=== Running Scheduled Jobs
-In a server running QQQ, if you wish to start running scheduled jobs, you need to initialize
-the `QScheduleManger` singleton class, then call its `start()` method.
-
-Note that internally, this class will check for a system property of `qqq.scheduleManager.enabled`
-or an environment variable of `QQQ_SCHEDULE_MANAGER_ENABLED`, and if the first value found is
-`"false"`, then the scheduler will not actually run its jobs (but, in the case of the `QuartzSchdeuler`,
-it will be available for managing scheduled jobs).
-
-The method `QScheduleManager.initInstance` requires 2 parameters:  Your `QInstance`, and a
-`Supplier<QSession>` lambda, which returns the session that will be used for scheduled jobs when they
-are executed.
-
-[source,java]
-.Starting the Schedule Manager service
-----
-QScheduleManager.initInstance(qInstance, () -> systemUserSession).start();
-----
-
-=== Examples
-[source,java]
-.Attach a schedule in meta-data to a Process
-----
-QProcessMetaData myProcess = new QProcessMetaData()
-   // ...
-   .withSchedule(new QScheduleMetaData()
-      .withSchedulerName("myScheduler")
-      .withDescription("Run myProcess every five minutes")
-      .withRepeatSeconds(300))
-----
-

docs/utilities/RecordLookupHelper.adoc

@@ -1,272 +0,0 @@
-== RecordLookupHelper
-include::../variables.adoc[]
-
-`RecordLookupHelper` is a utility class that exists to help you... lookup records :)
-
-OK, I'll try to give a little more context:
-
-=== Motivation 1:  Performance
-One of the most significant performance optimizations that the team behind QQQ has found time and time again,
-is to minimize the number of times you have to perform an I/O operation.
-To just say it more plainly:
-Make fewer calls to your database (or other backend).
-
-This is part of why the DML actions in QQQ (InsertAction, UpdateAction, DeleteAction) are all written to work on multiple records:
-If you've got to insert 1,000 records, the performance difference between doing that as 1,000 SQL INSERT statements vs. just 1 statement cannot be overstated.
-
-Similarly then, for looking up records:
-If we can do 1 round-trip to the database backend - that is - 1 query to fetch _n_ records,
-then in almost all cases it will be significantly faster than doing _n_ queries, one-by-one, for those _n_ records.
-
-The primary reason why `RecordLookupHelper` exists is to help you cut down on the number of times you have to make a round-trip to a backend data store to fetch records within a process.
-
-[sidebar]
-This basically is version of caching, isn't it?
-Take a set of data from "far away" (e.g., database), and bring it "closer" (local or instance variables), for faster access.
-So we may describe this as a "cache" through the rest of this document.
-
-=== Motivation 2:  Convenience
-
-So, given that one wants to try to minimize the number of queries being executed to look up data in a QQQ processes,
-one can certainly do this "by-hand" in each process that they write.
-
-Doing this kind of record caching in a QQQ Process `BackendStep` may be done as:
-
-* Adding a `Map<Integer, QRecord>` as a field in your class.
-* Setting up and running a `QueryAction`, with a filter based on the collection of the keys you need to look up, then iterating over (or streaming) the results into the map field.
-* Getting values out of the map when you need to use them (dealing with missing values as needed).
-
-That's not so bad, but, it does get a little verbose, especially if you're going to have several such caches in your class.
-
-As such, the second reason that `RecordLookupHelper` exists, is to be a reusable and convenient way to do this kind of optimization,
-by providing methods to perform the bulk query & map building operation described above,
-while also providing some convenient methods for accessing such data after it's been fetched.
-In addition, a single instance of `RecordLookupHelper` can provide this service for multiple tables at once
-(e.g., so you don't need to add a field to your class for each type of data that you're trying to cache).
-
-=== Use Cases
-==== Preload records, then access them
-Scenario:
-
-* We're writing a process `BackendStep` that uses `shipment` records as input.
-* We need to know the `order` record associated with each `shipment` (via an `orderId` foreign key), for some business logic that isn't germaine to the explanation of `RecordLookupHelper`.
-* We also to access some field on the `shippingPartner` record assigned to each `shipment`.
-** Note that here, the `shipment` table has a `partnerCode` field, which relates to the `code` unique-key in the `shippingPartner` table.
-** It's also worth mentioning, we only have a handful of `shippingPartner` records in our database, and we never expect to have very many more than that.
-
-[source,java]
-.Example of a process step using a RecordLookupHelper to preload records
-----
-public class MyShipmentProcessStep implements BackendStep
-{
-   // Add a new RecordLookupHelper field, which will "cache" both orders and shippingPartners
-   private RecordLookupHelper recordLookupHelper = new RecordLookupHelper();
-
-   @Override
-   public void run(RunBackendStepInput input, RunBackendStepOutput output) throws QException;
-   {
-      // lookup the full shippingPartner table (because it's cheap & easy to do so)
-      // use the partner's "code" as the key field (e.g,. they key in the helper's internal map).
-      recordLookupHelper.preloadRecords("shippingPartner", "code");
-
-      // get all of the orderIds from the input shipments
-      List<Serializable> orderIds = input.getRecords().stream()
-         .map(r -> r.getValue("id")).toList();
-
-      // fetch the orders related to by these shipments
-      recordLookupHelper.preloadRecords("order", "id", orderIds);
-
-      for(QRecord shipment : input.getRecords())
-      {
-         // get someConfigField from the shippingPartner assigned to the shipment
-         Boolean someConfig = recordLookupHelper.getRecordValue("shippingPartner", "someConfigField", "code", shipment.getValue("partnerCode"));
-
-         // get the order record assigned to the shipment
-         QRecord order = recordLookupHelper.getRecordByKey("order", "id", shipment.getValue("orderId"));
-      }
-   }
-}
-----
-
-==== Lazy fetching records
-Scenario:
-
-* We have a `BackendStep` that is taking in `purchaseOrderHeader` records, from an API partner.
-* For each record, we need to make an API call to the partner to fetch the `purchaseOrderLine` records under that header.
-** In this contrived example, the partner's API forces us to do these lookups order-by-order...
-* Each `purchaseOrderLine` that we fetch will have a `sku` on it - a reference to our `item` table.
-** We need to look up each `item` to apply some business logic.
-** We assume there are very many item records in the backend database, so we don't want to pre-load the full table.
-Also, we don't know what `sku` values we will need until we fetch the `purchaseOrderLine`.
-
-This is a situation where we can use `RecordLookupHelper` to lazily fetch the `item` records as we discover them,
-and it will take care of not re-fetching ones that it has already loaded.
-
-[source,java]
-.Example of a process step using a RecordLookupHelper to lazy fetch records
-----
-public class MyPurchaseOrderProcessStep implements BackendStep
-{
-   // Add a new RecordLookupHelper field, which will "cache" lazy-loaded item records
-   private RecordLookupHelper recordLookupHelper = new RecordLookupHelper();
-
-   @Override
-   public void run(RunBackendStepInput input, RunBackendStepOutput output) throws QException;
-   {
-      for(QRecord poHeader : input.getRecords())
-      {
-         // fetch the lines under the header
-         Serializable poNo = poHeader.getValue("poNo");
-         List<QRecord> poLines = new QueryAction().execute(new QueryInput("purchaseOrderLine")
-            .withFilter(new QQueryFilter(new QFilterCriteria("poNo", EQUALS, poNo))));
-
-         for(QRecord poLine : poLines)
-         {
-            // use recordLookupHelper to lazy-load item records by SKU.
-            QRecord item = recordLookupHelper.getRecordByKey("item", "sku", poLine.getValue("sku"));
-
-            // business logic related to item performed here.
-         }
-      }
-   }
-}
-----
-
-In this example, we will be doing exactly 1 query on the `item` table for each unique `sku` that is found across all of the `poLine` records we process.
-That is to say, if the same `sku` appears on only 1 `poLine`, or if it appears on 100 `poLines`, we will still only query once for that `sku`.
-
-A slight tweak could be made to the example above, to make 1 `item` table lookup for each `poHeader` record:
-
-[source,java]
-.Tweaked example doing 1 item lookup per poLine
-----
-// continuing from above, after the List<QRecord> poLines has been built
-
-// get all of the skus from the lines
-List<Serializable> skus = poLines.stream().map(r -> r.getValue("sku")).toList();
-
-// preload the items for the skus
-recordLookupHelper.preloadRecords("item", "sku", new QQueryFilter(new QFilterCriteria("sku", IN, skus)));
-
-for(QRecord poLine : poLines)
-{
-   // get the items from the helper
-   QRecord item = recordLookupHelper.getRecordByKey("item", "sku", poLine.getValue("sku"));
-
-----
-
-In this example, we've made a trade-off:  We will query the `item` table exactly 1 time for each `poHeader` that we process.
-However, if the same `sku` is on every PO that we process, we will end up fetching it multiple times.
-
-This could end up being better or worse than the previous example, depending on the distribution of the data we are dealing with.
-
-A further tweak, a hybrid approach, could potentially reap the benefits of both of these examples (at the tradeoff of, more code, more complexity):
-
-[source,java]
-.Tweaked example doing 1 item lookup per poLine, but only for not-previously-encountered skus
-----
-// Critically - we must tell our recordLookupHelper to NOT do any one-off lookups in this table
-recordLookupHelper.setMayNotDoOneOffLookups("item", "sku");
-
-// continuing from above, after the List<QRecord> poLines has been built
-
-// get all of the skus from the lines
-List<Serializable> skus = poLines.stream().map(r -> r.getValue("sku")).toList();
-
-// determine which skus have not yet been loaded - e.g., they are not in the recordLookupHelper.
-// this is why we needed to tell it above not to do one-off lookups; else it would lazy-load each sku here.
-List<Serializable> skusToLoad = new ArrayList<>();
-for(Serializable sku : skus)
-{
-   if(recordLookupHelper.getRecordByKey("item", "sku", sku) == null)
-   {
-      skusToLoad.add(sku);
-   }
-}
-
-// preload the item records for any skus that are still needed
-if(!skusToLoad.isEmpty())
-{
-   recordLookupHelper.preloadRecords("item", "sku",
-      new QQueryFilter(new QFilterCriteria("sku", IN, skusToLoad)));
-}
-
-// continue as above
-
-----
-
-In this example, we will start by querying the `item` table once for each `poHeader`, but,
-if we eventually encounter a PO where all of its `skus` have already been loaded, then we may be able to avoid any `item` queries for such a PO.
-
-=== Implementation Details
-
-* Internally, an instance of `RecordLookupHelper` maintains a number of `Maps`,
-with QQQ table names and field names as keys.
-* The accessing/lazy-fetching methods (e.g., any method whose name starts with `getRecord`)
-all begin by looking in these internal maps for the `tableName` and `keyFieldName` that they take as parameters.
-** If they find an entry in the maps, then it is used for producing a return value.
-** If they do not find an entry, then they will perform the a `QueryAction`,
-to try to fetch the requested record from the table's backend.
-*** Unless the `setMayNotDoOneOffLookups` method has been called for the `(tableName, keyFieldName)` pair.
-
-=== Full API
-
-==== Methods for accessing and lazy-fetching
-* `getRecordByKey(String tableName, String keyFieldName, Serializable key)`
-
-Get a `QRecord` from `tableName`, where `keyFieldName` = `key`.
-
-* `getRecordValue(String tableName, String requestedField, String keyFieldName, Serializable key)`
-
-Get the field `requestedField` from the record in `tableName`, where `keyFieldName` = `key`, as a `Serializable`.
-If the record is not found, `null` is returned.
-
-* `getRecordValue(String tableName, String requestedField, String keyFieldName, Serializable key, Class<T> type)`
-
-Get the field `requestedField` from the record in `tableName`, where `keyFieldName` = `key`, as an instance of `type`.
-If the record is not found, `null` is returned.
-
-* `getRecordId(String tableName, String keyFieldName, Serializable key)`
-
-Get the primary key of the record in `tableName`, where `keyFieldName` = `key`, as a `Serializable`.
-If the record is not found, `null` is returned.
-
-* `getRecordId(String tableName, String keyFieldName, Serializable key, Class<T> type)`
-
-Get the primary key of the record in `tableName`, where `keyFieldName` = `key`, as an instance of `type`.
-If the record is not found, `null` is returned.
-
-* `getRecordByUniqueKey(String tableName, Map<String, Serializable> uniqueKey)`
-
-Get a `QRecord` from `tableName`, where the record matches the field/value pairs in `uniqueKey`.
-
-_Note: this method does not use the same internal map as the rest of the class.
-As such, it does not take advantage of any data fetched via the preload methods.
-It is only used for caching lazy-fetches._
-
-==== Methods for preloading
-* `preloadRecords(String tableName, String keyFieldName)`
-
-Query for all records from `tableName`, storing them in an internal map keyed by the field `keyFieldName`.
-
-* `preloadRecords(String tableName, String keyFieldName, QQueryFilter filter)`
-
-Query for records matching `filter` from `tableName`,
-storing them in an internal map keyed by the field `keyFieldName`.
-
-* `preloadRecords(String tableName, String keyFieldName, List<Serializable> inList)`
-
-Query for records with the field `keyFieldName` having a value in `inList` from `tableName`,
-storing them in an internal map keyed by the field `keyFieldName`.
-
-==== Config Methods
-* `setMayNotDoOneOffLookups(String tableName, String fieldName)`
-
-For cases where you know that you have preloaded records for `tableName`, keyed by `fieldName`,
-and you know that some of the keys may not have been found,
-so you want to avoid doing a query when a missed key is found in one of the `getRecord...` methods,
-then if you call this method, an internal flag is set, which will prevent any such one-off lookups.
-
-In other words, if this method has been called for a `(tableName, fieldName)` pair,
-then the `getRecord...` methods will only look in the internal map for records,
-and no queries will be performed to look for records.

pom.xml

@@ -29,7 +29,6 @@
    <packaging>pom</packaging>
 
    <modules>
-      <module>qqq-bom</module>
       <module>qqq-backend-core</module>
       <module>qqq-backend-module-api</module>
       <module>qqq-backend-module-filesystem</module>
@@ -55,7 +54,7 @@
       <maven.compiler.showDeprecation>true</maven.compiler.showDeprecation>
       <maven.compiler.showWarnings>true</maven.compiler.showWarnings>
       <coverage.haltOnFailure>true</coverage.haltOnFailure>
-      <coverage.instructionCoveredRatioMinimum>0.75</coverage.instructionCoveredRatioMinimum>
+      <coverage.instructionCoveredRatioMinimum>0.80</coverage.instructionCoveredRatioMinimum>
       <coverage.classCoveredRatioMinimum>0.95</coverage.classCoveredRatioMinimum>
       <plugin.shade.phase>none</plugin.shade.phase>
    </properties>
@@ -81,12 +80,12 @@
          <dependency>
             <groupId>org.apache.logging.log4j</groupId>
             <artifactId>log4j-api</artifactId>
-            <version>2.23.0</version>
+            <version>2.17.1</version>
          </dependency>
          <dependency>
             <groupId>org.apache.logging.log4j</groupId>
             <artifactId>log4j-core</artifactId>
-            <version>2.23.0</version>
+            <version>2.17.1</version>
          </dependency>
          <dependency>
             <groupId>org.junit.jupiter</groupId>
@@ -150,7 +149,7 @@
                <dependency>
                   <groupId>com.puppycrawl.tools</groupId>
                   <artifactId>checkstyle</artifactId>
-                  <version>10.16.0</version>
+                  <version>9.0</version>
                </dependency>
             </dependencies>
             <executions>
@@ -246,7 +245,8 @@ echo "   See also target/site/jacoco/index.html"
 echo "   and https://www.jacoco.org/jacoco/trunk/doc/counters.html"
 echo "------------------------------------------------------------"
 
-if which xpath > /dev/null 2>&1; then
+which xpath > /dev/null 2>&1
+if [ "$?" == "0" ]; then
    echo "Element\nInstructions Missed\nInstruction Coverage\nBranches Missed\nBranch Coverage\nComplexity Missed\nComplexity Hit\nLines Missed\nLines Hit\nMethods Missed\nMethods Hit\nClasses Missed\nClasses Hit\n" > /tmp/$$.headers
    xpath -n -q -e '/html/body/table/tfoot/tr[1]/td/text()' target/site/jacoco/index.html > /tmp/$$.values
    paste /tmp/$$.headers /tmp/$$.values | tail +2 | awk -v FS='\t' '{printf("%-20s %s\n",$1,$2)}'
@@ -255,7 +255,8 @@ else
    echo "xpath is not installed.  Jacoco coverage summary will not be produced here...";
 fi
 
-if which html2text > /dev/null 2>&1; then
+which xpath > /dev/null 2>&1
+if [ "$?" == "0" ]; then
    echo "Untested classes, per Jacoco:"
    echo "-----------------------------"
    for i in target/site/jacoco/*/index.html; do
@@ -329,28 +330,28 @@ fi
 
    <!-- mvn javadoc:aggregate -->
    <reporting>
-      <plugins>
-         <plugin>
-            <groupId>org.apache.maven.plugins</groupId>
-            <artifactId>maven-javadoc-plugin</artifactId>
-            <version>3.6.2</version>
-            <reportSets>
-               <reportSet>
-                  <id>aggregate</id>
-                  <inherited>false</inherited>
-                  <reports>
-                     <report>aggregate</report>
-                  </reports>
-               </reportSet>
-               <reportSet>
-                  <id>default</id>
-                  <reports>
-                     <report>javadoc</report>
-                  </reports>
-               </reportSet>
-            </reportSets>
-         </plugin>
-      </plugins>
+    <plugins>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-javadoc-plugin</artifactId>
+        <version>3.6.2</version>
+        <reportSets>
+          <reportSet>
+            <id>aggregate</id>
+            <inherited>false</inherited>
+            <reports>
+              <report>aggregate</report>
+            </reports>
+          </reportSet>
+          <reportSet>
+            <id>default</id>
+            <reports>
+              <report>javadoc</report>
+            </reports>
+          </reportSet>
+        </reportSets>
+      </plugin>
+    </plugins>
    </reporting>
 
    <repositories>

qqq-backend-core/pom.xml

@@ -102,16 +102,6 @@
          <artifactId>fastexcel</artifactId>
          <version>0.12.15</version>
       </dependency>
-      <dependency>
-         <groupId>org.apache.poi</groupId>
-         <artifactId>poi</artifactId>
-         <version>5.2.5</version>
-      </dependency>
-      <dependency>
-         <groupId>org.apache.poi</groupId>
-         <artifactId>poi-ooxml</artifactId>
-         <version>5.2.5</version>
-      </dependency>
       <dependency>
          <groupId>com.auth0</groupId>
          <artifactId>auth0</artifactId>
@@ -173,45 +163,6 @@
          <version>1.12.321</version>
       </dependency>
 
-      <dependency>
-         <groupId>com.amazonaws</groupId>
-         <artifactId>aws-java-sdk-ses</artifactId>
-         <version>1.12.705</version>
-      </dependency>
-
-      <dependency>
-         <groupId>cloud.localstack</groupId>
-         <artifactId>localstack-utils</artifactId>
-         <version>0.2.20</version>
-         <scope>test</scope>
-      </dependency>
-
-      <dependency>
-         <groupId>org.quartz-scheduler</groupId>
-         <artifactId>quartz</artifactId>
-         <version>2.3.2</version>
-      </dependency>
-
-      <!-- bring in a newer version of this lib, which quartz transitively loads through c3p0 -->
-      <dependency>
-         <groupId>com.mchange</groupId>
-         <artifactId>mchange-commons-java</artifactId>
-         <version>0.3.0</version>
-      </dependency>
-
-      <!-- Many of the deps we bring in use slf4j.  This dep maps slf4j to our logger, log4j -->
-      <dependency>
-         <groupId>org.apache.logging.log4j</groupId>
-         <artifactId>log4j-slf4j-impl</artifactId>
-         <version>2.23.0</version>
-      </dependency>
-
-      <dependency>
-         <groupId>com.sun.mail</groupId>
-         <artifactId>jakarta.mail</artifactId>
-         <version>2.0.1</version>
-      </dependency>
-
       <!-- Common deps for all qqq modules -->
       <dependency>
          <groupId>org.apache.maven.plugins</groupId>

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/ActionHelper.java

@@ -37,7 +37,6 @@ import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
 import com.kingsrook.qqq.backend.core.model.session.QSession;
 import com.kingsrook.qqq.backend.core.modules.authentication.QAuthenticationModuleDispatcher;
 import com.kingsrook.qqq.backend.core.modules.authentication.QAuthenticationModuleInterface;
-import com.kingsrook.qqq.backend.core.utils.PrefixedDefaultThreadFactory;
 
 
 /*******************************************************************************
@@ -56,7 +55,7 @@ public class ActionHelper
    /////////////////////////////////////////////////////////////////////////////
    private static Integer         CORE_THREADS    = 8;
    private static Integer         MAX_THREADS     = 500;
-   private static ExecutorService executorService = new ThreadPoolExecutor(CORE_THREADS, MAX_THREADS, 60L, TimeUnit.SECONDS, new SynchronousQueue<>(), new PrefixedDefaultThreadFactory(ActionHelper.class));
+   private static ExecutorService executorService = new ThreadPoolExecutor(CORE_THREADS, MAX_THREADS, 60L, TimeUnit.SECONDS, new SynchronousQueue<>());
 
 
 

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/async/AsyncJobManager.java

@@ -42,7 +42,6 @@ import com.kingsrook.qqq.backend.core.state.InMemoryStateProvider;
 import com.kingsrook.qqq.backend.core.state.StateProviderInterface;
 import com.kingsrook.qqq.backend.core.state.StateType;
 import com.kingsrook.qqq.backend.core.state.UUIDAndTypeStateKey;
-import com.kingsrook.qqq.backend.core.utils.PrefixedDefaultThreadFactory;
 import com.kingsrook.qqq.backend.core.utils.StringUtils;
 import org.apache.logging.log4j.Level;
 import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
@@ -66,7 +65,7 @@ public class AsyncJobManager
    /////////////////////////////////////////////////////////////////////////////
    private static Integer         CORE_THREADS    = 8;
    private static Integer         MAX_THREADS     = 500;
-   private static ExecutorService executorService = new ThreadPoolExecutor(CORE_THREADS, MAX_THREADS, 60L, TimeUnit.SECONDS, new SynchronousQueue<>(), new PrefixedDefaultThreadFactory(AsyncJobManager.class));
+   private static ExecutorService executorService = new ThreadPoolExecutor(CORE_THREADS, MAX_THREADS, 60L, TimeUnit.SECONDS, new SynchronousQueue<>());
 
 
    private String forcedJobUUID = null;
@@ -160,7 +159,7 @@ public class AsyncJobManager
    private <T extends Serializable> T runAsyncJob(String jobName, AsyncJob<T> asyncJob, UUIDAndTypeStateKey uuidAndTypeStateKey, AsyncJobStatus asyncJobStatus)
    {
       String originalThreadName = Thread.currentThread().getName();
-      Thread.currentThread().setName("Job:" + jobName);
+      Thread.currentThread().setName("Job:" + jobName + ":" + uuidAndTypeStateKey.getUuid().toString().substring(0, 8));
       try
       {
          LOG.debug("Starting job " + uuidAndTypeStateKey.getUuid());
@@ -170,24 +169,17 @@ public class AsyncJobManager
          LOG.debug("Completed job " + uuidAndTypeStateKey.getUuid());
          return (result);
       }
-      catch(Throwable t)
+      catch(Exception e)
       {
          asyncJobStatus.setState(AsyncJobState.ERROR);
-         if(t instanceof Exception e)
-         {
-            asyncJobStatus.setCaughtException(e);
-         }
-         else
-         {
-            asyncJobStatus.setCaughtException(new QException("Caught throwable", t));
-         }
+         asyncJobStatus.setCaughtException(e);
          getStateProvider().put(uuidAndTypeStateKey, asyncJobStatus);
 
          //////////////////////////////////////////////////////
          // if user facing, just log an info, warn otherwise //
          //////////////////////////////////////////////////////
-         LOG.log((t instanceof QUserFacingException) ? Level.INFO : Level.WARN, "Job ended with an exception", t, logPair("jobId", uuidAndTypeStateKey.getUuid()));
-         throw (new CompletionException(t));
+         LOG.log((e instanceof QUserFacingException) ? Level.INFO : Level.WARN, "Job ended with an exception", e, logPair("jobId", uuidAndTypeStateKey.getUuid()));
+         throw (new CompletionException(e));
       }
       finally
       {

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/async/AsyncJobStatus.java

@@ -31,7 +31,6 @@ import java.io.Serializable;
  *******************************************************************************/
 public class AsyncJobStatus implements Serializable
 {
-   private String        jobName;
    private AsyncJobState state;
    private String        message;
    private Integer       current;
@@ -188,36 +187,4 @@ public class AsyncJobStatus implements Serializable
    {
       this.cancelRequested = cancelRequested;
    }
-
-
-
-   /*******************************************************************************
-    ** Getter for jobName
-    *******************************************************************************/
-   public String getJobName()
-   {
-      return (this.jobName);
-   }
-
-
-
-   /*******************************************************************************
-    ** Setter for jobName
-    *******************************************************************************/
-   public void setJobName(String jobName)
-   {
-      this.jobName = jobName;
-   }
-
-
-
-   /*******************************************************************************
-    ** Fluent setter for jobName
-    *******************************************************************************/
-   public AsyncJobStatus withJobName(String jobName)
-   {
-      this.jobName = jobName;
-      return (this);
-   }
-
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/async/AsyncRecordPipeLoop.java

@@ -83,15 +83,6 @@ public class AsyncRecordPipeLoop
       long    jobStartTime          = System.currentTimeMillis();
       boolean everCalledConsumer    = false;
 
-      ////////////////////////////////////////////////////////////////////////////
-      // in case the pipe capacity has been made very small (useful in tests!), //
-      // then make the minRecordsToConsume match it.                            //
-      ////////////////////////////////////////////////////////////////////////////
-      if(recordPipe.getCapacity() < minRecordsToConsume)
-      {
-         minRecordsToConsume = recordPipe.getCapacity();
-      }
-
       while(jobState.equals(AsyncJobState.RUNNING))
       {
          if(recordPipe.countAvailableRecords() < minRecordsToConsume)

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/async/NonPersistedAsyncJobCallback.java

@@ -1,62 +0,0 @@
-/*
- * QQQ - Low-code Application Framework for Engineers.
- * Copyright (C) 2021-2024.  Kingsrook, LLC
- * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
- * contact@kingsrook.com
- * https://github.com/Kingsrook/
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program.  If not, see <https://www.gnu.org/licenses/>.
- */
-
-package com.kingsrook.qqq.backend.core.actions.async;
-
-
-import java.util.UUID;
-
-
-/*******************************************************************************
- ** subclass designed to be used when we want there to be an instance (so code
- ** doesn't have to all be null-tolerant), but there's no one who will ever be
- ** reading the status data, so we don't need to store the object in a
- ** state provider.
- *******************************************************************************/
-public class NonPersistedAsyncJobCallback extends AsyncJobCallback
-{
-   private final AsyncJobStatus asyncJobStatus;
-
-
-
-   /*******************************************************************************
-    **
-    *******************************************************************************/
-   public NonPersistedAsyncJobCallback(UUID jobUUID, AsyncJobStatus asyncJobStatus)
-   {
-      super(jobUUID, asyncJobStatus);
-      this.asyncJobStatus = asyncJobStatus;
-   }
-
-
-
-   /*******************************************************************************
-    **
-    *******************************************************************************/
-   @Override
-   protected void storeUpdatedStatus()
-   {
-      ///////////////////////////////////////////////////////////////////////////////////////
-      // noop - cf. base class, which writes to persistence here (our point is, we do not) //
-      ///////////////////////////////////////////////////////////////////////////////////////
-   }
-
-}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/audits/AuditAction.java

@@ -295,7 +295,7 @@ public class AuditAction extends AbstractQActionFunction<AuditInput, AuditOutput
          }
          catch(Exception e)
          {
-            LOG.warn("Error performing an audit", e);
+            LOG.error("Error performing an audit", e);
          }
       }
 

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/audits/DMLAuditAction.java

@@ -78,7 +78,6 @@ public class DMLAuditAction extends AbstractQActionFunction<DMLAuditInput, DMLAu
    private static Set<String> loggedUnauditableTableNames = new HashSet<>();
 
 
-
    /*******************************************************************************
     **
     *******************************************************************************/
@@ -149,7 +148,7 @@ public class DMLAuditAction extends AbstractQActionFunction<DMLAuditInput, DMLAu
             // sort the field names by their labels //
             //////////////////////////////////////////
             List<String> sortedFieldNames = table.getFields().keySet().stream()
-               .sorted(Comparator.comparing(fieldName -> Objects.requireNonNullElse(table.getFields().get(fieldName).getLabel(), fieldName)))
+               .sorted(Comparator.comparing(fieldName -> table.getFields().get(fieldName).getLabel()))
                .toList();
 
             QFieldMetaData primaryKeyField = table.getField(table.getPrimaryKeyField());
@@ -187,7 +186,7 @@ public class DMLAuditAction extends AbstractQActionFunction<DMLAuditInput, DMLAu
       }
       catch(Exception e)
       {
-         LOG.warn("Error performing DML audit", e, logPair("type", String.valueOf(dmlType)), logPair("table", table.getName()));
+         LOG.error("Error performing DML audit", e, logPair("type", String.valueOf(dmlType)), logPair("table", table.getName()));
       }
 
       return (output);
@@ -211,19 +210,6 @@ public class DMLAuditAction extends AbstractQActionFunction<DMLAuditInput, DMLAu
          contextSuffix.append(" ").append(input.getAuditContext());
       }
 
-      //////////////////////////////////////////////////////////////
-      // look for a context value place directly into the session //
-      //////////////////////////////////////////////////////////////
-      QSession qSession = QContext.getQSession();
-      if(qSession != null)
-      {
-         String sessionContext = qSession.getValue(AUDIT_CONTEXT_FIELD_NAME);
-         if(StringUtils.hasContent(sessionContext))
-         {
-            contextSuffix.append(" ").append(sessionContext);
-         }
-      }
-
       /////////////////////////////////////////////////////////////////////////////////////
       // note process label (and a possible context from the process's state) if present //
       /////////////////////////////////////////////////////////////////////////////////////
@@ -247,20 +233,17 @@ public class DMLAuditAction extends AbstractQActionFunction<DMLAuditInput, DMLAu
       ///////////////////////////////////////////////////
       // use api label & version if present in session //
       ///////////////////////////////////////////////////
-      if(qSession != null)
+      QSession qSession   = QContext.getQSession();
+      String   apiVersion = qSession.getValue("apiVersion");
+      if(apiVersion != null)
       {
-         String apiVersion = qSession.getValue("apiVersion");
-         if(apiVersion != null)
+         String apiLabel = qSession.getValue("apiLabel");
+         if(!StringUtils.hasContent(apiLabel))
          {
-            String apiLabel = qSession.getValue("apiLabel");
-            if(!StringUtils.hasContent(apiLabel))
-            {
-               apiLabel = "API";
-            }
-            contextSuffix.append(" via ").append(apiLabel).append(" Version: ").append(apiVersion);
+            apiLabel = "API";
          }
+         contextSuffix.append(" via ").append(apiLabel).append(" Version: ").append(apiVersion);
       }
-
       return (contextSuffix.toString());
    }
 
@@ -303,7 +286,7 @@ public class DMLAuditAction extends AbstractQActionFunction<DMLAuditInput, DMLAu
          }
          else
          {
-            String formattedValue = getFormattedValueForAuditDetail(table, record, fieldName, field, value);
+            String formattedValue = getFormattedValueForAuditDetail(record, fieldName, field, value);
             detailRecord = new QRecord().withValue("message", "Set " + field.getLabel() + " to " + formattedValue);
             detailRecord.withValue("newValue", formattedValue);
          }
@@ -329,8 +312,8 @@ public class DMLAuditAction extends AbstractQActionFunction<DMLAuditInput, DMLAu
             }
             else
             {
-               String formattedValue    = getFormattedValueForAuditDetail(table, record, fieldName, field, value);
-               String formattedOldValue = getFormattedValueForAuditDetail(table, oldRecord, fieldName, field, oldValue);
+               String formattedValue    = getFormattedValueForAuditDetail(record, fieldName, field, value);
+               String formattedOldValue = getFormattedValueForAuditDetail(oldRecord, fieldName, field, oldValue);
 
                if(oldValue == null)
                {
@@ -464,7 +447,7 @@ public class DMLAuditAction extends AbstractQActionFunction<DMLAuditInput, DMLAu
    /*******************************************************************************
     **
     *******************************************************************************/
-   private static String getFormattedValueForAuditDetail(QTableMetaData table, QRecord record, String fieldName, QFieldMetaData field, Serializable value)
+   private static String getFormattedValueForAuditDetail(QRecord record, String fieldName, QFieldMetaData field, Serializable value)
    {
       String formattedValue = null;
       if(value != null)
@@ -479,8 +462,7 @@ public class DMLAuditAction extends AbstractQActionFunction<DMLAuditInput, DMLAu
          }
          else
          {
-            QValueFormatter.setDisplayValuesInRecord(table, table.getFields(), record);
-            formattedValue = record.getDisplayValue(fieldName);
+            formattedValue = QValueFormatter.formatValue(field, value);
          }
       }
 

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/automation/AutomationStatus.java

@@ -22,7 +22,6 @@
 package com.kingsrook.qqq.backend.core.actions.automation;
 
 
-import java.util.Objects;
 import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.PossibleValueEnum;
 
 
@@ -56,30 +55,6 @@ public enum AutomationStatus implements PossibleValueEnum<Integer>
 
 
 
-   /*******************************************************************************
-    ** Get instance by id
-    **
-    *******************************************************************************/
-   public static AutomationStatus getById(Integer id)
-   {
-      if(id == null)
-      {
-         return (null);
-      }
-
-      for(AutomationStatus value : AutomationStatus.values())
-      {
-         if(Objects.equals(value.id, id))
-         {
-            return (value);
-         }
-      }
-
-      return (null);
-   }
-
-
-
    /*******************************************************************************
     ** Getter for id
     **
@@ -127,13 +102,14 @@ public enum AutomationStatus implements PossibleValueEnum<Integer>
    /*******************************************************************************
     **
     *******************************************************************************/
+   @SuppressWarnings("checkstyle:indentation")
    public String getInsertOrUpdate()
    {
       return switch(this)
-      {
-         case PENDING_INSERT_AUTOMATIONS, RUNNING_INSERT_AUTOMATIONS, FAILED_INSERT_AUTOMATIONS -> "Insert";
-         case PENDING_UPDATE_AUTOMATIONS, RUNNING_UPDATE_AUTOMATIONS, FAILED_UPDATE_AUTOMATIONS -> "Update";
-         case OK -> "";
-      };
+         {
+            case PENDING_INSERT_AUTOMATIONS, RUNNING_INSERT_AUTOMATIONS, FAILED_INSERT_AUTOMATIONS -> "Insert";
+            case PENDING_UPDATE_AUTOMATIONS, RUNNING_UPDATE_AUTOMATIONS, FAILED_UPDATE_AUTOMATIONS -> "Update";
+            case OK -> "";
+         };
    }
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/automation/RecordAutomationStatusUpdater.java

@@ -22,40 +22,30 @@
 package com.kingsrook.qqq.backend.core.actions.automation;
 
 
-import java.io.Serializable;
-import java.time.Duration;
-import java.time.temporal.ChronoUnit;
 import java.util.Collections;
-import java.util.HashSet;
 import java.util.List;
-import java.util.Optional;
-import java.util.Set;
-import com.kingsrook.qqq.backend.core.actions.QBackendTransaction;
 import com.kingsrook.qqq.backend.core.actions.tables.CountAction;
-import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
 import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction;
 import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
-import com.kingsrook.qqq.backend.core.instances.QMetaDataVariableInterpreter;
 import com.kingsrook.qqq.backend.core.logging.QLogger;
 import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountOutput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
-import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
 import com.kingsrook.qqq.backend.core.model.automation.TableTrigger;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.AutomationStatusTrackingType;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.QTableAutomationDetails;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.TableAutomationAction;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.TriggerEvent;
+import com.kingsrook.qqq.backend.core.model.session.QSession;
 import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
-import com.kingsrook.qqq.backend.core.utils.memoization.Memoization;
 import org.apache.commons.lang.NotImplementedException;
-import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
 
 
 /*******************************************************************************
@@ -65,37 +55,19 @@ public class RecordAutomationStatusUpdater
 {
    private static final QLogger LOG = QLogger.getLogger(RecordAutomationStatusUpdater.class);
 
-   ///////////////////////////////////////////////////////////////////////////////////////////////////////
-   // feature flag - by default, will be true - before setting records to PENDING_UPDATE_AUTOMATIONS,   //
-   // we will fetch them (if we didn't take them in from the caller, which, UpdateAction does if its    //
-   // backend supports it), to check their current automationStatus - and if they are currently PENDING //
-   // or RUNNING inserts or updates, we won't update them.                                              //
-   ///////////////////////////////////////////////////////////////////////////////////////////////////////
-   private static boolean allowPreUpdateFetch = new QMetaDataVariableInterpreter().getBooleanFromPropertyOrEnvironment("qqq.recordAutomationStatusUpdater.allowPreUpdateFetch", "QQQ_RECORD_AUTOMATION_STATUS_UPDATER_ALLOW_PRE_UPDATE_FETCH", true);
-
-   ///////////////////////////////////////////////////////////////////////////////////////////////
-   // feature flag - by default, we'll memoize the check for triggers - but we can turn it off. //
-   ///////////////////////////////////////////////////////////////////////////////////////////////
-   private static boolean memoizeCheckForTriggers = new QMetaDataVariableInterpreter().getBooleanFromPropertyOrEnvironment("qqq.recordAutomationStatusUpdater.memoizeCheckForTriggers", "QQQ_RECORD_AUTOMATION_STATUS_UPDATER_MEMOIZE_CHECK_FOR_TRIGGERS", true);
-
-   private static Memoization<Key, Boolean> areThereTableTriggersForTableMemoization = new Memoization<Key, Boolean>().withTimeout(Duration.of(60, ChronoUnit.SECONDS));
-
 
 
    /*******************************************************************************
     ** for a list of records from a table, set their automation status - based on
     ** how the table is configured.
     *******************************************************************************/
-   public static boolean setAutomationStatusInRecords(QTableMetaData table, List<QRecord> records, AutomationStatus automationStatus, QBackendTransaction transaction, List<QRecord> oldRecordList)
+   public static boolean setAutomationStatusInRecords(QSession session, QTableMetaData table, List<QRecord> records, AutomationStatus automationStatus)
    {
       if(table == null || table.getAutomationDetails() == null || CollectionUtils.nullSafeIsEmpty(records))
       {
          return (false);
       }
 
-      QTableAutomationDetails automationDetails   = table.getAutomationDetails();
-      Set<Serializable>       pkeysWeMayNotUpdate = new HashSet<>();
-
       ///////////////////////////////////////////////////////////////////////////////////////////////////
       // In case an automation is running, and it updates records - don't let those records be marked  //
       // as PENDING_UPDATE_AUTOMATIONS... this is meant to avoid having a record's automation update   //
@@ -109,60 +81,12 @@ public class RecordAutomationStatusUpdater
          for(StackTraceElement stackTraceElement : e.getStackTrace())
          {
             String className = stackTraceElement.getClassName();
-            if(className.contains(RecordAutomationStatusUpdater.class.getPackageName()) && !className.equals(RecordAutomationStatusUpdater.class.getName()) && !className.endsWith("Test") && !className.contains("Test$"))
+            if(className.contains("com.kingsrook.qqq.backend.core.actions.automation") && !className.equals(RecordAutomationStatusUpdater.class.getName()) && !className.endsWith("Test"))
             {
                LOG.debug("Avoiding re-setting automation status to PENDING_UPDATE while running an automation");
                return (false);
             }
          }
-
-         ///////////////////////////////////////////////////////////////////////////////
-         // if table uses field-in-table status tracking, then check the old records, //
-         // before we set them to pending-updates, to avoid losing other pending or   //
-         // running status information.  We will allow moving  from OK or the 2       //
-         // failed statuses into pending-updates - which seems right.                 //
-         // This is added to fix cases where an update that comes in before insert    //
-         // -automations have run, will cause the pending-insert status to be missed. //
-         ///////////////////////////////////////////////////////////////////////////////
-         if(automationDetails.getStatusTracking() != null && AutomationStatusTrackingType.FIELD_IN_TABLE.equals(automationDetails.getStatusTracking().getType()))
-         {
-            try
-            {
-               if(CollectionUtils.nullSafeIsEmpty(oldRecordList))
-               {
-                  ///////////////////////////////////////////////////////////////////////////////////////////////
-                  // if we didn't get the oldRecordList as input (though UpdateAction should usually pass it?) //
-                  // then check feature-flag if we're allowed to do a lookup here & now.  If so, then do.      //
-                  ///////////////////////////////////////////////////////////////////////////////////////////////
-                  if(allowPreUpdateFetch)
-                  {
-                     List<Serializable> pkeysToLookup = records.stream().map(r -> r.getValue(table.getPrimaryKeyField())).toList();
-                     oldRecordList = new QueryAction().execute(new QueryInput(table.getName())
-                        .withFilter(new QQueryFilter(new QFilterCriteria(table.getPrimaryKeyField(), QCriteriaOperator.IN, pkeysToLookup)))
-                        .withTransaction(transaction)
-                     ).getRecords();
-                  }
-               }
-
-               for(QRecord freshRecord : CollectionUtils.nonNullList(oldRecordList))
-               {
-                  Serializable recordStatus = freshRecord.getValue(automationDetails.getStatusTracking().getFieldName());
-                  if(AutomationStatus.PENDING_INSERT_AUTOMATIONS.getId().equals(recordStatus)
-                     || AutomationStatus.PENDING_UPDATE_AUTOMATIONS.getId().equals(recordStatus)
-                     || AutomationStatus.RUNNING_INSERT_AUTOMATIONS.getId().equals(recordStatus)
-                     || AutomationStatus.RUNNING_UPDATE_AUTOMATIONS.getId().equals(recordStatus))
-                  {
-                     Serializable primaryKey = freshRecord.getValue(table.getPrimaryKeyField());
-                     LOG.debug("May not update automation status", logPair("table", table.getName()), logPair("id", primaryKey), logPair("currentStatus", recordStatus), logPair("requestedStatus", automationStatus.getId()));
-                     pkeysWeMayNotUpdate.add(primaryKey);
-                  }
-               }
-            }
-            catch(QException qe)
-            {
-               LOG.error("Error checking existing automation status before setting new automation status - more records will be updated than maybe should be...", qe);
-            }
-         }
       }
 
       ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
@@ -174,15 +98,19 @@ public class RecordAutomationStatusUpdater
          automationStatus = AutomationStatus.OK;
       }
 
+      QTableAutomationDetails automationDetails = table.getAutomationDetails();
       if(automationDetails.getStatusTracking() != null && AutomationStatusTrackingType.FIELD_IN_TABLE.equals(automationDetails.getStatusTracking().getType()))
       {
          for(QRecord record : records)
          {
-            if(!pkeysWeMayNotUpdate.contains(record.getValue(table.getPrimaryKeyField())))
-            {
-               record.setValue(automationDetails.getStatusTracking().getFieldName(), automationStatus.getId());
-               // todo - another field - for the automation timestamp??
-            }
+            ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+            // todo - seems like there's some case here, where if an order was in PENDING_INSERT, but then some other job updated the record, that we'd  //
+            //  lose that pending status, which would be a Bad Thing™...                                                                                 //
+            //  problem is - we may not have the full record in here, so we can't necessarily check the record to see what status it's currently in...   //
+            ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
+            record.setValue(automationDetails.getStatusTracking().getFieldName(), automationStatus.getId());
+            // todo - another field - for the automation timestamp??
          }
       }
 
@@ -260,29 +188,11 @@ public class RecordAutomationStatusUpdater
          return (false);
       }
 
-      if(memoizeCheckForTriggers)
-      {
-         ///////////////////////////////////////////////////////////////////////////////////////
-         // as within the lookup method, error on the side of "yes, maybe there are triggers" //
-         ///////////////////////////////////////////////////////////////////////////////////////
-         Optional<Boolean> result = areThereTableTriggersForTableMemoization.getResult(new Key(table, triggerEvent), key -> lookupIfThereAreTriggersForTable(table, triggerEvent));
-         return result.orElse(true);
-      }
-      else
-      {
-         return lookupIfThereAreTriggersForTable(table, triggerEvent);
-      }
-   }
-
-
-
-   /*******************************************************************************
-    **
-    *******************************************************************************/
-   private static Boolean lookupIfThereAreTriggersForTable(QTableMetaData table, TriggerEvent triggerEvent)
-   {
       try
       {
+         ///////////////////
+         // todo - cache? //
+         ///////////////////
          CountInput countInput = new CountInput();
          countInput.setTableName(TableTrigger.TABLE_NAME);
          countInput.setFilter(new QQueryFilter(
@@ -297,7 +207,6 @@ public class RecordAutomationStatusUpdater
          ///////////////////////////////////////////////////////////////////////////////////////////////////////////
          // if the count query failed, we're a bit safer to err on the side of "yeah, there might be automations" //
          ///////////////////////////////////////////////////////////////////////////////////////////////////////////
-         LOG.warn("Error looking if there are triggers for table", e, logPair("tableName", table.getName()));
          return (true);
       }
    }
@@ -308,12 +217,12 @@ public class RecordAutomationStatusUpdater
     ** for a list of records, update their automation status and actually Update the
     ** backend as well.
     *******************************************************************************/
-   public static void setAutomationStatusInRecordsAndUpdate(QTableMetaData table, List<QRecord> records, AutomationStatus automationStatus, QBackendTransaction transaction) throws QException
+   public static void setAutomationStatusInRecordsAndUpdate(QInstance instance, QSession session, QTableMetaData table, List<QRecord> records, AutomationStatus automationStatus) throws QException
    {
       QTableAutomationDetails automationDetails = table.getAutomationDetails();
       if(automationDetails != null && AutomationStatusTrackingType.FIELD_IN_TABLE.equals(automationDetails.getStatusTracking().getType()))
       {
-         boolean didSetStatusField = setAutomationStatusInRecords(table, records, automationStatus, transaction, null);
+         boolean didSetStatusField = setAutomationStatusInRecords(session, table, records, automationStatus);
          if(didSetStatusField)
          {
             UpdateInput updateInput = new UpdateInput();
@@ -328,7 +237,6 @@ public class RecordAutomationStatusUpdater
                .withValue(table.getPrimaryKeyField(), r.getValue(table.getPrimaryKeyField()))
                .withValue(automationDetails.getStatusTracking().getFieldName(), r.getValue(automationDetails.getStatusTracking().getFieldName()))).toList());
             updateInput.setAreAllValuesBeingUpdatedTheSame(true);
-            updateInput.setTransaction(transaction);
             updateInput.setOmitDmlAudit(true);
 
             new UpdateAction().execute(updateInput);
@@ -342,8 +250,4 @@ public class RecordAutomationStatusUpdater
       }
    }
 
-
-
-   private record Key(QTableMetaData table, TriggerEvent triggerEvent) {}
-
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/automation/polling/PollingAutomationPerTableRunner.java

@@ -28,7 +28,6 @@ import java.util.Comparator;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
-import java.util.Optional;
 import java.util.function.Supplier;
 import java.util.stream.Collectors;
 import com.kingsrook.qqq.backend.core.actions.async.AsyncRecordPipeLoop;
@@ -61,8 +60,6 @@ import com.kingsrook.qqq.backend.core.model.automation.TableTrigger;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
 import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
 import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
-import com.kingsrook.qqq.backend.core.model.metadata.fields.DynamicDefaultValueBehavior;
-import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.AutomationStatusTrackingType;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.QTableAutomationDetails;
@@ -132,11 +129,6 @@ public class PollingAutomationPerTableRunner implements Runnable
        *******************************************************************************/
       String tableName();
 
-      /*******************************************************************************
-       **
-       *******************************************************************************/
-      QTableAutomationDetails tableAutomationDetails();
-
       /*******************************************************************************
        **
        *******************************************************************************/
@@ -148,7 +140,7 @@ public class PollingAutomationPerTableRunner implements Runnable
    /*******************************************************************************
     ** Wrapper for a pair of (tableName, automationStatus)
     *******************************************************************************/
-   public record TableActions(String tableName, QTableAutomationDetails tableAutomationDetails, AutomationStatus status) implements TableActionsInterface
+   public record TableActions(String tableName, AutomationStatus status) implements TableActionsInterface
    {
       /*******************************************************************************
        **
@@ -164,7 +156,7 @@ public class PollingAutomationPerTableRunner implements Runnable
     ** extended version of TableAction, for sharding use-case - adds the shard
     ** details.
     *******************************************************************************/
-   public record ShardedTableActions(String tableName, QTableAutomationDetails tableAutomationDetails, AutomationStatus status, String shardByFieldName, Serializable shardValue, String shardLabel) implements TableActionsInterface
+   public record ShardedTableActions(String tableName, AutomationStatus status, String shardByFieldName, Serializable shardValue, String shardLabel) implements TableActionsInterface
    {
       /*******************************************************************************
        **
@@ -203,8 +195,8 @@ public class PollingAutomationPerTableRunner implements Runnable
                   {
                      Serializable shardId = record.getValue(automationDetails.getShardIdFieldName());
                      String       label   = record.getValueString(automationDetails.getShardLabelFieldName());
-                     tableActionList.add(new ShardedTableActions(table.getName(), automationDetails, AutomationStatus.PENDING_INSERT_AUTOMATIONS, automationDetails.getShardByFieldName(), shardId, label));
-                     tableActionList.add(new ShardedTableActions(table.getName(), automationDetails, AutomationStatus.PENDING_UPDATE_AUTOMATIONS, automationDetails.getShardByFieldName(), shardId, label));
+                     tableActionList.add(new ShardedTableActions(table.getName(), AutomationStatus.PENDING_INSERT_AUTOMATIONS, automationDetails.getShardByFieldName(), shardId, label));
+                     tableActionList.add(new ShardedTableActions(table.getName(), AutomationStatus.PENDING_UPDATE_AUTOMATIONS, automationDetails.getShardByFieldName(), shardId, label));
                   }
                }
                catch(Exception e)
@@ -214,11 +206,11 @@ public class PollingAutomationPerTableRunner implements Runnable
             }
             else
             {
-               //////////////////////////////////////////////////////////////////
-               // for non-sharded, we just need table name & automation status //
-               //////////////////////////////////////////////////////////////////
-               tableActionList.add(new TableActions(table.getName(), automationDetails, AutomationStatus.PENDING_INSERT_AUTOMATIONS));
-               tableActionList.add(new TableActions(table.getName(), automationDetails, AutomationStatus.PENDING_UPDATE_AUTOMATIONS));
+               ///////////////////////////////////////////////////////////////////
+               // for non-sharded, we just need tabler name & automation status //
+               ///////////////////////////////////////////////////////////////////
+               tableActionList.add(new TableActions(table.getName(), AutomationStatus.PENDING_INSERT_AUTOMATIONS));
+               tableActionList.add(new TableActions(table.getName(), AutomationStatus.PENDING_UPDATE_AUTOMATIONS));
             }
          }
       }
@@ -260,11 +252,12 @@ public class PollingAutomationPerTableRunner implements Runnable
 
       try
       {
-         processTableInsertOrUpdate(instance.getTable(tableActions.tableName()), tableActions.status());
+         QSession session = sessionSupplier != null ? sessionSupplier.get() : new QSession();
+         processTableInsertOrUpdate(instance.getTable(tableActions.tableName()), session, tableActions.status());
       }
       catch(Exception e)
       {
-         LOG.warn("Error running automations", e, logPair("tableName", tableActions.tableName()), logPair("status", tableActions.status()));
+         LOG.warn("Error running automations", e);
       }
       finally
       {
@@ -278,7 +271,7 @@ public class PollingAutomationPerTableRunner implements Runnable
    /*******************************************************************************
     ** Query for and process records that have a PENDING_INSERT or PENDING_UPDATE status on a given table.
     *******************************************************************************/
-   public void processTableInsertOrUpdate(QTableMetaData table, AutomationStatus automationStatus) throws QException
+   public void processTableInsertOrUpdate(QTableMetaData table, QSession session, AutomationStatus automationStatus) throws QException
    {
       /////////////////////////////////////////////////////////////////////////
       // get the actions to run against this table in this automation status //
@@ -308,9 +301,7 @@ public class PollingAutomationPerTableRunner implements Runnable
             AutomationStatusTrackingType statusTrackingType = automationDetails.getStatusTracking().getType();
             if(AutomationStatusTrackingType.FIELD_IN_TABLE.equals(statusTrackingType))
             {
-               QQueryFilter filter = new QQueryFilter().withCriteria(new QFilterCriteria(automationDetails.getStatusTracking().getFieldName(), QCriteriaOperator.EQUALS, List.of(automationStatus.getId())));
-               addOrderByToQueryFilter(table, automationStatus, filter);
-               queryInput.setFilter(filter);
+               queryInput.setFilter(new QQueryFilter().withCriteria(new QFilterCriteria(automationDetails.getStatusTracking().getFieldName(), QCriteriaOperator.EQUALS, List.of(automationStatus.getId()))));
             }
             else
             {
@@ -331,7 +322,7 @@ public class PollingAutomationPerTableRunner implements Runnable
          }, () ->
          {
             List<QRecord> records = recordPipe.consumeAvailableRecords();
-            applyActionsToRecords(table, records, actions, automationStatus);
+            applyActionsToRecords(session, table, records, actions, automationStatus);
             return (records.size());
          }
       );
@@ -339,38 +330,6 @@ public class PollingAutomationPerTableRunner implements Runnable
 
 
 
-   /*******************************************************************************
-    **
-    *******************************************************************************/
-   static void addOrderByToQueryFilter(QTableMetaData table, AutomationStatus automationStatus, QQueryFilter filter)
-   {
-      ////////////////////////////////////////////////////////////////////////////////////
-      // look for a field in the table with either create-date or modify-date behavior, //
-      // based on if doing insert or update automations                                 //
-      ////////////////////////////////////////////////////////////////////////////////////
-      DynamicDefaultValueBehavior dynamicDefaultValueBehavior = automationStatus.equals(AutomationStatus.PENDING_INSERT_AUTOMATIONS) ? DynamicDefaultValueBehavior.CREATE_DATE : DynamicDefaultValueBehavior.MODIFY_DATE;
-      Optional<QFieldMetaData> field = table.getFields().values().stream()
-         .filter(f -> dynamicDefaultValueBehavior.equals(f.getBehaviorOrDefault(QContext.getQInstance(), DynamicDefaultValueBehavior.class)))
-         .findFirst();
-
-      if(field.isPresent())
-      {
-         //////////////////////////////////////////////////////////////////////
-         // if a create/modify date field was found, order by it (ascending) //
-         //////////////////////////////////////////////////////////////////////
-         filter.addOrderBy(new QFilterOrderBy(field.get().getName()));
-      }
-      else
-      {
-         ////////////////////////////////////
-         // else, order by the primary key //
-         ////////////////////////////////////
-         filter.addOrderBy(new QFilterOrderBy(table.getPrimaryKeyField()));
-      }
-   }
-
-
-
    /*******************************************************************************
     ** get the actions to run against a table in an automation status.  both from
     ** metaData and tableTriggers/data.
@@ -471,7 +430,7 @@ public class PollingAutomationPerTableRunner implements Runnable
     ** table's actions against them - IF they are found to match the action's filter
     ** (assuming it has one - if it doesn't, then all records match).
     *******************************************************************************/
-   private void applyActionsToRecords(QTableMetaData table, List<QRecord> records, List<TableAutomationAction> actions, AutomationStatus automationStatus) throws QException
+   private void applyActionsToRecords(QSession session, QTableMetaData table, List<QRecord> records, List<TableAutomationAction> actions, AutomationStatus automationStatus) throws QException
    {
       if(CollectionUtils.nullSafeIsEmpty(records))
       {
@@ -481,7 +440,7 @@ public class PollingAutomationPerTableRunner implements Runnable
       ///////////////////////////////////////////////////
       // mark the records as RUNNING their automations //
       ///////////////////////////////////////////////////
-      RecordAutomationStatusUpdater.setAutomationStatusInRecordsAndUpdate(table, records, pendingToRunningStatusMap.get(automationStatus), null);
+      RecordAutomationStatusUpdater.setAutomationStatusInRecordsAndUpdate(instance, session, table, records, pendingToRunningStatusMap.get(automationStatus));
 
       ///////////////////////////////////////////////////////////////////////////////////////////////////////////////
       // foreach action - run it against the records (but only if they match the action's filter, if there is one) //
@@ -499,15 +458,13 @@ public class PollingAutomationPerTableRunner implements Runnable
       ////////////////////////////////////////
       // update status on all these records //
       ////////////////////////////////////////
-      AutomationStatus statusToUpdateTo = anyActionsFailed ? pendingToFailedStatusMap.get(automationStatus) : AutomationStatus.OK;
-      try
+      if(anyActionsFailed)
       {
-         RecordAutomationStatusUpdater.setAutomationStatusInRecordsAndUpdate(table, records, statusToUpdateTo, null);
+         RecordAutomationStatusUpdater.setAutomationStatusInRecordsAndUpdate(instance, session, table, records, pendingToFailedStatusMap.get(automationStatus));
       }
-      catch(Exception e)
+      else
       {
-         LOG.warn("Error updating automationStatus after running automations", logPair("tableName", table), logPair("count", records.size()), logPair("status", statusToUpdateTo));
-         throw (e);
+         RecordAutomationStatusUpdater.setAutomationStatusInRecordsAndUpdate(instance, session, table, records, AutomationStatus.OK);
       }
    }
 
@@ -526,7 +483,7 @@ public class PollingAutomationPerTableRunner implements Runnable
          // note - this method - will re-query the objects, so we should have confidence that their data is fresh... //
          //////////////////////////////////////////////////////////////////////////////////////////////////////////////
          List<QRecord> matchingQRecords = getRecordsMatchingActionFilter(table, records, action);
-         LOG.debug("Of the [" + records.size() + "] records that were pending automations, [" + matchingQRecords.size() + "] of them match the filter on the action:" +  action);
+         LOG.debug("Of the {} records that were pending automations, {} of them match the filter on the action {}", records.size(), matchingQRecords.size(), action);
          if(CollectionUtils.nullSafeHasContents(matchingQRecords))
          {
             LOG.debug("  Processing " + matchingQRecords.size() + " records in " + table + " for action " + action);
@@ -537,7 +494,7 @@ public class PollingAutomationPerTableRunner implements Runnable
       }
       catch(Exception e)
       {
-         LOG.warn("Caught exception processing automations", e, logPair("tableName", table), logPair("action", action.getName()));
+         LOG.warn("Caught exception processing records on " + table + " for action " + action, e);
          return (true);
       }
    }
@@ -601,7 +558,7 @@ public class PollingAutomationPerTableRunner implements Runnable
 
    /*******************************************************************************
     ** Finally, actually run action code against a list of known matching records.
-    **
+    ** todo not commit - move to somewhere genericer
     *******************************************************************************/
    public static void applyActionToMatchingRecords(QTableMetaData table, List<QRecord> records, TableAutomationAction action) throws Exception
    {

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/AbstractPostDeleteCustomizer.java

@@ -47,24 +47,12 @@ import com.kingsrook.qqq.backend.core.model.data.QRecord;
  ** records that the delete action marked in error - the user might want to do
  ** something special with them (idk, try some other way to delete them?)
  *******************************************************************************/
-public abstract class AbstractPostDeleteCustomizer implements TableCustomizerInterface
+public abstract class AbstractPostDeleteCustomizer
 {
    protected DeleteInput deleteInput;
 
 
 
-   /*******************************************************************************
-    **
-    *******************************************************************************/
-   @Override
-   public List<QRecord> postDelete(DeleteInput deleteInput, List<QRecord> records) throws QException
-   {
-      this.deleteInput = deleteInput;
-      return apply(records);
-   }
-
-
-
    /*******************************************************************************
     **
     *******************************************************************************/

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/AbstractPostInsertCustomizer.java

@@ -42,24 +42,12 @@ import com.kingsrook.qqq.backend.core.model.data.QRecord;
  **
  ** Note that the full insertInput is available as a field in this class.
  *******************************************************************************/
-public abstract class AbstractPostInsertCustomizer implements TableCustomizerInterface
+public abstract class AbstractPostInsertCustomizer
 {
    protected InsertInput insertInput;
 
 
 
-   /*******************************************************************************
-    **
-    *******************************************************************************/
-   @Override
-   public List<QRecord> postInsert(InsertInput insertInput, List<QRecord> records) throws QException
-   {
-      this.insertInput = insertInput;
-      return (apply(records));
-   }
-
-
-
    /*******************************************************************************
     **
     *******************************************************************************/

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/AbstractPostQueryCustomizer.java

@@ -23,29 +23,16 @@ package com.kingsrook.qqq.backend.core.actions.customizers;
 
 
 import java.util.List;
-import com.kingsrook.qqq.backend.core.exceptions.QException;
-import com.kingsrook.qqq.backend.core.model.actions.tables.QueryOrGetInputInterface;
+import com.kingsrook.qqq.backend.core.model.actions.AbstractTableActionInput;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
 
 
 /*******************************************************************************
  **
  *******************************************************************************/
-public abstract class AbstractPostQueryCustomizer implements TableCustomizerInterface
+public abstract class AbstractPostQueryCustomizer
 {
-   protected QueryOrGetInputInterface input;
-
-
-
-   /*******************************************************************************
-    **
-    *******************************************************************************/
-   @Override
-   public List<QRecord> postQuery(QueryOrGetInputInterface queryInput, List<QRecord> records) throws QException
-   {
-      input = queryInput;
-      return apply(records);
-   }
+   protected AbstractTableActionInput input;
 
 
 
@@ -60,7 +47,7 @@ public abstract class AbstractPostQueryCustomizer implements TableCustomizerInte
     ** Getter for input
     **
     *******************************************************************************/
-   public QueryOrGetInputInterface getInput()
+   public AbstractTableActionInput getInput()
    {
       return (input);
    }
@@ -71,7 +58,7 @@ public abstract class AbstractPostQueryCustomizer implements TableCustomizerInte
     ** Setter for input
     **
     *******************************************************************************/
-   public void setInput(QueryOrGetInputInterface input)
+   public void setInput(AbstractTableActionInput input)
    {
       this.input = input;
    }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/AbstractPostUpdateCustomizer.java

@@ -26,7 +26,6 @@ import java.io.Serializable;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
-import java.util.Optional;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
@@ -49,7 +48,7 @@ import com.kingsrook.qqq.backend.core.model.data.QRecord;
  ** available (if the backend supports it) - both as a list (`getOldRecordList`)
  ** and as a memoized (by this class) map of primaryKey to record (`getOldRecordMap`).
  *******************************************************************************/
-public abstract class AbstractPostUpdateCustomizer implements TableCustomizerInterface
+public abstract class AbstractPostUpdateCustomizer
 {
    protected UpdateInput   updateInput;
    protected List<QRecord> oldRecordList;
@@ -58,19 +57,6 @@ public abstract class AbstractPostUpdateCustomizer implements TableCustomizerInt
 
 
 
-   /*******************************************************************************
-    **
-    *******************************************************************************/
-   @Override
-   public List<QRecord> postUpdate(UpdateInput updateInput, List<QRecord> records, Optional<List<QRecord>> oldRecordList) throws QException
-   {
-      this.updateInput = updateInput;
-      this.oldRecordList = oldRecordList.orElse(null);
-      return apply(records);
-   }
-
-
-
    /*******************************************************************************
     **
     *******************************************************************************/

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/AbstractPreDeleteCustomizer.java

@@ -50,7 +50,7 @@ import com.kingsrook.qqq.backend.core.model.data.QRecord;
  ** Note that the full deleteInput is available as a field in this class.
  **
  *******************************************************************************/
-public abstract class AbstractPreDeleteCustomizer implements TableCustomizerInterface
+public abstract class AbstractPreDeleteCustomizer
 {
    protected DeleteInput deleteInput;
 
@@ -58,19 +58,6 @@ public abstract class AbstractPreDeleteCustomizer implements TableCustomizerInte
 
 
 
-   /*******************************************************************************
-    **
-    *******************************************************************************/
-   @Override
-   public List<QRecord> preDelete(DeleteInput deleteInput, List<QRecord> records, boolean isPreview) throws QException
-   {
-      this.deleteInput = deleteInput;
-      this.isPreview = isPreview;
-      return apply(records);
-   }
-
-
-
    /*******************************************************************************
     **
     *******************************************************************************/

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/AbstractPreInsertCustomizer.java

@@ -47,7 +47,7 @@ import com.kingsrook.qqq.backend.core.model.data.QRecord;
  **
  ** Note that the full insertInput is available as a field in this class.
  *******************************************************************************/
-public abstract class AbstractPreInsertCustomizer implements TableCustomizerInterface
+public abstract class AbstractPreInsertCustomizer
 {
    protected InsertInput insertInput;
 
@@ -70,30 +70,6 @@ public abstract class AbstractPreInsertCustomizer implements TableCustomizerInte
 
 
 
-   /*******************************************************************************
-    **
-    *******************************************************************************/
-   @Override
-   public List<QRecord> preInsert(InsertInput insertInput, List<QRecord> records, boolean isPreview) throws QException
-   {
-      this.insertInput = insertInput;
-      this.isPreview = isPreview;
-      return (apply(records));
-   }
-
-
-
-   /*******************************************************************************
-    **
-    *******************************************************************************/
-   @Override
-   public WhenToRun whenToRunPreInsert(InsertInput insertInput, boolean isPreview)
-   {
-      return getWhenToRun();
-   }
-
-
-
    /*******************************************************************************
     **
     *******************************************************************************/

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/AbstractPreUpdateCustomizer.java

@@ -26,7 +26,6 @@ import java.io.Serializable;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
-import java.util.Optional;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
@@ -54,7 +53,7 @@ import com.kingsrook.qqq.backend.core.model.data.QRecord;
  ** available (if the backend supports it) - both as a list (`getOldRecordList`)
  ** and as a memoized (by this class) map of primaryKey to record (`getOldRecordMap`).
  *******************************************************************************/
-public abstract class AbstractPreUpdateCustomizer implements TableCustomizerInterface
+public abstract class AbstractPreUpdateCustomizer
 {
    protected UpdateInput   updateInput;
    protected List<QRecord> oldRecordList;
@@ -64,20 +63,6 @@ public abstract class AbstractPreUpdateCustomizer implements TableCustomizerInte
 
 
 
-   /*******************************************************************************
-    **
-    *******************************************************************************/
-   @Override
-   public List<QRecord> preUpdate(UpdateInput updateInput, List<QRecord> records, boolean isPreview, Optional<List<QRecord>> oldRecordList) throws QException
-   {
-      this.updateInput = updateInput;
-      this.isPreview = isPreview;
-      this.oldRecordList = oldRecordList.orElse(null);
-      return apply(records);
-   }
-
-
-
    /*******************************************************************************
     **
     *******************************************************************************/

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/QCodeLoader.java

@@ -22,7 +22,6 @@
 package com.kingsrook.qqq.backend.core.actions.customizers;
 
 
-import java.lang.reflect.Constructor;
 import java.util.Optional;
 import java.util.function.Function;
 import com.kingsrook.qqq.backend.core.actions.automation.RecordAutomationHandler;
@@ -35,35 +34,42 @@ import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeType;
 import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSource;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.TableAutomationAction;
-import com.kingsrook.qqq.backend.core.utils.lambdas.UnsafeFunction;
-import com.kingsrook.qqq.backend.core.utils.memoization.Memoization;
 import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
 
 
 /*******************************************************************************
  ** Utility to load code for running QQQ customizers.
- **
- ** TODO - redo all to go through method that memoizes class & constructor
- ** lookup.  That memoziation causes 1,000,000 such calls to go from ~500ms
- ** to ~100ms.
  *******************************************************************************/
 public class QCodeLoader
 {
    private static final QLogger LOG = QLogger.getLogger(QCodeLoader.class);
 
-   private static Memoization<String, Constructor<?>> constructorMemoization = new Memoization<>();
-
 
 
    /*******************************************************************************
     **
     *******************************************************************************/
-   public static Optional<TableCustomizerInterface> getTableCustomizer(QTableMetaData table, String customizerName)
+   public static <T, R> Optional<Function<T, R>> getTableCustomizerFunction(QTableMetaData table, String customizerName)
    {
       Optional<QCodeReference> codeReference = table.getCustomizer(customizerName);
       if(codeReference.isPresent())
       {
-         return (Optional.ofNullable(QCodeLoader.getAdHoc(TableCustomizerInterface.class, codeReference.get())));
+         return (Optional.ofNullable(QCodeLoader.getFunction(codeReference.get())));
+      }
+      return (Optional.empty());
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public static <T> Optional<T> getTableCustomizer(Class<T> expectedClass, QTableMetaData table, String customizerName)
+   {
+      Optional<QCodeReference> codeReference = table.getCustomizer(customizerName);
+      if(codeReference.isPresent())
+      {
+         return (Optional.ofNullable(QCodeLoader.getAdHoc(expectedClass, codeReference.get())));
       }
       return (Optional.empty());
    }
@@ -96,7 +102,7 @@ public class QCodeLoader
       }
       catch(Exception e)
       {
-         LOG.error("Error initializing customizer", e, logPair("codeReference", codeReference));
+         LOG.error("Error initializing customizer", logPair("codeReference", codeReference), e);
 
          //////////////////////////////////////////////////////////////////////////////////////////////////////////
          // return null here - under the assumption that during normal run-time operations, we'll never hit here //
@@ -135,7 +141,7 @@ public class QCodeLoader
       }
       catch(Exception e)
       {
-         LOG.error("Error initializing customizer", e, logPair("codeReference", codeReference));
+         LOG.error("Error initializing customizer", logPair("codeReference", codeReference), e);
 
          //////////////////////////////////////////////////////////////////////////////////////////////////////////
          // return null here - under the assumption that during normal run-time operations, we'll never hit here //
@@ -169,25 +175,12 @@ public class QCodeLoader
 
       try
       {
-         Optional<Constructor<?>> constructor = constructorMemoization.getResultThrowing(codeReference.getName(), (UnsafeFunction<String, Constructor<?>, Exception>) s ->
-         {
-            Class<?> customizerClass = Class.forName(codeReference.getName());
-            return customizerClass.getConstructor();
-         });
-
-         if(constructor.isPresent())
-         {
-            return ((T) constructor.get().newInstance());
-         }
-         else
-         {
-            LOG.error("Could not get constructor for code reference", logPair("codeReference", codeReference));
-            return (null);
-         }
+         Class<?> customizerClass = Class.forName(codeReference.getName());
+         return ((T) customizerClass.getConstructor().newInstance());
       }
       catch(Exception e)
       {
-         LOG.error("Error initializing customizer", e, logPair("codeReference", codeReference));
+         LOG.error("Error initializing customizer", logPair("codeReference", codeReference), e);
 
          //////////////////////////////////////////////////////////////////////////////////////////////////////////
          // return null here - under the assumption that during normal run-time operations, we'll never hit here //

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/RecordCustomizerUtilityInterface.java

@@ -50,7 +50,10 @@ public interface RecordCustomizerUtilityInterface
    /*******************************************************************************
     ** Container for an old value and a new value.
     *******************************************************************************/
-   record Change(Serializable oldValue, Serializable newValue) {}
+   @SuppressWarnings("checkstyle:MethodName")
+   record Change(Serializable oldValue, Serializable newValue)
+   {
+   }
 
 
    /*******************************************************************************

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/TableCustomizerInterface.java

@@ -1,202 +0,0 @@
-/*
- * QQQ - Low-code Application Framework for Engineers.
- * Copyright (C) 2021-2024.  Kingsrook, LLC
- * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
- * contact@kingsrook.com
- * https://github.com/Kingsrook/
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program.  If not, see <https://www.gnu.org/licenses/>.
- */
-
-package com.kingsrook.qqq.backend.core.actions.customizers;
-
-
-import java.util.List;
-import java.util.Optional;
-import com.kingsrook.qqq.backend.core.exceptions.QException;
-import com.kingsrook.qqq.backend.core.logging.QLogger;
-import com.kingsrook.qqq.backend.core.model.actions.tables.QueryOrGetInputInterface;
-import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput;
-import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
-import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
-import com.kingsrook.qqq.backend.core.model.data.QRecord;
-import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
-
-
-/*******************************************************************************
- ** Common interface used by all (core) TableCustomizer types (e.g., post-query,
- ** and {pre,post}-{insert,update,delete}.
- **
- ** Note that the abstract-base classes for each action still exist, though have
- ** been back-ported to be implementors of this interface.  The action classes
- ** will now expect this type, and call this type's methods.
- **
- *******************************************************************************/
-public interface TableCustomizerInterface
-{
-   QLogger LOG = QLogger.getLogger(TableCustomizerInterface.class);
-
-
-   /*******************************************************************************
-    ** custom actions to run after a query (or get!) takes place.
-    **
-    *******************************************************************************/
-   default List<QRecord> postQuery(QueryOrGetInputInterface queryInput, List<QRecord> records) throws QException
-   {
-      LOG.info("A default implementation of postQuery is running...  Probably not expected!", logPair("tableName", queryInput.getTableName()));
-      return (records);
-   }
-
-
-   /*******************************************************************************
-    ** custom actions before an insert takes place.
-    **
-    ** It's important for implementations to be aware of the isPreview field, which
-    ** is set to true when the code is running to give users advice, e.g., on a review
-    ** screen - vs. being false when the action is ACTUALLY happening.  So, if you're doing
-    ** things like storing data, you don't want to do that if isPreview is true!!
-    **
-    ** General implementation would be, to iterate over the records (the inputs to
-    ** the insert action), and look at their values:
-    ** - possibly adding Errors (`addError`) or Warnings (`addWarning`) to the records
-    ** - possibly manipulating values (`setValue`)
-    ** - possibly throwing an exception - if you really don't want the insert operation to continue.
-    ** - doing "whatever else" you may want to do.
-    ** - returning the list of records (can be the input list) that you want to go on to the backend implementation class.
-    *******************************************************************************/
-   default List<QRecord> preInsert(InsertInput insertInput, List<QRecord> records, boolean isPreview) throws QException
-   {
-      LOG.info("A default implementation of preInsert is running...  Probably not expected!", logPair("tableName", insertInput.getTableName()));
-      return (records);
-   }
-
-
-   /*******************************************************************************
-    **
-    *******************************************************************************/
-   default AbstractPreInsertCustomizer.WhenToRun whenToRunPreInsert(InsertInput insertInput, boolean isPreview)
-   {
-      return (AbstractPreInsertCustomizer.WhenToRun.AFTER_ALL_VALIDATIONS);
-   }
-
-
-   /*******************************************************************************
-    ** custom actions after an insert takes place.
-    **
-    ** General implementation would be, to iterate over the records (the outputs of
-    ** the insert action), and look at their values:
-    ** - possibly adding Errors (`addError`) or Warnings (`addWarning`) to the records
-    ** - possibly throwing an exception - though doing so won't stop the update, and instead
-    **   will just set a warning on all of the updated records...
-    ** - doing "whatever else" you may want to do.
-    ** - returning the list of records (can be the input list) that you want to go back to the caller.
-    *******************************************************************************/
-   default List<QRecord> postInsert(InsertInput insertInput, List<QRecord> records) throws QException
-   {
-      LOG.info("A default implementation of postInsert is running...  Probably not expected!", logPair("tableName", insertInput.getTableName()));
-      return (records);
-   }
-
-
-   /*******************************************************************************
-    ** custom actions before an update takes place.
-    **
-    ** It's important for implementations to be aware of the isPreview field, which
-    ** is set to true when the code is running to give users advice, e.g., on a review
-    ** screen - vs. being false when the action is ACTUALLY happening.  So, if you're doing
-    ** things like storing data, you don't want to do that if isPreview is true!!
-    **
-    ** General implementation would be, to iterate over the records (the inputs to
-    ** the update action), and look at their values:
-    ** - possibly adding Errors (`addError`) or Warnings (`addWarning`) to the records
-    ** - possibly manipulating values (`setValue`)
-    ** - possibly throwing an exception - if you really don't want the update operation to continue.
-    ** - doing "whatever else" you may want to do.
-    ** - returning the list of records (can be the input list) that you want to go on to the backend implementation class.
-    **
-    ** Note, "old records" (e.g., with values freshly fetched from the backend) will be
-    ** available (if the backend supports it)
-    *******************************************************************************/
-   default List<QRecord> preUpdate(UpdateInput updateInput, List<QRecord> records, boolean isPreview, Optional<List<QRecord>> oldRecordList) throws QException
-   {
-      LOG.info("A default implementation of preUpdate is running...  Probably not expected!", logPair("tableName", updateInput.getTableName()));
-      return (records);
-   }
-
-
-   /*******************************************************************************
-    ** custom actions after an update takes place.
-    **
-    ** General implementation would be, to iterate over the records (the outputs of
-    ** the update action), and look at their values:
-    ** - possibly adding Errors (`addError`) or Warnings (`addWarning`) to the records?
-    ** - possibly throwing an exception - though doing so won't stop the update, and instead
-    **   will just set a warning on all of the updated records...
-    ** - doing "whatever else" you may want to do.
-    ** - returning the list of records (can be the input list) that you want to go back to the caller.
-    **
-    ** Note, "old records" (e.g., with values freshly fetched from the backend) will be
-    ** available (if the backend supports it).
-    *******************************************************************************/
-   default List<QRecord> postUpdate(UpdateInput updateInput, List<QRecord> records, Optional<List<QRecord>> oldRecordList) throws QException
-   {
-      LOG.info("A default implementation of postUpdate is running...  Probably not expected!", logPair("tableName", updateInput.getTableName()));
-      return (records);
-   }
-
-
-   /*******************************************************************************
-    ** Custom actions before a delete takes place.
-    **
-    ** It's important for implementations to be aware of the isPreview param, which
-    ** is set to true when the code is running to give users advice, e.g., on a review
-    ** screen - vs. being false when the action is ACTUALLY happening.  So, if you're doing
-    ** things like storing data, you don't want to do that if isPreview is true!!
-    **
-    ** General implementation would be, to iterate over the records (which the DeleteAction
-    ** would look up based on the inputs to the delete action), and look at their values:
-    ** - possibly adding Errors (`addError`) or Warnings (`addWarning`) to the records
-    ** - possibly throwing an exception - if you really don't want the delete operation to continue.
-    ** - doing "whatever else" you may want to do.
-    ** - returning the list of records (can be the input list) - this is how errors
-    **   and warnings are propagated to the DeleteAction.  Note that any records with
-    **   an error will NOT proceed to the backend's delete interface - but those with
-    **   warnings will.
-    *******************************************************************************/
-   default List<QRecord> preDelete(DeleteInput deleteInput, List<QRecord> records, boolean isPreview) throws QException
-   {
-      LOG.info("A default implementation of preDelete is running...  Probably not expected!", logPair("tableName", deleteInput.getTableName()));
-      return (records);
-   }
-
-
-   /*******************************************************************************
-    ** Custom actions after a delete takes place.
-    **
-    ** General implementation would be, to iterate over the records (ones which didn't
-    ** have a delete error), and look at their values:
-    ** - possibly adding Errors (`addError`) or Warnings (`addWarning`) to the records?
-    ** - possibly throwing an exception - though doing so won't stop the delete, and instead
-    **   will just set a warning on all of the deleted records...
-    ** - doing "whatever else" you may want to do.
-    ** - returning the list of records (can be the input list) that you want to go back
-    **   to the caller - this is how errors and warnings are propagated .
-    *******************************************************************************/
-   default List<QRecord> postDelete(DeleteInput deleteInput, List<QRecord> records) throws QException
-   {
-      LOG.info("A default implementation of postDelete is running...  Probably not expected!", logPair("tableName", deleteInput.getTableName()));
-      return (records);
-   }
-
-}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/TableCustomizers.java

@@ -29,13 +29,13 @@ package com.kingsrook.qqq.backend.core.actions.customizers;
  *******************************************************************************/
 public enum TableCustomizers
 {
-   POST_QUERY_RECORD("postQueryRecord", TableCustomizerInterface.class),
-   PRE_INSERT_RECORD("preInsertRecord", TableCustomizerInterface.class),
-   POST_INSERT_RECORD("postInsertRecord", TableCustomizerInterface.class),
-   PRE_UPDATE_RECORD("preUpdateRecord", TableCustomizerInterface.class),
-   POST_UPDATE_RECORD("postUpdateRecord", TableCustomizerInterface.class),
-   PRE_DELETE_RECORD("preDeleteRecord", TableCustomizerInterface.class),
-   POST_DELETE_RECORD("postDeleteRecord", TableCustomizerInterface.class);
+   POST_QUERY_RECORD("postQueryRecord", AbstractPostQueryCustomizer.class),
+   PRE_INSERT_RECORD("preInsertRecord", AbstractPreInsertCustomizer.class),
+   POST_INSERT_RECORD("postInsertRecord", AbstractPostInsertCustomizer.class),
+   PRE_UPDATE_RECORD("preUpdateRecord", AbstractPreUpdateCustomizer.class),
+   POST_UPDATE_RECORD("postUpdateRecord", AbstractPostUpdateCustomizer.class),
+   PRE_DELETE_RECORD("preDeleteRecord", AbstractPreDeleteCustomizer.class),
+   POST_DELETE_RECORD("postDeleteRecord", AbstractPostDeleteCustomizer.class);
 
 
    private final String   role;

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/dashboard/AbstractHTMLWidgetRenderer.java

@@ -41,7 +41,6 @@ import com.kingsrook.qqq.backend.core.model.actions.widgets.RenderWidgetInput;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.DisplayFormat;
 import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
 import com.kingsrook.qqq.backend.core.utils.JsonUtils;
-import com.kingsrook.qqq.backend.core.utils.QQueryFilterDeduper;
 import com.kingsrook.qqq.backend.core.utils.StringUtils;
 
 
@@ -161,7 +160,7 @@ public abstract class AbstractHTMLWidgetRenderer extends AbstractWidgetRenderer
    public static String linkTableCreateWithDefaultValues(RenderWidgetInput input, String tableName, Map<String, Serializable> defaultValues) throws QException
    {
       String tablePath = QContext.getQInstance().getTablePath(tableName);
-      return (tablePath + "/create#defaultValues=" + URLEncoder.encode(JsonUtils.toJson(defaultValues), Charset.defaultCharset()));
+      return (tablePath + "/create?defaultValues=" + URLEncoder.encode(JsonUtils.toJson(defaultValues), Charset.defaultCharset()));
    }
 
 
@@ -177,7 +176,6 @@ public abstract class AbstractHTMLWidgetRenderer extends AbstractWidgetRenderer
       {
          return (totalString);
       }
-      filter = QQueryFilterDeduper.dedupeFilter(filter);
       return ("<a href='" + tablePath + "?filter=" + JsonUtils.toJson(filter) + "'>" + totalString + "</a>");
    }
 
@@ -194,7 +192,6 @@ public abstract class AbstractHTMLWidgetRenderer extends AbstractWidgetRenderer
          return;
       }
 
-      filter = QQueryFilterDeduper.dedupeFilter(filter);
       urls.add(tablePath + "?filter=" + JsonUtils.toJson(filter));
    }
 
@@ -211,7 +208,6 @@ public abstract class AbstractHTMLWidgetRenderer extends AbstractWidgetRenderer
          return (null);
       }
 
-      filter = QQueryFilterDeduper.dedupeFilter(filter);
       return (tablePath + "?filter=" + JsonUtils.toJson(filter));
    }
 
@@ -228,7 +224,6 @@ public abstract class AbstractHTMLWidgetRenderer extends AbstractWidgetRenderer
          return (null);
       }
 
-      filter = QQueryFilterDeduper.dedupeFilter(filter);
       return (tablePath + "?filter=" + URLEncoder.encode(JsonUtils.toJson(filter), Charset.defaultCharset()));
    }
 
@@ -331,7 +326,6 @@ public abstract class AbstractHTMLWidgetRenderer extends AbstractWidgetRenderer
       }
 
       String tablePath = QContext.getQInstance().getTablePath(tableName);
-      filter = QQueryFilterDeduper.dedupeFilter(filter);
       return (tablePath + "/" + processName + "?recordsParam=filterJSON&filterJSON=" + URLEncoder.encode(JsonUtils.toJson(filter), StandardCharsets.UTF_8));
    }
 

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/dashboard/widgets/AbstractWidgetRenderer.java

@@ -42,7 +42,6 @@ import com.kingsrook.qqq.backend.core.model.actions.widgets.RenderWidgetOutput;
 import com.kingsrook.qqq.backend.core.model.dashboard.widgets.QWidgetData;
 import com.kingsrook.qqq.backend.core.model.metadata.dashboard.QWidgetMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.dashboard.WidgetDropdownData;
-import com.kingsrook.qqq.backend.core.model.metadata.dashboard.WidgetDropdownType;
 import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValue;
 import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSource;
 import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
@@ -73,104 +72,80 @@ public abstract class AbstractWidgetRenderer
     *******************************************************************************/
    protected boolean setupDropdowns(RenderWidgetInput input, QWidgetMetaData metaData, QWidgetData widgetData) throws QException
    {
-      List<List<Map<String, String>>> dataList                  = new ArrayList<>();
-      List<String>                    labelList                 = new ArrayList<>();
-      List<String>                    nameList                  = new ArrayList<>();
+      List<List<Map<String, String>>> pvsData                   = new ArrayList<>();
+      List<String>                    pvsLabels                 = new ArrayList<>();
+      List<String>                    pvsNames                  = new ArrayList<>();
       List<String>                    missingRequiredSelections = new ArrayList<>();
       for(WidgetDropdownData dropdownData : CollectionUtils.nonNullList(metaData.getDropdowns()))
       {
-         if(WidgetDropdownType.DATE_PICKER.equals(dropdownData.getType()))
-         {
-            String name = dropdownData.getName();
-            nameList.add(name);
-            labelList.add(dropdownData.getLabel());
-            dataList.add(new ArrayList<>());
+         String               possibleValueSourceName = dropdownData.getPossibleValueSourceName();
+         QPossibleValueSource possibleValueSource     = input.getInstance().getPossibleValueSource(possibleValueSourceName);
 
-            ////////////////////////////////////////////////////////////////////////////////////////////////////////////
-            // sure that something has been selected, and if not, display a message that a selection needs made       //
-            ////////////////////////////////////////////////////////////////////////////////////////////////////////////
-            if(dropdownData.getIsRequired())
+         /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+         // this looks complicated, but is just look for a label in the dropdown data and if found use it,                                                                                  //
+         // otherwise look for label in PVS and if found use that, otherwise just use the PVS name                                                                                          //
+         /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+         String pvsLabel = dropdownData.getLabel() != null ? dropdownData.getLabel() : (possibleValueSource.getLabel() != null ? possibleValueSource.getLabel() : possibleValueSourceName);
+         pvsLabels.add(pvsLabel);
+         pvsNames.add(possibleValueSourceName);
+
+         SearchPossibleValueSourceInput pvsInput = new SearchPossibleValueSourceInput();
+         pvsInput.setPossibleValueSourceName(possibleValueSourceName);
+
+         if(dropdownData.getForeignKeyFieldName() != null)
+         {
+            ////////////////////////////////////////
+            // look for an id in the query params //
+            ////////////////////////////////////////
+            Integer id = null;
+            if(input.getQueryParams() != null && input.getQueryParams().containsKey("id") && StringUtils.hasContent(input.getQueryParams().get("id")))
             {
-               if(!input.getQueryParams().containsKey(name) || !StringUtils.hasContent(input.getQueryParams().get(name)))
-               {
-                  missingRequiredSelections.add(dropdownData.getLabel());
-               }
+               id = Integer.parseInt(input.getQueryParams().get("id"));
+            }
+            if(id != null)
+            {
+               pvsInput.setDefaultQueryFilter(new QQueryFilter().withCriteria(
+                  new QFilterCriteria(
+                     dropdownData.getForeignKeyFieldName(),
+                     QCriteriaOperator.EQUALS,
+                     id)));
             }
          }
-         else
+
+         SearchPossibleValueSourceOutput output = new SearchPossibleValueSourceAction().execute(pvsInput);
+
+         List<Map<String, String>> dropdownOptionList = new ArrayList<>();
+         pvsData.add(dropdownOptionList);
+
+         //////////////////////////////////////////
+         // sort results, dedupe, and add to map //
+         //////////////////////////////////////////
+         Set<String> exists = new HashSet<>();
+         output.getResults().removeIf(pvs -> !exists.add(pvs.getLabel()));
+         for(QPossibleValue<?> possibleValue : output.getResults())
          {
-            String possibleValueSourceName = dropdownData.getPossibleValueSourceName();
-            if(possibleValueSourceName != null)
+            dropdownOptionList.add(MapBuilder.of(
+               "id", String.valueOf(possibleValue.getId()),
+               "label", possibleValue.getLabel()
+            ));
+         }
+
+         ////////////////////////////////////////////////////////////////////////////////////////////////////////////
+         // because we know the dropdowns and what the field names will be when something is selected, we can make //
+         // sure that something has been selected, and if not, display a message that a selection needs made       //
+         ////////////////////////////////////////////////////////////////////////////////////////////////////////////
+         if(dropdownData.getIsRequired())
+         {
+            if(!input.getQueryParams().containsKey(possibleValueSourceName) || !StringUtils.hasContent(input.getQueryParams().get(possibleValueSourceName)))
             {
-               QPossibleValueSource possibleValueSource = input.getInstance().getPossibleValueSource(possibleValueSourceName);
-
-               /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-               // this looks complicated, but is just look for a label in the dropdown data and if found use it,                                                                                  //
-               // otherwise look for label in PVS and if found use that, otherwise just use the PVS name                                                                                          //
-               /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-               String pvsLabel = dropdownData.getLabel() != null ? dropdownData.getLabel() : (possibleValueSource.getLabel() != null ? possibleValueSource.getLabel() : possibleValueSourceName);
-               labelList.add(pvsLabel);
-               nameList.add(possibleValueSourceName);
-
-               SearchPossibleValueSourceInput pvsInput = new SearchPossibleValueSourceInput();
-               pvsInput.setPossibleValueSourceName(possibleValueSourceName);
-
-               if(dropdownData.getForeignKeyFieldName() != null)
-               {
-                  ////////////////////////////////////////
-                  // look for an id in the query params //
-                  ////////////////////////////////////////
-                  Integer id = null;
-                  if(input.getQueryParams() != null && input.getQueryParams().containsKey("id") && StringUtils.hasContent(input.getQueryParams().get("id")))
-                  {
-                     id = Integer.parseInt(input.getQueryParams().get("id"));
-                  }
-                  if(id != null)
-                  {
-                     pvsInput.setDefaultQueryFilter(new QQueryFilter().withCriteria(
-                        new QFilterCriteria(
-                           dropdownData.getForeignKeyFieldName(),
-                           QCriteriaOperator.EQUALS,
-                           id)));
-                  }
-               }
-
-               SearchPossibleValueSourceOutput output = new SearchPossibleValueSourceAction().execute(pvsInput);
-
-               List<Map<String, String>> dropdownOptionList = new ArrayList<>();
-               dataList.add(dropdownOptionList);
-
-               //////////////////////////////////////////
-               // sort results, dedupe, and add to map //
-               //////////////////////////////////////////
-               Set<String> exists = new HashSet<>();
-               output.getResults().removeIf(pvs -> !exists.add(pvs.getLabel()));
-               for(QPossibleValue<?> possibleValue : output.getResults())
-               {
-                  dropdownOptionList.add(MapBuilder.of(
-                     "id", String.valueOf(possibleValue.getId()),
-                     "label", possibleValue.getLabel()
-                  ));
-               }
-
-               ////////////////////////////////////////////////////////////////////////////////////////////////////////////
-               // because we know the dropdowns and what the field names will be when something is selected, we can make //
-               // sure that something has been selected, and if not, display a message that a selection needs made       //
-               ////////////////////////////////////////////////////////////////////////////////////////////////////////////
-               if(dropdownData.getIsRequired())
-               {
-                  if(!input.getQueryParams().containsKey(possibleValueSourceName) || !StringUtils.hasContent(input.getQueryParams().get(possibleValueSourceName)))
-                  {
-                     missingRequiredSelections.add(pvsLabel);
-                  }
-               }
+               missingRequiredSelections.add(pvsLabel);
             }
          }
       }
 
-      widgetData.setDropdownNameList(nameList);
-      widgetData.setDropdownLabelList(labelList);
-      widgetData.setDropdownDataList(dataList);
+      widgetData.setDropdownNameList(pvsNames);
+      widgetData.setDropdownLabelList(pvsLabels);
+      widgetData.setDropdownDataList(pvsData);
 
       ////////////////////////////////////////////////////////////////////////////////
       // if there are any missing required dropdowns, build up a message to display //

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/dashboard/widgets/Aggregate2DTableWidgetRenderer.java

@@ -1,202 +0,0 @@
-/*
- * QQQ - Low-code Application Framework for Engineers.
- * Copyright (C) 2021-2024.  Kingsrook, LLC
- * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
- * contact@kingsrook.com
- * https://github.com/Kingsrook/
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program.  If not, see <https://www.gnu.org/licenses/>.
- */
-
-package com.kingsrook.qqq.backend.core.actions.dashboard.widgets;
-
-
-import java.io.Serializable;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.LinkedHashMap;
-import java.util.LinkedHashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Objects;
-import java.util.Set;
-import com.kingsrook.qqq.backend.core.actions.tables.AggregateAction;
-import com.kingsrook.qqq.backend.core.context.QContext;
-import com.kingsrook.qqq.backend.core.exceptions.QException;
-import com.kingsrook.qqq.backend.core.logging.QLogger;
-import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.Aggregate;
-import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateInput;
-import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateOperator;
-import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateOutput;
-import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateResult;
-import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.GroupBy;
-import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.QFilterOrderByAggregate;
-import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.QFilterOrderByGroupBy;
-import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
-import com.kingsrook.qqq.backend.core.model.actions.widgets.RenderWidgetInput;
-import com.kingsrook.qqq.backend.core.model.actions.widgets.RenderWidgetOutput;
-import com.kingsrook.qqq.backend.core.model.dashboard.widgets.TableData;
-import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
-import com.kingsrook.qqq.backend.core.utils.StringUtils;
-import com.kingsrook.qqq.backend.core.utils.ValueUtils;
-
-
-/*******************************************************************************
- ** Generic widget that does an aggregate query, and presents its results
- ** as a table, using group-by values as both row & column labels.
- *******************************************************************************/
-public class Aggregate2DTableWidgetRenderer extends AbstractWidgetRenderer
-{
-   private static final QLogger LOG = QLogger.getLogger(Aggregate2DTableWidgetRenderer.class);
-
-
-
-   /*******************************************************************************
-    **
-    *******************************************************************************/
-   @Override
-   public RenderWidgetOutput render(RenderWidgetInput input) throws QException
-   {
-      Map<String, Serializable> values = input.getWidgetMetaData().getDefaultValues();
-
-      String         tableName   = ValueUtils.getValueAsString(values.get("tableName"));
-      String         valueField  = ValueUtils.getValueAsString(values.get("valueField"));
-      String         rowField    = ValueUtils.getValueAsString(values.get("rowField"));
-      String         columnField = ValueUtils.getValueAsString(values.get("columnField"));
-      QTableMetaData table       = QContext.getQInstance().getTable(tableName);
-
-      AggregateInput aggregateInput = new AggregateInput();
-      aggregateInput.setTableName(tableName);
-
-      // todo - allow input of "list of columns" (e.g., in case some miss sometimes, or as a version of filter)
-      // todo - max rows, max cols?
-
-      // todo - from input map
-      QQueryFilter filter = new QQueryFilter();
-      aggregateInput.setFilter(filter);
-
-      Aggregate aggregate = new Aggregate(valueField, AggregateOperator.COUNT);
-      aggregateInput.withAggregate(aggregate);
-
-      GroupBy rowGroupBy    = new GroupBy(table.getField(rowField));
-      GroupBy columnGroupBy = new GroupBy(table.getField(columnField));
-      aggregateInput.withGroupBy(rowGroupBy);
-      aggregateInput.withGroupBy(columnGroupBy);
-
-      String orderBys = ValueUtils.getValueAsString(values.get("orderBys"));
-      if(StringUtils.hasContent(orderBys))
-      {
-         for(String orderBy : orderBys.split(","))
-         {
-            switch(orderBy)
-            {
-               case "row" -> filter.addOrderBy(new QFilterOrderByGroupBy(rowGroupBy));
-               case "column" -> filter.addOrderBy(new QFilterOrderByGroupBy(columnGroupBy));
-               case "value" -> filter.addOrderBy(new QFilterOrderByAggregate(aggregate));
-               default -> LOG.warn("Unrecognized orderBy: " + orderBy);
-            }
-         }
-      }
-
-      AggregateOutput aggregateOutput = new AggregateAction().execute(aggregateInput);
-
-      Map<Serializable, Map<Serializable, Serializable>> data       = new LinkedHashMap<>();
-      Set<Serializable>                                  columnsSet = new LinkedHashSet<>();
-
-      for(AggregateResult result : aggregateOutput.getResults())
-      {
-         Serializable column = result.getGroupByValue(columnGroupBy);
-         Serializable row    = result.getGroupByValue(rowGroupBy);
-         Serializable value  = result.getAggregateValue(aggregate);
-
-         Map<Serializable, Serializable> rowMap = data.computeIfAbsent(row, (k) -> new LinkedHashMap<>());
-         rowMap.put(column, value);
-         columnsSet.add(column);
-      }
-
-      // todo - possible values from rows, cols
-
-      ////////////////////////////////////
-      // setup datastructures for table //
-      ////////////////////////////////////
-      List<Map<String, Object>> tableRows    = new ArrayList<>();
-      List<TableData.Column>    tableColumns = new ArrayList<>();
-      tableColumns.add(new TableData.Column("default", table.getField(rowField).getLabel(), "_row", "2fr", "left"));
-
-      for(Serializable column : columnsSet)
-      {
-         tableColumns.add(new TableData.Column("default", String.valueOf(column) /* todo display value */, String.valueOf(column), "1fr", "right"));
-      }
-
-      tableColumns.add(new TableData.Column("default", "Total", "_total", "1fr", "right"));
-
-      TableData tableData = new TableData(null, tableColumns, tableRows)
-         .withRowsPerPage(100)
-         .withFixedStickyLastRow(false)
-         .withHidePaginationDropdown(true);
-
-      Map<Serializable, Integer> columnSums = new HashMap<>();
-      int                        grandTotal = 0;
-      for(Map.Entry<Serializable, Map<Serializable, Serializable>> rowEntry : data.entrySet())
-      {
-         Map<String, Object> rowMap = new HashMap<>();
-         tableRows.add(rowMap);
-
-         rowMap.put("_row", rowEntry.getKey() /* todo display value */);
-         int rowTotal = 0;
-         for(Serializable column : columnsSet)
-         {
-            Serializable value = rowEntry.getValue().get(column);
-            if(value == null)
-            {
-               value = 0; // todo?
-            }
-
-            Integer valueAsInteger = Objects.requireNonNullElse(ValueUtils.getValueAsInteger(value), 0);
-            rowTotal += valueAsInteger;
-            columnSums.putIfAbsent(column, 0);
-            columnSums.put(column, columnSums.get(column) + valueAsInteger);
-
-            rowMap.put(String.valueOf(column), value); // todo format commas?
-         }
-
-         rowMap.put("_total", rowTotal);
-         grandTotal += rowTotal;
-      }
-
-      ///////////////
-      // total row //
-      ///////////////
-      Map<String, Object> totalRowMap = new HashMap<>();
-      tableRows.add(totalRowMap);
-
-      totalRowMap.put("_row", "Total");
-      int rowTotal = 0;
-      for(Serializable column : columnsSet)
-      {
-         Serializable value = columnSums.get(column);
-         if(value == null)
-         {
-            value = 0; // todo?
-         }
-
-         totalRowMap.put(String.valueOf(column), value); // todo format commas?
-      }
-
-      totalRowMap.put("_total", grandTotal);
-
-      return (new RenderWidgetOutput(tableData));
-   }
-
-}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/dashboard/widgets/ChildRecordListRenderer.java

@@ -36,7 +36,6 @@ import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
 import com.kingsrook.qqq.backend.core.actions.values.QValueFormatter;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.exceptions.QNotFoundException;
-import com.kingsrook.qqq.backend.core.logging.QLogger;
 import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetOutput;
@@ -60,7 +59,6 @@ import com.kingsrook.qqq.backend.core.utils.JsonUtils;
 import com.kingsrook.qqq.backend.core.utils.StringUtils;
 import com.kingsrook.qqq.backend.core.utils.ValueUtils;
 import org.apache.commons.lang.BooleanUtils;
-import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
 
 
 /*******************************************************************************
@@ -68,9 +66,6 @@ import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
  *******************************************************************************/
 public class ChildRecordListRenderer extends AbstractWidgetRenderer
 {
-   private static final QLogger LOG = QLogger.getLogger(ChildRecordListRenderer.class);
-
-
 
    /*******************************************************************************
     **
@@ -142,7 +137,7 @@ public class ChildRecordListRenderer extends AbstractWidgetRenderer
        *******************************************************************************/
       public Builder withCanAddChildRecord(boolean b)
       {
-         widgetMetaData.withDefaultValue("canAddChildRecord", b);
+         widgetMetaData.withDefaultValue("canAddChildRecord", true);
          return (this);
       }
 
@@ -156,17 +151,6 @@ public class ChildRecordListRenderer extends AbstractWidgetRenderer
          widgetMetaData.withDefaultValue("disabledFieldsForNewChildRecords", new HashSet<>(disabledFieldsForNewChildRecords));
          return (this);
       }
-
-
-
-      /*******************************************************************************
-       **
-       *******************************************************************************/
-      public Builder withManageAssociationName(String manageAssociationName)
-      {
-         widgetMetaData.withDefaultValue("manageAssociationName", manageAssociationName);
-         return (this);
-      }
    }
 
 
@@ -177,134 +161,98 @@ public class ChildRecordListRenderer extends AbstractWidgetRenderer
    @Override
    public RenderWidgetOutput render(RenderWidgetInput input) throws QException
    {
-      try
+      String         widgetLabel = input.getQueryParams().get("widgetLabel");
+      String         joinName    = input.getQueryParams().get("joinName");
+      QJoinMetaData  join        = input.getInstance().getJoin(joinName);
+      String         id          = input.getQueryParams().get("id");
+      QTableMetaData leftTable   = input.getInstance().getTable(join.getLeftTable());
+      QTableMetaData rightTable  = input.getInstance().getTable(join.getRightTable());
+
+      Integer maxRows = null;
+      if(StringUtils.hasContent(input.getQueryParams().get("maxRows")))
       {
-         String         widgetLabel = input.getQueryParams().get("widgetLabel");
-         String         joinName    = input.getQueryParams().get("joinName");
-         QJoinMetaData  join        = input.getInstance().getJoin(joinName);
-         String         id          = input.getQueryParams().get("id");
-         QTableMetaData leftTable   = input.getInstance().getTable(join.getLeftTable());
-         QTableMetaData rightTable  = input.getInstance().getTable(join.getRightTable());
-
-         Integer maxRows = null;
-         if(StringUtils.hasContent(input.getQueryParams().get("maxRows")))
-         {
-            maxRows = ValueUtils.getValueAsInteger(input.getQueryParams().get("maxRows"));
-         }
-         else if(input.getWidgetMetaData().getDefaultValues().containsKey("maxRows"))
-         {
-            maxRows = ValueUtils.getValueAsInteger(input.getWidgetMetaData().getDefaultValues().containsKey("maxRows"));
-         }
-
-         //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-         // fetch the record that we're getting children for. e.g., the left-side of the join, with the input id                                     //
-         // but - only try this if we were given an id.  note, this widget could be called for on an INSERT screen, where we don't have a record yet //
-         // but we still want to be able to return all the other data in here that otherwise comes from the widget meta data, join, etc.             //
-         //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-         int          totalRows     = 0;
-         QRecord      primaryRecord = null;
-         QQueryFilter filter        = null;
-         QueryOutput  queryOutput   = new QueryOutput(new QueryInput());
-         if(StringUtils.hasContent(id))
-         {
-            GetInput getInput = new GetInput();
-            getInput.setTableName(join.getLeftTable());
-            getInput.setPrimaryKey(id);
-            GetOutput getOutput = new GetAction().execute(getInput);
-            primaryRecord = getOutput.getRecord();
-
-            if(primaryRecord == null)
-            {
-               throw (new QNotFoundException("Could not find " + (leftTable == null ? "" : leftTable.getLabel()) + " with primary key " + id));
-            }
-
-            ////////////////////////////////////////////////////////////////////
-            // set up the query - for the table on the right side of the join //
-            ////////////////////////////////////////////////////////////////////
-            filter = new QQueryFilter();
-            for(JoinOn joinOn : join.getJoinOns())
-            {
-               filter.addCriteria(new QFilterCriteria(joinOn.getRightField(), QCriteriaOperator.EQUALS, List.of(primaryRecord.getValue(joinOn.getLeftField()))));
-            }
-            filter.setOrderBys(join.getOrderBys());
-            filter.setLimit(maxRows);
-
-            QueryInput queryInput = new QueryInput();
-            queryInput.setTableName(join.getRightTable());
-            queryInput.setShouldTranslatePossibleValues(true);
-            queryInput.setShouldGenerateDisplayValues(true);
-            queryInput.setFilter(filter);
-            queryOutput = new QueryAction().execute(queryInput);
-
-            QValueFormatter.setBlobValuesToDownloadUrls(rightTable, queryOutput.getRecords());
-
-            totalRows = queryOutput.getRecords().size();
-            if(maxRows != null && (queryOutput.getRecords().size() == maxRows))
-            {
-               /////////////////////////////////////////////////////////////////////////////////////
-               // if the input said to only do some max, and the # of results we got is that max, //
-               // then do a count query, for displaying 1-n of <count>                            //
-               /////////////////////////////////////////////////////////////////////////////////////
-               CountInput countInput = new CountInput();
-               countInput.setTableName(join.getRightTable());
-               countInput.setFilter(filter);
-               totalRows = new CountAction().execute(countInput).getCount();
-            }
-         }
-
-         String tablePath   = input.getInstance().getTablePath(rightTable.getName());
-         String viewAllLink = tablePath == null ? null : (tablePath + "?filter=" + URLEncoder.encode(JsonUtils.toJson(filter), Charset.defaultCharset()));
-
-         ChildRecordListData widgetData = new ChildRecordListData(widgetLabel, queryOutput, rightTable, tablePath, viewAllLink, totalRows);
-
-         if(BooleanUtils.isTrue(ValueUtils.getValueAsBoolean(input.getQueryParams().get("canAddChildRecord"))))
-         {
-            widgetData.setCanAddChildRecord(true);
-
-            //////////////////////////////////////////////////////////
-            // new child records must have values from the join-ons //
-            //////////////////////////////////////////////////////////
-            Map<String, Serializable> defaultValuesForNewChildRecords = new HashMap<>();
-            if(primaryRecord != null)
-            {
-               for(JoinOn joinOn : join.getJoinOns())
-               {
-                  defaultValuesForNewChildRecords.put(joinOn.getRightField(), primaryRecord.getValue(joinOn.getLeftField()));
-               }
-            }
-
-            widgetData.setDefaultValuesForNewChildRecords(defaultValuesForNewChildRecords);
-
-            Map<String, Serializable> widgetValues = input.getWidgetMetaData().getDefaultValues();
-            if(widgetValues.containsKey("disabledFieldsForNewChildRecords"))
-            {
-               widgetData.setDisabledFieldsForNewChildRecords((Set<String>) widgetValues.get("disabledFieldsForNewChildRecords"));
-            }
-            else
-            {
-               /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-               // if there are no disabled fields specified - then normally any fields w/ a default value get implicitly disabled //
-               // but - if we didn't look-up the primary record, then we'll want to explicit disable fields from joins            //
-               /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-               if(primaryRecord == null)
-               {
-                  Set<String> implicitlyDisabledFields = new HashSet<>();
-                  widgetData.setDisabledFieldsForNewChildRecords(implicitlyDisabledFields);
-                  for(JoinOn joinOn : join.getJoinOns())
-                  {
-                     implicitlyDisabledFields.add(joinOn.getRightField());
-                  }
-               }
-            }
-         }
-
-         return (new RenderWidgetOutput(widgetData));
+         maxRows = ValueUtils.getValueAsInteger(input.getQueryParams().get("maxRows"));
       }
-      catch(Exception e)
+      else if(input.getWidgetMetaData().getDefaultValues().containsKey("maxRows"))
       {
-         LOG.warn("Error rendering child record list", e, logPair("widgetName", () -> input.getWidgetMetaData().getName()));
-         throw (e);
+         maxRows = ValueUtils.getValueAsInteger(input.getWidgetMetaData().getDefaultValues().containsKey("maxRows"));
       }
+
+      ////////////////////////////////////////////////////////
+      // fetch the record that we're getting children for.  //
+      // e.g., the left-side of the join, with the input id //
+      ////////////////////////////////////////////////////////
+      GetInput getInput = new GetInput();
+      getInput.setTableName(join.getLeftTable());
+      getInput.setPrimaryKey(id);
+      GetOutput getOutput = new GetAction().execute(getInput);
+      QRecord   record    = getOutput.getRecord();
+
+      if(record == null)
+      {
+         throw (new QNotFoundException("Could not find " + (leftTable == null ? "" : leftTable.getLabel()) + " with primary key " + id));
+      }
+
+      ////////////////////////////////////////////////////////////////////
+      // set up the query - for the table on the right side of the join //
+      ////////////////////////////////////////////////////////////////////
+      QQueryFilter filter = new QQueryFilter();
+      for(JoinOn joinOn : join.getJoinOns())
+      {
+         filter.addCriteria(new QFilterCriteria(joinOn.getRightField(), QCriteriaOperator.EQUALS, List.of(record.getValue(joinOn.getLeftField()))));
+      }
+      filter.setOrderBys(join.getOrderBys());
+      filter.setLimit(maxRows);
+
+      QueryInput queryInput = new QueryInput();
+      queryInput.setTableName(join.getRightTable());
+      queryInput.setShouldTranslatePossibleValues(true);
+      queryInput.setShouldGenerateDisplayValues(true);
+      queryInput.setFilter(filter);
+      QueryOutput queryOutput = new QueryAction().execute(queryInput);
+
+      QValueFormatter.setBlobValuesToDownloadUrls(rightTable, queryOutput.getRecords());
+
+      int totalRows = queryOutput.getRecords().size();
+      if(maxRows != null && (queryOutput.getRecords().size() == maxRows))
+      {
+         /////////////////////////////////////////////////////////////////////////////////////
+         // if the input said to only do some max, and the # of results we got is that max, //
+         // then do a count query, for displaying 1-n of <count>                            //
+         /////////////////////////////////////////////////////////////////////////////////////
+         CountInput countInput = new CountInput();
+         countInput.setTableName(join.getRightTable());
+         countInput.setFilter(filter);
+         totalRows = new CountAction().execute(countInput).getCount();
+      }
+
+      String tablePath   = input.getInstance().getTablePath(rightTable.getName());
+      String viewAllLink = tablePath == null ? null : (tablePath + "?filter=" + URLEncoder.encode(JsonUtils.toJson(filter), Charset.defaultCharset()));
+
+      ChildRecordListData widgetData = new ChildRecordListData(widgetLabel, queryOutput, rightTable, tablePath, viewAllLink, totalRows);
+
+      if(BooleanUtils.isTrue(ValueUtils.getValueAsBoolean(input.getQueryParams().get("canAddChildRecord"))))
+      {
+         widgetData.setCanAddChildRecord(true);
+
+         //////////////////////////////////////////////////////////
+         // new child records must have values from the join-ons //
+         //////////////////////////////////////////////////////////
+         Map<String, Serializable> defaultValuesForNewChildRecords = new HashMap<>();
+         for(JoinOn joinOn : join.getJoinOns())
+         {
+            defaultValuesForNewChildRecords.put(joinOn.getRightField(), record.getValue(joinOn.getLeftField()));
+         }
+         widgetData.setDefaultValuesForNewChildRecords(defaultValuesForNewChildRecords);
+
+         Map<String, Serializable> widgetValues = input.getWidgetMetaData().getDefaultValues();
+         if(widgetValues.containsKey("disabledFieldsForNewChildRecords"))
+         {
+            widgetData.setDisabledFieldsForNewChildRecords((Set<String>) widgetValues.get("disabledFieldsForNewChildRecords"));
+         }
+      }
+
+      return (new RenderWidgetOutput(widgetData));
    }
 
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/dashboard/widgets/DateTimeGroupBy.java

@@ -256,6 +256,7 @@ public enum DateTimeGroupBy
    /*******************************************************************************
     **
     *******************************************************************************/
+   @SuppressWarnings("checkstyle:indentation")
    public Instant roundDown(Instant instant, ZoneId zoneId)
    {
       ZonedDateTime zoned = instant.atZone(zoneId);
@@ -296,21 +297,4 @@ public enum DateTimeGroupBy
       ZonedDateTime zoned = instant.atZone(zoneId);
       return (zoned.plus(noOfChronoUnitsToAdd, chronoUnitToAdd).toInstant());
    }
-
-
-
-   /*******************************************************************************
-    **
-    *******************************************************************************/
-   public static DateTimeFormatter sqlDateFormatToSelectedDateTimeFormatter(String sqlDateFormat)
-   {
-      for(DateTimeGroupBy value : values())
-      {
-         if(value.sqlDateFormat.equals(sqlDateFormat))
-         {
-            return (value.selectedStringFormatter);
-         }
-      }
-      return null;
-   }
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/dashboard/widgets/ProcessAlertWidget.java

@@ -1,86 +0,0 @@
-/*
- * QQQ - Low-code Application Framework for Engineers.
- * Copyright (C) 2021-2024.  Kingsrook, LLC
- * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
- * contact@kingsrook.com
- * https://github.com/Kingsrook/
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program.  If not, see <https://www.gnu.org/licenses/>.
- */
-
-package com.kingsrook.qqq.backend.core.actions.dashboard.widgets;
-
-
-import com.kingsrook.qqq.backend.core.exceptions.QException;
-import com.kingsrook.qqq.backend.core.model.MetaDataProducerInterface;
-import com.kingsrook.qqq.backend.core.model.actions.widgets.RenderWidgetInput;
-import com.kingsrook.qqq.backend.core.model.actions.widgets.RenderWidgetOutput;
-import com.kingsrook.qqq.backend.core.model.dashboard.widgets.AlertData;
-import com.kingsrook.qqq.backend.core.model.dashboard.widgets.WidgetType;
-import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
-import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
-import com.kingsrook.qqq.backend.core.model.metadata.dashboard.QWidgetMetaData;
-
-
-/*******************************************************************************
- ** Widget that can add an Alert to a process screen.
- **
- ** In the process, you'll want values:
- ** - alertType - name of entry in AlertType enum (ERROR, WARNING, SUCCESS)
- ** - alertHtml - html to display inside the alert (other than its icon)
- *******************************************************************************/
-public class ProcessAlertWidget extends AbstractWidgetRenderer implements MetaDataProducerInterface<QWidgetMetaData>
-{
-   public static final String NAME = "ProcessAlertWidget";
-
-
-
-   /*******************************************************************************
-    **
-    *******************************************************************************/
-   @Override
-   public RenderWidgetOutput render(RenderWidgetInput input) throws QException
-   {
-      AlertData.AlertType alertType = AlertData.AlertType.WARNING;
-      if(input.getQueryParams().containsKey("alertType"))
-      {
-         alertType = AlertData.AlertType.valueOf(input.getQueryParams().get("alertType"));
-      }
-
-      String html = "Warning";
-      if(input.getQueryParams().containsKey("alertHtml"))
-      {
-         html = input.getQueryParams().get("alertHtml");
-      }
-
-      return (new RenderWidgetOutput(new AlertData(alertType, html)));
-   }
-
-
-
-   /*******************************************************************************
-    **
-    *******************************************************************************/
-   @Override
-   public QWidgetMetaData produce(QInstance qInstance) throws QException
-   {
-      return new QWidgetMetaData()
-         .withType(WidgetType.ALERT.getType())
-         .withGridColumns(12)
-         .withName(NAME)
-         .withIsCard(false)
-         .withShowReloadButton(false)
-         .withCodeReference(new QCodeReference(getClass()));
-   }
-}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/interfaces/QStorageInterface.java

@@ -1,69 +0,0 @@
-/*
- * QQQ - Low-code Application Framework for Engineers.
- * Copyright (C) 2021-2024.  Kingsrook, LLC
- * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
- * contact@kingsrook.com
- * https://github.com/Kingsrook/
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program.  If not, see <https://www.gnu.org/licenses/>.
- */
-
-package com.kingsrook.qqq.backend.core.actions.interfaces;
-
-
-import java.io.InputStream;
-import java.io.OutputStream;
-import com.kingsrook.qqq.backend.core.exceptions.QException;
-import com.kingsrook.qqq.backend.core.model.actions.tables.storage.StorageInput;
-
-
-/*******************************************************************************
- ** Interface for actions that a backend can perform, based on streaming data
- ** into the backend's storage.
- *******************************************************************************/
-public interface QStorageInterface
-{
-
-   /*******************************************************************************
-    **
-    *******************************************************************************/
-   OutputStream createOutputStream(StorageInput storageInput) throws QException;
-
-
-   /*******************************************************************************
-    **
-    *******************************************************************************/
-   InputStream getInputStream(StorageInput storageInput) throws QException;
-
-
-   /*******************************************************************************
-    **
-    *******************************************************************************/
-   default void makePublic(StorageInput storageInput) throws QException
-   {
-      //////////
-      // noop //
-      //////////
-   }
-
-
-   /*******************************************************************************
-    **
-    *******************************************************************************/
-   default String getDownloadURL(StorageInput storageInput) throws QException
-   {
-      return (null);
-   }
-
-}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/messaging/SendMessageAction.java

@@ -1,64 +0,0 @@
-/*
- * QQQ - Low-code Application Framework for Engineers.
- * Copyright (C) 2021-2024.  Kingsrook, LLC
- * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
- * contact@kingsrook.com
- * https://github.com/Kingsrook/
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program.  If not, see <https://www.gnu.org/licenses/>.
- */
-
-package com.kingsrook.qqq.backend.core.actions.messaging;
-
-
-import com.kingsrook.qqq.backend.core.actions.AbstractQActionFunction;
-import com.kingsrook.qqq.backend.core.context.QContext;
-import com.kingsrook.qqq.backend.core.exceptions.QException;
-import com.kingsrook.qqq.backend.core.model.actions.messaging.SendMessageInput;
-import com.kingsrook.qqq.backend.core.model.actions.messaging.SendMessageOutput;
-import com.kingsrook.qqq.backend.core.model.metadata.messaging.QMessagingProviderMetaData;
-import com.kingsrook.qqq.backend.core.modules.messaging.MessagingProviderInterface;
-import com.kingsrook.qqq.backend.core.modules.messaging.QMessagingProviderDispatcher;
-import com.kingsrook.qqq.backend.core.utils.StringUtils;
-
-
-/*******************************************************************************
- **
- *******************************************************************************/
-public class SendMessageAction extends AbstractQActionFunction<SendMessageInput, SendMessageOutput>
-{
-
-   /*******************************************************************************
-    **
-    *******************************************************************************/
-   @Override
-   public SendMessageOutput execute(SendMessageInput input) throws QException
-   {
-      if(!StringUtils.hasContent(input.getMessagingProviderName()))
-      {
-         throw (new QException("Messaging provider name was not given in SendMessageInput."));
-      }
-
-      QMessagingProviderMetaData messagingProvider = QContext.getQInstance().getMessagingProvider(input.getMessagingProviderName());
-      if(messagingProvider == null)
-      {
-         throw (new QException("Messaging provider named [" + input.getMessagingProviderName() + "] was not found in this QInstance."));
-      }
-
-      MessagingProviderInterface messagingProviderInterface = new QMessagingProviderDispatcher().getMessagingProviderInterface(messagingProvider.getType());
-
-      return (messagingProviderInterface.sendMessage(input));
-   }
-
-}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/metadata/JoinGraph.java

@@ -29,7 +29,6 @@ import java.util.List;
 import java.util.Objects;
 import java.util.Set;
 import java.util.TreeSet;
-import com.kingsrook.qqq.backend.core.instances.QMetaDataVariableInterpreter;
 import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
 import com.kingsrook.qqq.backend.core.model.metadata.joins.QJoinMetaData;
 import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
@@ -42,18 +41,8 @@ import com.kingsrook.qqq.backend.core.utils.StringUtils;
  *******************************************************************************/
 public class JoinGraph
 {
-   private Set<Edge> edges = new HashSet<>();
 
-   ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-   // as an instance grows, with the number of joins (say, more than 50?), especially as they may have a lot of connections, //
-   // it can become very very slow to process a full join graph (e.g., 10 seconds, maybe much worse, per Big-O...)           //
-   // also, it's not frequently useful to look at a join path that's more than a handful of tables long.                     //
-   // thus - this property exists - to limit the max length of a join path.  Keeping it small keeps instance enrichment      //
-   // and validation reasonably performant, at the possible cost of, some join-path that's longer than this limit may not    //
-   // be found - but - chances are, you don't want some 12-element join path to be used anyway, thus, this makes sense.      //
-   // but - it can be adjusted, per system property or ENV var.                                                              //
-   ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-   private int maxPathLength = new QMetaDataVariableInterpreter().getIntegerFromPropertyOrEnvironment("qqq.instance.joinGraph.maxPathLength", "QQQ_INSTANCE_JOIN_GRAPH_MAX_PATH_LENGTH", 3);
+   private Set<Edge> edges = new HashSet<>();
 
 
 
@@ -314,13 +303,6 @@ public class JoinGraph
 
                if(otherTableName != null)
                {
-                  if(newPath.size() > maxPathLength)
-                  {
-                     ////////////////////////////////////////////////////////////////
-                     // performance hack.  see comment at maxPathLength definition //
-                     ////////////////////////////////////////////////////////////////
-                     continue;
-                  }
 
                   JoinConnectionList newConnectionList = connectionList.copy();
                   JoinConnection     joinConnection    = new JoinConnection(otherTableName, edge.joinName);

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/metadata/MetaDataAction.java

@@ -84,7 +84,7 @@ public class MetaDataAction
          }
 
          QBackendMetaData backendForTable = metaDataInput.getInstance().getBackendForTable(tableName);
-         tables.put(tableName, new QFrontendTableMetaData(backendForTable, table, false, false));
+         tables.put(tableName, new QFrontendTableMetaData(metaDataInput, backendForTable, table, false, false));
          treeNodes.put(tableName, new AppTreeNode(table));
       }
       metaDataOutput.setTables(tables);
@@ -218,8 +218,6 @@ public class MetaDataAction
 
       metaDataOutput.setEnvironmentValues(metaDataInput.getInstance().getEnvironmentValues());
 
-      metaDataOutput.setHelpContents(metaDataInput.getInstance().getHelpContent());
-
       // todo post-customization - can do whatever w/ the result if you want?
 
       return metaDataOutput;

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/metadata/TableMetaDataAction.java

@@ -54,7 +54,7 @@ public class TableMetaDataAction
          throw (new QNotFoundException("Table [" + tableMetaDataInput.getTableName() + "] was not found."));
       }
       QBackendMetaData backendForTable = tableMetaDataInput.getInstance().getBackendForTable(table.getName());
-      tableMetaDataOutput.setTable(new QFrontendTableMetaData(backendForTable, table, true, true));
+      tableMetaDataOutput.setTable(new QFrontendTableMetaData(tableMetaDataInput, backendForTable, table, true, true));
 
       // todo post-customization - can do whatever w/ the result if you want
 

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/permissions/PermissionsHelper.java

@@ -34,18 +34,15 @@ import com.kingsrook.qqq.backend.core.exceptions.QPermissionDeniedException;
 import com.kingsrook.qqq.backend.core.logging.QLogger;
 import com.kingsrook.qqq.backend.core.model.actions.AbstractActionInput;
 import com.kingsrook.qqq.backend.core.model.actions.AbstractTableActionInput;
-import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
 import com.kingsrook.qqq.backend.core.model.metadata.dashboard.QWidgetMetaDataInterface;
 import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.permissions.DenyBehavior;
 import com.kingsrook.qqq.backend.core.model.metadata.permissions.MetaDataWithName;
 import com.kingsrook.qqq.backend.core.model.metadata.permissions.MetaDataWithPermissionRules;
-import com.kingsrook.qqq.backend.core.model.metadata.permissions.PermissionLevel;
 import com.kingsrook.qqq.backend.core.model.metadata.permissions.QPermissionRules;
 import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportMetaData;
-import com.kingsrook.qqq.backend.core.model.metadata.tables.Capability;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 import com.kingsrook.qqq.backend.core.model.session.QSession;
 import com.kingsrook.qqq.backend.core.utils.StringUtils;
@@ -65,7 +62,7 @@ public class PermissionsHelper
     *******************************************************************************/
    public static void checkTablePermissionThrowing(AbstractTableActionInput tableActionInput, TablePermissionSubType permissionSubType) throws QPermissionDeniedException
    {
-      checkTablePermissionThrowing(tableActionInput.getTableName(), permissionSubType);
+      checkTablePermissionThrowing(tableActionInput, tableActionInput.getTableName(), permissionSubType);
    }
 
 
@@ -73,7 +70,7 @@ public class PermissionsHelper
    /*******************************************************************************
     **
     *******************************************************************************/
-   private static void checkTablePermissionThrowing(String tableName, TablePermissionSubType permissionSubType) throws QPermissionDeniedException
+   private static void checkTablePermissionThrowing(AbstractActionInput actionInput, String tableName, TablePermissionSubType permissionSubType) throws QPermissionDeniedException
    {
       warnAboutPermissionSubTypeForTables(permissionSubType);
       QTableMetaData table = QContext.getQInstance().getTable(tableName);
@@ -99,11 +96,11 @@ public class PermissionsHelper
    /*******************************************************************************
     **
     *******************************************************************************/
-   public static boolean hasTablePermission(String tableName, TablePermissionSubType permissionSubType)
+   public static boolean hasTablePermission(AbstractActionInput actionInput, String tableName, TablePermissionSubType permissionSubType)
    {
       try
       {
-         checkTablePermissionThrowing(tableName, permissionSubType);
+         checkTablePermissionThrowing(actionInput, tableName, permissionSubType);
          return (true);
       }
       catch(QPermissionDeniedException e)
@@ -336,25 +333,9 @@ public class PermissionsHelper
          QPermissionRules rules    = getEffectivePermissionRules(tableMetaData, instance);
          String           baseName = getEffectivePermissionBaseName(rules, tableMetaData.getName());
 
-         QBackendMetaData backend = instance.getBackend(tableMetaData.getBackendName());
-         if(tableMetaData.isCapabilityEnabled(backend, Capability.TABLE_INSERT))
+         for(TablePermissionSubType permissionSubType : TablePermissionSubType.values())
          {
-            addEffectiveAvailablePermission(rules, TablePermissionSubType.INSERT, rs, baseName, tableMetaData, "Table");
-         }
-
-         if(tableMetaData.isCapabilityEnabled(backend, Capability.TABLE_UPDATE))
-         {
-            addEffectiveAvailablePermission(rules, TablePermissionSubType.EDIT, rs, baseName, tableMetaData, "Table");
-         }
-
-         if(tableMetaData.isCapabilityEnabled(backend, Capability.TABLE_DELETE))
-         {
-            addEffectiveAvailablePermission(rules, TablePermissionSubType.DELETE, rs, baseName, tableMetaData, "Table");
-         }
-
-         if(tableMetaData.isCapabilityEnabled(backend, Capability.TABLE_QUERY) || tableMetaData.isCapabilityEnabled(backend, Capability.TABLE_GET))
-         {
-            addEffectiveAvailablePermission(rules, TablePermissionSubType.READ, rs, baseName, tableMetaData, "Table");
+            addEffectiveAvailablePermission(rules, permissionSubType, rs, baseName, tableMetaData, "Table");
          }
       }
 
@@ -388,10 +369,7 @@ public class PermissionsHelper
       {
          QPermissionRules rules    = getEffectivePermissionRules(widgetMetaData, instance);
          String           baseName = getEffectivePermissionBaseName(rules, widgetMetaData.getName());
-         if(!rules.getLevel().equals(PermissionLevel.NOT_PROTECTED))
-         {
-            addEffectiveAvailablePermission(rules, PrivatePermissionSubType.HAS_ACCESS, rs, baseName, widgetMetaData, "Widget");
-         }
+         addEffectiveAvailablePermission(rules, PrivatePermissionSubType.HAS_ACCESS, rs, baseName, widgetMetaData, "Widget");
       }
 
       return (rs);
@@ -500,6 +478,7 @@ public class PermissionsHelper
    /*******************************************************************************
     **
     *******************************************************************************/
+   @SuppressWarnings("checkstyle:indentation")
    static PermissionSubType getEffectivePermissionSubType(QPermissionRules rules, PermissionSubType originalPermissionSubType)
    {
       if(rules == null || rules.getLevel() == null)
@@ -514,10 +493,10 @@ public class PermissionsHelper
       if(PrivatePermissionSubType.HAS_ACCESS.equals(originalPermissionSubType))
       {
          return switch(rules.getLevel())
-         {
-            case NOT_PROTECTED -> null;
-            default -> PrivatePermissionSubType.HAS_ACCESS;
-         };
+            {
+               case NOT_PROTECTED -> null;
+               default -> PrivatePermissionSubType.HAS_ACCESS;
+            };
       }
       else
       {
@@ -526,30 +505,30 @@ public class PermissionsHelper
          // permission sub-type to what we expect to be set for the table                                      //
          ////////////////////////////////////////////////////////////////////////////////////////////////////////
          return switch(rules.getLevel())
-         {
-            case NOT_PROTECTED -> null;
-            case HAS_ACCESS_PERMISSION -> PrivatePermissionSubType.HAS_ACCESS;
-            case READ_WRITE_PERMISSIONS ->
             {
-               if(PrivatePermissionSubType.READ.equals(originalPermissionSubType) || PrivatePermissionSubType.WRITE.equals(originalPermissionSubType))
+               case NOT_PROTECTED -> null;
+               case HAS_ACCESS_PERMISSION -> PrivatePermissionSubType.HAS_ACCESS;
+               case READ_WRITE_PERMISSIONS ->
                {
-                  yield (originalPermissionSubType);
+                  if(PrivatePermissionSubType.READ.equals(originalPermissionSubType) || PrivatePermissionSubType.WRITE.equals(originalPermissionSubType))
+                  {
+                     yield (originalPermissionSubType);
+                  }
+                  else if(TablePermissionSubType.INSERT.equals(originalPermissionSubType) || TablePermissionSubType.EDIT.equals(originalPermissionSubType) || TablePermissionSubType.DELETE.equals(originalPermissionSubType))
+                  {
+                     yield (PrivatePermissionSubType.WRITE);
+                  }
+                  else if(TablePermissionSubType.READ.equals(originalPermissionSubType))
+                  {
+                     yield (PrivatePermissionSubType.READ);
+                  }
+                  else
+                  {
+                     throw new IllegalStateException("Unexpected permissionSubType: " + originalPermissionSubType);
+                  }
                }
-               else if(TablePermissionSubType.INSERT.equals(originalPermissionSubType) || TablePermissionSubType.EDIT.equals(originalPermissionSubType) || TablePermissionSubType.DELETE.equals(originalPermissionSubType))
-               {
-                  yield (PrivatePermissionSubType.WRITE);
-               }
-               else if(TablePermissionSubType.READ.equals(originalPermissionSubType))
-               {
-                  yield (PrivatePermissionSubType.READ);
-               }
-               else
-               {
-                  throw new IllegalStateException("Unexpected permissionSubType: " + originalPermissionSubType);
-               }
-            }
-            case READ_INSERT_EDIT_DELETE_PERMISSIONS -> originalPermissionSubType;
-         };
+               case READ_INSERT_EDIT_DELETE_PERMISSIONS -> originalPermissionSubType;
+            };
       }
    }
 

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/processes/CancelProcessAction.java

@@ -1,110 +0,0 @@
-/*
- * QQQ - Low-code Application Framework for Engineers.
- * Copyright (C) 2021-2022.  Kingsrook, LLC
- * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
- * contact@kingsrook.com
- * https://github.com/Kingsrook/
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program.  If not, see <https://www.gnu.org/licenses/>.
- */
-
-package com.kingsrook.qqq.backend.core.actions.processes;
-
-
-import java.util.Optional;
-import java.util.UUID;
-import com.kingsrook.qqq.backend.core.actions.ActionHelper;
-import com.kingsrook.qqq.backend.core.exceptions.QBadRequestException;
-import com.kingsrook.qqq.backend.core.exceptions.QException;
-import com.kingsrook.qqq.backend.core.logging.QLogger;
-import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessState;
-import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput;
-import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessOutput;
-import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
-import com.kingsrook.qqq.backend.core.state.StateType;
-import com.kingsrook.qqq.backend.core.state.UUIDAndTypeStateKey;
-import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
-
-
-/*******************************************************************************
- ** Action handler for running the cancel step of a qqq process
- *
- *******************************************************************************/
-public class CancelProcessAction extends RunProcessAction
-{
-   private static final QLogger LOG = QLogger.getLogger(CancelProcessAction.class);
-
-
-
-   /*******************************************************************************
-    **
-    *******************************************************************************/
-   public RunProcessOutput execute(RunProcessInput runProcessInput) throws QException
-   {
-      ActionHelper.validateSession(runProcessInput);
-
-      QProcessMetaData process = runProcessInput.getInstance().getProcess(runProcessInput.getProcessName());
-      if(process == null)
-      {
-         throw new QBadRequestException("Process [" + runProcessInput.getProcessName() + "] is not defined in this instance.");
-      }
-
-      if(runProcessInput.getProcessUUID() == null)
-      {
-         throw (new QBadRequestException("Cannot cancel process - processUUID was not given."));
-      }
-
-      UUIDAndTypeStateKey    stateKey     = new UUIDAndTypeStateKey(UUID.fromString(runProcessInput.getProcessUUID()), StateType.PROCESS_STATUS);
-      Optional<ProcessState> processState = getState(runProcessInput.getProcessUUID());
-      if(processState.isEmpty())
-      {
-         throw (new QBadRequestException("Cannot cancel process - State for process UUID [" + runProcessInput.getProcessUUID() + "] was not found."));
-      }
-
-      RunProcessOutput runProcessOutput = new RunProcessOutput();
-      try
-      {
-         if(process.getCancelStep() != null)
-         {
-            LOG.info("Running cancel step for process", logPair("processName", process.getName()));
-            runBackendStep(runProcessInput, process, runProcessOutput, stateKey, process.getCancelStep(), process, processState.get());
-         }
-         else
-         {
-            LOG.debug("Process does not have a custom cancel step to run.", logPair("processName", process.getName()));
-         }
-      }
-      catch(QException qe)
-      {
-         ////////////////////////////////////////////////////////////
-         // upon exception (e.g., one thrown by a step), throw it. //
-         ////////////////////////////////////////////////////////////
-         throw (qe);
-      }
-      catch(Exception e)
-      {
-         throw (new QException("Error cancelling process", e));
-      }
-      finally
-      {
-         //////////////////////////////////////////////////////
-         // always put the final state in the process result //
-         //////////////////////////////////////////////////////
-         runProcessOutput.setProcessState(processState.get());
-      }
-
-      return (runProcessOutput);
-   }
-
-}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/processes/QProcessCallbackFactory.java

@@ -23,19 +23,11 @@ package com.kingsrook.qqq.backend.core.actions.processes;
 
 
 import java.io.Serializable;
-import java.util.Collection;
 import java.util.Collections;
 import java.util.List;
 import java.util.Map;
-import com.kingsrook.qqq.backend.core.context.QContext;
-import com.kingsrook.qqq.backend.core.exceptions.QRuntimeException;
-import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
-import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
-import com.kingsrook.qqq.backend.core.model.data.QRecord;
-import com.kingsrook.qqq.backend.core.model.data.QRecordEntity;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
-import com.kingsrook.qqq.backend.core.utils.StringUtils;
 
 
 /*******************************************************************************
@@ -73,56 +65,4 @@ public class QProcessCallbackFactory
       };
    }
 
-
-
-   /*******************************************************************************
-    **
-    *******************************************************************************/
-   public static QProcessCallback forRecordEntity(QRecordEntity entity)
-   {
-      return forRecord(entity.toQRecord());
-   }
-
-
-
-   /*******************************************************************************
-    **
-    *******************************************************************************/
-   public static QProcessCallback forRecord(QRecord record)
-   {
-      String primaryKeyField = "id";
-      if(StringUtils.hasContent(record.getTableName()))
-      {
-         primaryKeyField = QContext.getQInstance().getTable(record.getTableName()).getPrimaryKeyField();
-      }
-
-      Serializable primaryKeyValue = record.getValue(primaryKeyField);
-      if(primaryKeyValue == null)
-      {
-         throw (new QRuntimeException("Record did not have value in its primary key field [" + primaryKeyField + "]"));
-      }
-
-      return (forPrimaryKey(primaryKeyField, primaryKeyValue));
-   }
-
-
-
-   /*******************************************************************************
-    **
-    *******************************************************************************/
-   public static QProcessCallback forPrimaryKey(String fieldName, Serializable value)
-   {
-      return (forFilter(new QQueryFilter().withCriteria(new QFilterCriteria(fieldName, QCriteriaOperator.EQUALS, value))));
-   }
-
-
-
-   /*******************************************************************************
-    **
-    *******************************************************************************/
-   public static QProcessCallback forPrimaryKeys(String fieldName, Collection<? extends Serializable> values)
-   {
-      return (forFilter(new QQueryFilter().withCriteria(new QFilterCriteria(fieldName, QCriteriaOperator.IN, values))));
-   }
-
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/processes/RunBackendStepAction.java

@@ -26,7 +26,6 @@ import java.io.Serializable;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
-import java.util.Objects;
 import java.util.stream.Collectors;
 import com.kingsrook.qqq.backend.core.actions.ActionHelper;
 import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
@@ -35,7 +34,6 @@ import com.kingsrook.qqq.backend.core.exceptions.QUserFacingException;
 import com.kingsrook.qqq.backend.core.logging.QLogger;
 import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
 import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
-import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput;
 import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
@@ -73,17 +71,7 @@ public class RunBackendStepAction
       QStepMetaData stepMetaData = process.getStep(runBackendStepInput.getStepName());
       if(stepMetaData == null)
       {
-         if(process.getCancelStep() != null && Objects.equals(process.getCancelStep().getName(), runBackendStepInput.getStepName()))
-         {
-            /////////////////////////////////////
-            // special case for cancel step... //
-            /////////////////////////////////////
-            stepMetaData = process.getCancelStep();
-         }
-         else
-         {
-            throw new QException("Step [" + runBackendStepInput.getStepName() + "] is not defined in the process [" + process.getName() + "]");
-         }
+         throw new QException("Step [" + runBackendStepInput.getStepName() + "] is not defined in the process [" + process.getName() + "]");
       }
 
       if(!(stepMetaData instanceof QBackendStepMetaData backendStepMetaData))
@@ -94,7 +82,7 @@ public class RunBackendStepAction
       //////////////////////////////////////////////////////////////////////////////////////
       // ensure input data is set as needed - use callback object to get anything missing //
       //////////////////////////////////////////////////////////////////////////////////////
-      ensureRecordsAreInRequest(runBackendStepInput, backendStepMetaData, process);
+      ensureRecordsAreInRequest(runBackendStepInput, backendStepMetaData);
       ensureInputFieldsAreInRequest(runBackendStepInput, backendStepMetaData);
 
       ////////////////////////////////////////////////////////////////////
@@ -179,7 +167,7 @@ public class RunBackendStepAction
     ** check if this step uses a record list - and if so, if we need to get one
     ** via the callback
     *******************************************************************************/
-   private void ensureRecordsAreInRequest(RunBackendStepInput runBackendStepInput, QBackendStepMetaData step, QProcessMetaData process) throws QException
+   private void ensureRecordsAreInRequest(RunBackendStepInput runBackendStepInput, QBackendStepMetaData step) throws QException
    {
       QFunctionInputMetaData inputMetaData = step.getInputMetaData();
       if(inputMetaData != null && inputMetaData.getRecordListMetaData() != null)
@@ -202,44 +190,9 @@ public class RunBackendStepAction
 
             queryInput.setFilter(callback.getQueryFilter());
 
-            //////////////////////////////////////////////////////////////////////////////////////////
-            // if process has a max-no of records, set a limit on the process of that number plus 1 //
-            // (the plus 1 being so we can see "oh, you selected more than that many; error!"       //
-            //////////////////////////////////////////////////////////////////////////////////////////
-            if(process.getMaxInputRecords() != null)
-            {
-               if(callback.getQueryFilter() == null)
-               {
-                  queryInput.setFilter(new QQueryFilter());
-               }
-
-               queryInput.getFilter().setLimit(process.getMaxInputRecords() + 1);
-            }
-
             QueryOutput queryOutput = new QueryAction().execute(queryInput);
             runBackendStepInput.setRecords(queryOutput.getRecords());
-
-            ////////////////////////////////////////////////////////////////////////////////
-            // if process defines a max, and more than the max were found, throw an error //
-            ////////////////////////////////////////////////////////////////////////////////
-            if(process.getMaxInputRecords() != null)
-            {
-               if(queryOutput.getRecords().size() > process.getMaxInputRecords())
-               {
-                  throw (new QUserFacingException("Too many records were selected for this process.  At most, only " + process.getMaxInputRecords() + " can be selected."));
-               }
-            }
-
-            /////////////////////////////////////////////////////////////////////////////////
-            // if process defines a min, and fewer than the min were found, throw an error //
-            /////////////////////////////////////////////////////////////////////////////////
-            if(process.getMinInputRecords() != null)
-            {
-               if(queryOutput.getRecords().size() < process.getMinInputRecords())
-               {
-                  throw (new QUserFacingException("Too few records were selected for this process.  At least " + process.getMinInputRecords() + " must be selected."));
-               }
-            }
+            // todo - handle 0 results found?
          }
       }
    }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/processes/RunProcessAction.java

@@ -82,7 +82,7 @@ public class RunProcessAction
    public static final String BASEPULL_THIS_RUNTIME_KEY = "basepullThisRuntimeKey";
    public static final String BASEPULL_LAST_RUNTIME_KEY = "basepullLastRuntimeKey";
    public static final String BASEPULL_TIMESTAMP_FIELD  = "basepullTimestampField";
-   public static final String BASEPULL_CONFIGURATION    = "basepullConfiguration";
+   public static final String BASEPULL_CONFIGURATION = "basepullConfiguration";
 
    ////////////////////////////////////////////////////////////////////////////////////////////////
    // indicator that the timestamp field should be updated - e.g., the execute step is finished. //
@@ -190,25 +190,7 @@ public class RunProcessAction
                // Run backend steps //
                ///////////////////////
                LOG.debug("Running backend step [" + step.getName() + "] in process [" + process.getName() + "]");
-               RunBackendStepOutput runBackendStepOutput = runBackendStep(runProcessInput, process, runProcessOutput, stateKey, backendStepMetaData, process, processState);
-
-               /////////////////////////////////////////////////////////////////////////////////////////
-               // if the step returned an override lastStepName, use that to determine how we proceed //
-               /////////////////////////////////////////////////////////////////////////////////////////
-               if(runBackendStepOutput.getOverrideLastStepName() != null)
-               {
-                  LOG.debug("Process step [" + lastStepName + "] returned an overrideLastStepName [" + runBackendStepOutput.getOverrideLastStepName() + "]!");
-                  lastStepName = runBackendStepOutput.getOverrideLastStepName();
-               }
-
-               /////////////////////////////////////////////////////////////////////////////////////////////
-               // similarly, if the step produced an updatedFrontendStepList, propagate that data outward //
-               /////////////////////////////////////////////////////////////////////////////////////////////
-               if(runBackendStepOutput.getUpdatedFrontendStepList() != null)
-               {
-                  LOG.debug("Process step [" + lastStepName + "] generated an updatedFrontendStepList [" + runBackendStepOutput.getUpdatedFrontendStepList().stream().map(s -> s.getName()).toList() + "]!");
-                  runProcessOutput.setUpdatedFrontendStepList(runBackendStepOutput.getUpdatedFrontendStepList());
-               }
+               runBackendStep(runProcessInput, process, runProcessOutput, stateKey, backendStepMetaData, process, processState);
             }
             else
             {
@@ -335,13 +317,6 @@ public class RunProcessAction
          ///////////////////////////////////////////////////
          runProcessInput.seedFromProcessState(optionalProcessState.get());
 
-         ///////////////////////////////////////////////////////////////////////////////////////////////////
-         // if we're restoring an old state, we can discard a previously stored updatedFrontendStepList - //
-         // it is only needed on the transitional edge from a backend-step to a frontend step, but not    //
-         // in the other directly                                                                         //
-         ///////////////////////////////////////////////////////////////////////////////////////////////////
-         optionalProcessState.get().setUpdatedFrontendStepList(null);
-
          ///////////////////////////////////////////////////////////////////////////
          // if there were values from the caller, put those (back) in the request //
          ///////////////////////////////////////////////////////////////////////////
@@ -364,7 +339,7 @@ public class RunProcessAction
    /*******************************************************************************
     ** Run a single backend step.
     *******************************************************************************/
-   protected RunBackendStepOutput runBackendStep(RunProcessInput runProcessInput, QProcessMetaData process, RunProcessOutput runProcessOutput, UUIDAndTypeStateKey stateKey, QBackendStepMetaData backendStep, QProcessMetaData qProcessMetaData, ProcessState processState) throws Exception
+   private void runBackendStep(RunProcessInput runProcessInput, QProcessMetaData process, RunProcessOutput runProcessOutput, UUIDAndTypeStateKey stateKey, QBackendStepMetaData backendStep, QProcessMetaData qProcessMetaData, ProcessState processState) throws Exception
    {
       RunBackendStepInput runBackendStepInput = new RunBackendStepInput(processState);
       runBackendStepInput.setProcessName(process.getName());
@@ -393,16 +368,14 @@ public class RunProcessAction
          runBackendStepInput.setBasepullLastRunTime((Instant) runProcessInput.getValues().get(BASEPULL_LAST_RUNTIME_KEY));
       }
 
-      RunBackendStepOutput runBackendStepOutput = new RunBackendStepAction().execute(runBackendStepInput);
-      storeState(stateKey, runBackendStepOutput.getProcessState());
+      RunBackendStepOutput lastFunctionResult = new RunBackendStepAction().execute(runBackendStepInput);
+      storeState(stateKey, lastFunctionResult.getProcessState());
 
-      if(runBackendStepOutput.getException() != null)
+      if(lastFunctionResult.getException() != null)
       {
-         runProcessOutput.setException(runBackendStepOutput.getException());
-         throw (runBackendStepOutput.getException());
+         runProcessOutput.setException(lastFunctionResult.getException());
+         throw (lastFunctionResult.getException());
       }
-
-      return (runBackendStepOutput);
    }
 
 
@@ -522,15 +495,15 @@ public class RunProcessAction
       String basepullKeyValue = (basepullConfiguration.getKeyValue() != null) ? basepullConfiguration.getKeyValue() : process.getName();
 
       //////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-      // if process specifies that it uses variants, look for that data in the session and append to our basepull key //
+      // if backend specifies that it uses variants, look for that data in the session and append to our basepull key //
       //////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-      if(process.getVariantBackend() != null)
+      if(process.getSchedule() != null && process.getSchedule().getVariantBackend() != null)
       {
          QSession         session         = QContext.getQSession();
-         QBackendMetaData backendMetaData = QContext.getQInstance().getBackend(process.getVariantBackend());
+         QBackendMetaData backendMetaData = QContext.getQInstance().getBackend(process.getSchedule().getVariantBackend());
          if(session.getBackendVariants() == null || !session.getBackendVariants().containsKey(backendMetaData.getVariantOptionsTableTypeValue()))
          {
-            LOG.warn("Could not find Backend Variant information for Backend '" + backendMetaData.getName() + "'");
+            LOG.info("Could not find Backend Variant information for Backend '" + backendMetaData.getName() + "'");
          }
          else
          {

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/CsvExportStreamer.java

@@ -31,7 +31,6 @@ import com.kingsrook.qqq.backend.core.logging.QLogger;
 import com.kingsrook.qqq.backend.core.model.actions.reporting.ExportInput;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
-import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportView;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 import com.kingsrook.qqq.backend.core.utils.StringUtils;
 
@@ -66,12 +65,12 @@ public class CsvExportStreamer implements ExportStreamerInterface
     **
     *******************************************************************************/
    @Override
-   public void start(ExportInput exportInput, List<QFieldMetaData> fields, String label, QReportView view) throws QReportingException
+   public void start(ExportInput exportInput, List<QFieldMetaData> fields, String label) throws QReportingException
    {
       this.exportInput = exportInput;
       this.fields = fields;
       table = exportInput.getTable();
-      outputStream = this.exportInput.getReportDestination().getReportOutputStream();
+      outputStream = this.exportInput.getReportOutputStream();
 
       writeTitleAndHeader();
    }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/DistinctFilteringRecordPipe.java

@@ -1,146 +0,0 @@
-/*
- * QQQ - Low-code Application Framework for Engineers.
- * Copyright (C) 2021-2023.  Kingsrook, LLC
- * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
- * contact@kingsrook.com
- * https://github.com/Kingsrook/
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program.  If not, see <https://www.gnu.org/licenses/>.
- */
-
-package com.kingsrook.qqq.backend.core.actions.reporting;
-
-
-import java.io.Serializable;
-import java.util.ArrayList;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-import com.kingsrook.qqq.backend.core.exceptions.QException;
-import com.kingsrook.qqq.backend.core.model.data.QRecord;
-import com.kingsrook.qqq.backend.core.model.metadata.tables.UniqueKey;
-
-
-/*******************************************************************************
- ** Subclass of record pipe that ony allows through distinct records, based on
- ** the set of fields specified in the constructor as a uniqueKey.
- *******************************************************************************/
-public class DistinctFilteringRecordPipe extends RecordPipe
-{
-   private UniqueKey         uniqueKey;
-   private Set<Serializable> seenValues = new HashSet<>();
-
-
-
-   /*******************************************************************************
-    ** Constructor
-    **
-    *******************************************************************************/
-   public DistinctFilteringRecordPipe(UniqueKey uniqueKey)
-   {
-      this.uniqueKey = uniqueKey;
-   }
-
-
-
-   /*******************************************************************************
-    ** Constructor that accepts pipe's overrideCapacity (allowed to be null)
-    **
-    *******************************************************************************/
-   public DistinctFilteringRecordPipe(UniqueKey uniqueKey, Integer overrideCapacity)
-   {
-      super(overrideCapacity);
-      this.uniqueKey = uniqueKey;
-   }
-
-
-
-   /*******************************************************************************
-    **
-    *******************************************************************************/
-   @Override
-   public void addRecords(List<QRecord> records) throws QException
-   {
-      List<QRecord> recordsToAdd = new ArrayList<>();
-      for(QRecord record : records)
-      {
-         if(!seenBefore(record))
-         {
-            recordsToAdd.add(record);
-         }
-      }
-
-      if(recordsToAdd.isEmpty())
-      {
-         return;
-      }
-
-      super.addRecords(recordsToAdd);
-   }
-
-
-
-   /*******************************************************************************
-    **
-    *******************************************************************************/
-   @Override
-   public void addRecord(QRecord record) throws QException
-   {
-      if(seenBefore(record))
-      {
-         return;
-      }
-
-      super.addRecord(record);
-   }
-
-
-
-   /*******************************************************************************
-    ** return true if we've seen this record before (based on the unique key) -
-    ** also - update the set of seen values!
-    *******************************************************************************/
-   private boolean seenBefore(QRecord record)
-   {
-      Serializable ukValues = extractUKValues(record);
-      if(seenValues.contains(ukValues))
-      {
-         return true;
-      }
-      seenValues.add(ukValues);
-      return false;
-   }
-
-
-
-   /*******************************************************************************
-    **
-    *******************************************************************************/
-   private Serializable extractUKValues(QRecord record)
-   {
-      if(uniqueKey.getFieldNames().size() == 1)
-      {
-         return (record.getValue(uniqueKey.getFieldNames().get(0)));
-      }
-      else
-      {
-         ArrayList<Serializable> rs = new ArrayList<>();
-         for(String fieldName : uniqueKey.getFieldNames())
-         {
-            rs.add(record.getValue(fieldName));
-         }
-         return (rs);
-      }
-   }
-}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/ExcelExportStreamer.java

@@ -1,6 +1,6 @@
 /*
  * QQQ - Low-code Application Framework for Engineers.
- * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
  * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
  * contact@kingsrook.com
  * https://github.com/Kingsrook/
@@ -19,7 +19,7 @@
  * along with this program.  If not, see <https://www.gnu.org/licenses/>.
  */
 
-package com.kingsrook.qqq.backend.core.actions.reporting.excel.fastexcel;
+package com.kingsrook.qqq.backend.core.actions.reporting;
 
 
 import java.io.OutputStream;
@@ -34,8 +34,8 @@ import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
-import com.kingsrook.qqq.backend.core.actions.reporting.ExportStreamerInterface;
-import com.kingsrook.qqq.backend.core.context.QContext;
+import com.kingsrook.qqq.backend.core.actions.reporting.excelformatting.ExcelStylerInterface;
+import com.kingsrook.qqq.backend.core.actions.reporting.excelformatting.PlainExcelStyler;
 import com.kingsrook.qqq.backend.core.exceptions.QReportingException;
 import com.kingsrook.qqq.backend.core.logging.QLogger;
 import com.kingsrook.qqq.backend.core.model.actions.reporting.ExportInput;
@@ -43,9 +43,7 @@ import com.kingsrook.qqq.backend.core.model.data.QRecord;
 import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.DisplayFormat;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
-import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportView;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
-import com.kingsrook.qqq.backend.core.utils.ObjectUtils;
 import com.kingsrook.qqq.backend.core.utils.StringUtils;
 import com.kingsrook.qqq.backend.core.utils.ValueUtils;
 import org.dhatim.fastexcel.StyleSetter;
@@ -54,19 +52,19 @@ import org.dhatim.fastexcel.Worksheet;
 
 
 /*******************************************************************************
- ** Excel export format implementation - built using fastexcel library
+ ** Excel export format implementation
  *******************************************************************************/
-public class ExcelFastexcelExportStreamer implements ExportStreamerInterface
+public class ExcelExportStreamer implements ExportStreamerInterface
 {
-   private static final QLogger LOG = QLogger.getLogger(ExcelFastexcelExportStreamer.class);
+   private static final QLogger LOG = QLogger.getLogger(ExcelExportStreamer.class);
 
    private ExportInput          exportInput;
    private QTableMetaData       table;
    private List<QFieldMetaData> fields;
    private OutputStream         outputStream;
 
-   private FastExcelStylerInterface fastExcelStylerInterface = new PlainFastExcelStyler();
-   private Map<String, String>      excelCellFormats;
+   private ExcelStylerInterface excelStylerInterface = new PlainExcelStyler();
+   private Map<String, String>  excelCellFormats;
 
    private Workbook  workbook;
    private Worksheet worksheet;
@@ -78,7 +76,7 @@ public class ExcelFastexcelExportStreamer implements ExportStreamerInterface
    /*******************************************************************************
     **
     *******************************************************************************/
-   public ExcelFastexcelExportStreamer()
+   public ExcelExportStreamer()
    {
    }
 
@@ -107,14 +105,14 @@ public class ExcelFastexcelExportStreamer implements ExportStreamerInterface
     ** Starts a new worksheet in the current workbook.  Can be called multiple times.
     *******************************************************************************/
    @Override
-   public void start(ExportInput exportInput, List<QFieldMetaData> fields, String label, QReportView view) throws QReportingException
+   public void start(ExportInput exportInput, List<QFieldMetaData> fields, String label) throws QReportingException
    {
       try
       {
          this.exportInput = exportInput;
          this.fields = fields;
          table = exportInput.getTable();
-         outputStream = this.exportInput.getReportDestination().getReportOutputStream();
+         outputStream = this.exportInput.getReportOutputStream();
          this.row = 0;
          this.sheetCount++;
 
@@ -123,7 +121,7 @@ public class ExcelFastexcelExportStreamer implements ExportStreamerInterface
          /////////////////////////////////////////////////////////////////////////////////////////////////////
          if(workbook == null)
          {
-            String    appName = ObjectUtils.tryAndRequireNonNullElse(() -> QContext.getQInstance().getBranding().getAppName(), "QQQ");
+            String    appName  = "QQQ";
             QInstance instance = exportInput.getInstance();
             if(instance != null && instance.getBranding() != null && instance.getBranding().getCompanyName() != null)
             {
@@ -169,7 +167,7 @@ public class ExcelFastexcelExportStreamer implements ExportStreamerInterface
             worksheet.range(row, 0, row, fields.size() - 1).merge();
 
             StyleSetter titleStyle = worksheet.range(row, 0, row, fields.size() - 1).style();
-            fastExcelStylerInterface.styleTitleRow(titleStyle);
+            excelStylerInterface.styleTitleRow(titleStyle);
             titleStyle.set();
 
             row++;
@@ -189,7 +187,7 @@ public class ExcelFastexcelExportStreamer implements ExportStreamerInterface
             }
 
             StyleSetter headerStyle = worksheet.range(row, 0, row, fields.size() - 1).style();
-            fastExcelStylerInterface.styleHeaderRow(headerStyle);
+            excelStylerInterface.styleHeaderRow(headerStyle);
             headerStyle.set();
 
             row++;
@@ -317,7 +315,7 @@ public class ExcelFastexcelExportStreamer implements ExportStreamerInterface
       writeRecord(record);
 
       StyleSetter totalsRowStyle = worksheet.range(row, 0, row, fields.size() - 1).style();
-      fastExcelStylerInterface.styleTotalsRow(totalsRowStyle);
+      excelStylerInterface.styleTotalsRow(totalsRowStyle);
       totalsRowStyle.set();
    }
 

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/ExportAction.java

@@ -44,7 +44,6 @@ import com.kingsrook.qqq.backend.core.logging.QLogger;
 import com.kingsrook.qqq.backend.core.model.actions.reporting.ExportInput;
 import com.kingsrook.qqq.backend.core.model.actions.reporting.ExportOutput;
 import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportFormat;
-import com.kingsrook.qqq.backend.core.model.actions.tables.QueryHint;
 import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountOutput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
@@ -53,7 +52,6 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryJoin;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
-import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportView;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.ExposedJoin;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleDispatcher;
@@ -140,7 +138,7 @@ public class ExportAction
       ///////////////////////////////////////////////////////////////////////////////////////////////////////////
       // check if this report format has a max-rows limit -- if so, do a count to verify we're under the limit //
       ///////////////////////////////////////////////////////////////////////////////////////////////////////////
-      ReportFormat reportFormat = exportInput.getReportDestination().getReportFormat();
+      ReportFormat reportFormat = exportInput.getReportFormat();
       verifyCountUnderMax(exportInput, backendModule, reportFormat);
 
       preExecuteRan = true;
@@ -191,9 +189,6 @@ public class ExportAction
       Set<String>     addedJoinNames = new HashSet<>();
       if(CollectionUtils.nullSafeHasContents(exportInput.getFieldNames()))
       {
-         /////////////////////////////////////////////////////////////////////////////////////////////
-         // make sure that any tables being selected from are included as (LEFT) joins in the query //
-         /////////////////////////////////////////////////////////////////////////////////////////////
          for(String fieldName : exportInput.getFieldNames())
          {
             if(fieldName.contains("."))
@@ -202,7 +197,27 @@ public class ExportAction
                String   joinTableName = parts[0];
                if(!addedJoinNames.contains(joinTableName))
                {
-                  queryJoins.add(new QueryJoin(joinTableName).withType(QueryJoin.Type.LEFT).withSelect(true));
+                  QueryJoin queryJoin = new QueryJoin(joinTableName).withType(QueryJoin.Type.LEFT).withSelect(true);
+                  queryJoins.add(queryJoin);
+
+                  /////////////////////////////////////////////////////////////////////////////////////////////
+                  // in at least some cases, we need to let the queryJoin know what join-meta-data to use... //
+                  // This code basically mirrors what QFMD is doing right now, so it's better -              //
+                  // but shouldn't all of this just be in JoinsContext?  it does some of this...             //
+                  /////////////////////////////////////////////////////////////////////////////////////////////
+                  QTableMetaData        table               = exportInput.getTable();
+                  Optional<ExposedJoin> exposedJoinOptional = CollectionUtils.nonNullList(table.getExposedJoins()).stream().filter(ej -> ej.getJoinTable().equals(joinTableName)).findFirst();
+                  if(exposedJoinOptional.isEmpty())
+                  {
+                     throw (new QException("Could not find exposed join between base table " + table.getName() + " and requested join table " + joinTableName));
+                  }
+                  ExposedJoin exposedJoin = exposedJoinOptional.get();
+
+                  if(exposedJoin.getJoinPath().size() == 1)
+                  {
+                     queryJoin.setJoinMetaData(QContext.getQInstance().getJoin(exposedJoin.getJoinPath().get(exposedJoin.getJoinPath().size() - 1)));
+                  }
+
                   addedJoinNames.add(joinTableName);
                }
             }
@@ -217,8 +232,6 @@ public class ExportAction
       }
       queryInput.getFilter().setLimit(exportInput.getLimit());
       queryInput.setShouldTranslatePossibleValues(true);
-      queryInput.withQueryHint(QueryHint.POTENTIALLY_LARGE_NUMBER_OF_RESULTS);
-      queryInput.withQueryHint(QueryHint.MAY_USE_READ_ONLY_BACKEND);
 
       /////////////////////////////////////////////////////////////////
       // tell this query that it needs to put its output into a pipe //
@@ -229,19 +242,10 @@ public class ExportAction
       ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
       // set up a report streamer, which will read rows from the pipe, and write formatted report rows to the output stream //
       ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-      ReportFormat            reportFormat   = exportInput.getReportDestination().getReportFormat();
+      ReportFormat            reportFormat   = exportInput.getReportFormat();
       ExportStreamerInterface reportStreamer = reportFormat.newReportStreamer();
       List<QFieldMetaData>    fields         = getFields(exportInput);
-
-      //////////////////////////////////////////////////////////
-      // it seems we can pass a view with just a name in here //
-      //////////////////////////////////////////////////////////
-      List<QReportView> views = new ArrayList<>();
-      views.add(new QReportView()
-         .withName("export"));
-
-      reportStreamer.preRun(exportInput.getReportDestination(), views);
-      reportStreamer.start(exportInput, fields, "Sheet 1", views.get(0));
+      reportStreamer.start(exportInput, fields, "Sheet 1");
 
       //////////////////////////////////////////
       // run the query action as an async job //
@@ -330,7 +334,7 @@ public class ExportAction
 
       try
       {
-         exportInput.getReportDestination().getReportOutputStream().close();
+         exportInput.getReportOutputStream().close();
       }
       catch(Exception e)
       {

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/ExportStreamerInterface.java

@@ -26,10 +26,8 @@ import java.util.List;
 import java.util.Map;
 import com.kingsrook.qqq.backend.core.exceptions.QReportingException;
 import com.kingsrook.qqq.backend.core.model.actions.reporting.ExportInput;
-import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportDestination;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
-import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportView;
 
 
 /*******************************************************************************
@@ -37,14 +35,20 @@ import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportView;
  *******************************************************************************/
 public interface ExportStreamerInterface
 {
+   /*******************************************************************************
+    ** Called once, before any rows are available.  Meant to write a header, for example.
+    *******************************************************************************/
+   void start(ExportInput exportInput, List<QFieldMetaData> fields, String label) throws QReportingException;
 
    /*******************************************************************************
-    ** Called once, before any sheets are actually being produced.
+    ** Called as records flow into the pipe.
+    ******************************************************************************/
+   void addRecords(List<QRecord> recordList) throws QReportingException;
+
+   /*******************************************************************************
+    ** Called once, after all rows are available.  Meant to write a footer, or close resources, for example.
     *******************************************************************************/
-   default void preRun(ReportDestination reportDestination, List<QReportView> views) throws QReportingException
-   {
-      // noop in base class
-   }
+   void finish() throws QReportingException;
 
    /*******************************************************************************
     **
@@ -54,20 +58,6 @@ public interface ExportStreamerInterface
       // noop in base class
    }
 
-   /*******************************************************************************
-    ** Called once per sheet, before any rows are available.  Meant to write a
-    ** header, for example.
-    **
-    ** If multiple sheets are being created, there is no separate end-sheet call.
-    ** Rather, a new one will just get started...
-    *******************************************************************************/
-   void start(ExportInput exportInput, List<QFieldMetaData> fields, String label, QReportView view) throws QReportingException;
-
-   /*******************************************************************************
-    ** Called as records flow into the pipe.
-    ******************************************************************************/
-   void addRecords(List<QRecord> recordList) throws QReportingException;
-
    /*******************************************************************************
     **
     *******************************************************************************/
@@ -75,11 +65,4 @@ public interface ExportStreamerInterface
    {
       addRecords(List.of(record));
    }
-
-   /*******************************************************************************
-    ** Called after all sheets are complete.  Meant to do a final write, or close
-    ** resources, for example.
-    *******************************************************************************/
-   void finish() throws QReportingException;
-
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/GenerateReportAction.java

@@ -24,8 +24,6 @@ package com.kingsrook.qqq.backend.core.actions.reporting;
 
 import java.io.Serializable;
 import java.math.BigDecimal;
-import java.time.Instant;
-import java.time.LocalDate;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashMap;
@@ -34,23 +32,18 @@ import java.util.List;
 import java.util.Map;
 import java.util.Objects;
 import java.util.Set;
-import java.util.concurrent.atomic.AtomicInteger;
 import java.util.function.Function;
 import java.util.function.Supplier;
 import java.util.stream.Collectors;
-import com.kingsrook.qqq.backend.core.actions.AbstractQActionFunction;
 import com.kingsrook.qqq.backend.core.actions.async.AsyncRecordPipeLoop;
 import com.kingsrook.qqq.backend.core.actions.customizers.QCodeLoader;
 import com.kingsrook.qqq.backend.core.actions.reporting.customizers.DataSourceQueryInputCustomizer;
 import com.kingsrook.qqq.backend.core.actions.reporting.customizers.ReportViewCustomizer;
-import com.kingsrook.qqq.backend.core.actions.tables.CountAction;
 import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
 import com.kingsrook.qqq.backend.core.actions.values.QValueFormatter;
-import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.exceptions.QFormulaException;
 import com.kingsrook.qqq.backend.core.exceptions.QReportingException;
-import com.kingsrook.qqq.backend.core.exceptions.QUserFacingException;
 import com.kingsrook.qqq.backend.core.instances.QMetaDataVariableInterpreter;
 import com.kingsrook.qqq.backend.core.logging.QLogger;
 import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
@@ -58,10 +51,6 @@ import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutp
 import com.kingsrook.qqq.backend.core.model.actions.reporting.ExportInput;
 import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportFormat;
 import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportInput;
-import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportOutput;
-import com.kingsrook.qqq.backend.core.model.actions.tables.QueryHint;
-import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput;
-import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountOutput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.JoinsContext;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
@@ -79,17 +68,11 @@ import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwith
 import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.BackendStepPostRunInput;
 import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.BackendStepPostRunOutput;
 import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
-import com.kingsrook.qqq.backend.core.utils.ObjectUtils;
 import com.kingsrook.qqq.backend.core.utils.Pair;
 import com.kingsrook.qqq.backend.core.utils.StringUtils;
 import com.kingsrook.qqq.backend.core.utils.aggregates.AggregatesInterface;
 import com.kingsrook.qqq.backend.core.utils.aggregates.BigDecimalAggregates;
-import com.kingsrook.qqq.backend.core.utils.aggregates.InstantAggregates;
 import com.kingsrook.qqq.backend.core.utils.aggregates.IntegerAggregates;
-import com.kingsrook.qqq.backend.core.utils.aggregates.LocalDateAggregates;
-import com.kingsrook.qqq.backend.core.utils.aggregates.LongAggregates;
-import com.kingsrook.qqq.backend.core.utils.aggregates.StringAggregates;
-import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
 
 
 /*******************************************************************************
@@ -104,7 +87,7 @@ import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
  ** - summaries (like pivot tables, but called summary to avoid confusion with "native" pivot tables),
  ** - native pivot tables (not initially supported, due to lack of support in fastexcel...).
  *******************************************************************************/
-public class GenerateReportAction extends AbstractQActionFunction<ReportInput, ReportOutput>
+public class GenerateReportAction
 {
    private static final QLogger LOG = QLogger.getLogger(GenerateReportAction.class);
 
@@ -118,67 +101,50 @@ public class GenerateReportAction extends AbstractQActionFunction<ReportInput, R
    // Aggregates: (count:47;sum:10,000;max:2,000;min:15)                                                      //
    // salesSummaryReport > [(state:MO),(city:St.Louis)] > salePrice  > (count:47;sum:10,000;max:2,000;min:15) //
    /////////////////////////////////////////////////////////////////////////////////////////////////////////////
-   Map<String, Map<SummaryKey, Map<String, AggregatesInterface<?, ?>>>> summaryAggregates  = new HashMap<>();
-   Map<String, Map<SummaryKey, Map<String, AggregatesInterface<?, ?>>>> varianceAggregates = new HashMap<>();
+   Map<String, Map<SummaryKey, Map<String, AggregatesInterface<?>>>> summaryAggregates  = new HashMap<>();
+   Map<String, Map<SummaryKey, Map<String, AggregatesInterface<?>>>> varianceAggregates = new HashMap<>();
 
-   Map<String, AggregatesInterface<?, ?>> totalAggregates         = new HashMap<>();
-   Map<String, AggregatesInterface<?, ?>> varianceTotalAggregates = new HashMap<>();
+   Map<String, AggregatesInterface<?>> totalAggregates         = new HashMap<>();
+   Map<String, AggregatesInterface<?>> varianceTotalAggregates = new HashMap<>();
 
+   private QReportMetaData         report;
+   private ReportFormat            reportFormat;
    private ExportStreamerInterface reportStreamer;
-   private List<QReportDataSource> dataSources;
-   private List<QReportView>       views;
-
-   private Map<String, Integer> countByDataSource = new HashMap<>();
 
 
 
    /*******************************************************************************
     **
     *******************************************************************************/
-   public ReportOutput execute(ReportInput reportInput) throws QException
+   public void execute(ReportInput reportInput) throws QException
    {
-      ReportOutput    reportOutput = new ReportOutput();
-      QReportMetaData report       = getReportMetaData(reportInput);
-
-      this.views = report.getViews();
-      this.dataSources = report.getDataSources();
-
-      ReportFormat reportFormat = reportInput.getReportDestination().getReportFormat();
+      report = reportInput.getInstance().getReport(reportInput.getReportName());
+      reportFormat = reportInput.getReportFormat();
       if(reportFormat == null)
       {
          throw new QException("Report format was not specified.");
       }
-
-      if(reportInput.getOverrideExportStreamerSupplier() != null)
-      {
-         reportStreamer = reportInput.getOverrideExportStreamerSupplier().get();
-      }
-      else
-      {
-         reportStreamer = reportFormat.newReportStreamer();
-      }
-
-      reportStreamer.preRun(reportInput.getReportDestination(), views);
+      reportStreamer = reportFormat.newReportStreamer();
 
       ////////////////////////////////////////////////////////////////////////////////////////////////
       // foreach data source, do a query (possibly more than 1, if it goes to multiple table views) //
       ////////////////////////////////////////////////////////////////////////////////////////////////
-      for(QReportDataSource dataSource : dataSources)
+      for(QReportDataSource dataSource : report.getDataSources())
       {
          //////////////////////////////////////////////////////////////////////////////
          // make a list of the views that use this data source for various purposes. //
          //////////////////////////////////////////////////////////////////////////////
-         List<QReportView> dataSourceTableViews = views.stream()
+         List<QReportView> dataSourceTableViews = report.getViews().stream()
             .filter(v -> v.getType().equals(ReportType.TABLE))
             .filter(v -> v.getDataSourceName().equals(dataSource.getName()))
             .toList();
 
-         List<QReportView> dataSourceSummaryViews = views.stream()
+         List<QReportView> dataSourceSummaryViews = report.getViews().stream()
             .filter(v -> v.getType().equals(ReportType.SUMMARY))
             .filter(v -> v.getDataSourceName().equals(dataSource.getName()))
             .toList();
 
-         List<QReportView> dataSourceVariantViews = views.stream()
+         List<QReportView> dataSourceVariantViews = report.getViews().stream()
             .filter(v -> v.getType().equals(ReportType.SUMMARY))
             .filter(v -> v.getVarianceDataSourceName() != null && v.getVarianceDataSourceName().equals(dataSource.getName()))
             .toList();
@@ -217,50 +183,24 @@ public class GenerateReportAction extends AbstractQActionFunction<ReportInput, R
                ////////////////////////////////////////////////////////////////////////////////////
                // start the table-view (e.g., open this tab in xlsx) and then run the query-loop //
                ////////////////////////////////////////////////////////////////////////////////////
-               startTableView(reportInput, dataSource, dataSourceTableView, reportFormat);
+               startTableView(reportInput, dataSource, dataSourceTableView);
                gatherData(reportInput, dataSource, dataSourceTableView, dataSourceSummaryViews, dataSourceVariantViews);
             }
          }
       }
 
-      //////////////////////
-      // add pivot sheets //
-      //////////////////////
-      for(QReportView view : views)
-      {
-         if(view.getType().equals(ReportType.PIVOT))
-         {
-            if(reportFormat.getSupportsNativePivotTables())
-            {
-               startTableView(reportInput, null, view, reportFormat);
-            }
-            else
-            {
-               LOG.warn("Request to render a report with a PIVOT type view, for a format that does not support native pivot tables", logPair("reportFormat", reportFormat));
-            }
-
-            //////////////////////////////////////////////////////////////////////////
-            // there's no data to add to a pivot table, so nothing else to do here. //
-            //////////////////////////////////////////////////////////////////////////
-         }
-      }
-
       outputSummaries(reportInput);
 
-      reportOutput.setTotalRecordCount(countByDataSource.values().stream().mapToInt(Integer::intValue).sum());
-
       reportStreamer.finish();
 
       try
       {
-         reportInput.getReportDestination().getReportOutputStream().close();
+         reportInput.getReportOutputStream().close();
       }
       catch(Exception e)
       {
          throw (new QReportingException("Error completing report", e));
       }
-
-      return (reportOutput);
    }
 
 
@@ -268,48 +208,28 @@ public class GenerateReportAction extends AbstractQActionFunction<ReportInput, R
    /*******************************************************************************
     **
     *******************************************************************************/
-   private QReportMetaData getReportMetaData(ReportInput reportInput) throws QException
+   private void startTableView(ReportInput reportInput, QReportDataSource dataSource, QReportView reportView) throws QException
    {
-      if(reportInput.getReportMetaData() != null)
-      {
-         return reportInput.getReportMetaData();
-      }
+      QTableMetaData table = reportInput.getInstance().getTable(dataSource.getSourceTable());
 
-      if(StringUtils.hasContent(reportInput.getReportName()))
-      {
-         return QContext.getQInstance().getReport(reportInput.getReportName());
-      }
-
-      throw (new QReportingException("ReportInput did not contain required parameters to identify the report being generated"));
-   }
-
-
-
-   /*******************************************************************************
-    **
-    *******************************************************************************/
-   private void startTableView(ReportInput reportInput, QReportDataSource dataSource, QReportView reportView, ReportFormat reportFormat) throws QException
-   {
       QMetaDataVariableInterpreter variableInterpreter = new QMetaDataVariableInterpreter();
       variableInterpreter.addValueMap("input", reportInput.getInputValues());
 
       ExportInput exportInput = new ExportInput();
-      exportInput.setReportDestination(reportInput.getReportDestination());
+      exportInput.setReportFormat(reportFormat);
+      exportInput.setFilename(reportInput.getFilename());
       exportInput.setTitleRow(getTitle(reportView, variableInterpreter));
       exportInput.setIncludeHeaderRow(reportView.getIncludeHeaderRow());
+      exportInput.setReportOutputStream(reportInput.getReportOutputStream());
 
       JoinsContext joinsContext = null;
-      if(dataSource != null)
+      if(StringUtils.hasContent(dataSource.getSourceTable()))
       {
-         if(StringUtils.hasContent(dataSource.getSourceTable()))
-         {
-            joinsContext = new JoinsContext(exportInput.getInstance(), dataSource.getSourceTable(), dataSource.getQueryJoins(), dataSource.getQueryFilter());
-            countDataSourceRecords(reportInput, dataSource, reportFormat);
-         }
+         joinsContext = new JoinsContext(exportInput.getInstance(), dataSource.getSourceTable(), dataSource.getQueryJoins(), dataSource.getQueryFilter());
       }
 
       List<QFieldMetaData> fields = new ArrayList<>();
-      for(QReportField column : CollectionUtils.nonNullList(reportView.getColumns()))
+      for(QReportField column : reportView.getColumns())
       {
          if(column.getIsVirtual())
          {
@@ -321,7 +241,7 @@ public class GenerateReportAction extends AbstractQActionFunction<ReportInput, R
             JoinsContext.FieldAndTableNameOrAlias fieldAndTableNameOrAlias = joinsContext == null ? null : joinsContext.getFieldAndTableNameOrAlias(effectiveFieldName);
             if(fieldAndTableNameOrAlias == null || fieldAndTableNameOrAlias.field() == null)
             {
-               throw new QReportingException("Could not find field named [" + effectiveFieldName + "] in dataSource [" + (dataSource == null ? null : dataSource.getName()) + "]");
+               throw new QReportingException("Could not find field named [" + effectiveFieldName + "] in dataSource [" + dataSource.getName() + "]");
             }
 
             QFieldMetaData field = fieldAndTableNameOrAlias.field().clone();
@@ -335,7 +255,7 @@ public class GenerateReportAction extends AbstractQActionFunction<ReportInput, R
       }
 
       reportStreamer.setDisplayFormats(getDisplayFormatMap(fields));
-      reportStreamer.start(exportInput, fields, reportView.getLabel(), reportView);
+      reportStreamer.start(exportInput, fields, reportView.getLabel());
    }
 
 
@@ -343,36 +263,7 @@ public class GenerateReportAction extends AbstractQActionFunction<ReportInput, R
    /*******************************************************************************
     **
     *******************************************************************************/
-   private void countDataSourceRecords(ReportInput reportInput, QReportDataSource dataSource, ReportFormat reportFormat) throws QException
-   {
-      QQueryFilter queryFilter = dataSource.getQueryFilter() == null ? new QQueryFilter() : dataSource.getQueryFilter().clone();
-      setInputValuesInQueryFilter(reportInput, queryFilter);
-
-      CountInput countInput = new CountInput();
-      countInput.setTableName(dataSource.getSourceTable());
-      countInput.setFilter(queryFilter);
-      countInput.setQueryJoins(dataSource.getQueryJoins());
-      CountOutput countOutput = new CountAction().execute(countInput);
-
-      if(countOutput.getCount() != null)
-      {
-         countByDataSource.put(dataSource.getName(), countOutput.getCount());
-
-         if(reportFormat.getMaxRows() != null && countOutput.getCount() > reportFormat.getMaxRows())
-         {
-            throw (new QUserFacingException("The requested report would include more rows ("
-               + String.format("%,d", countOutput.getCount()) + ") than the maximum allowed ("
-               + String.format("%,d", reportFormat.getMaxRows()) + ") for the selected file format (" + reportFormat + ")."));
-         }
-      }
-   }
-
-
-
-   /*******************************************************************************
-    **
-    *******************************************************************************/
-   private Integer gatherData(ReportInput reportInput, QReportDataSource dataSource, QReportView tableView, List<QReportView> summaryViews, List<QReportView> variantViews) throws QException
+   private void gatherData(ReportInput reportInput, QReportDataSource dataSource, QReportView tableView, List<QReportView> summaryViews, List<QReportView> variantViews) throws QException
    {
       ////////////////////////////////////////////////////////////////////////////////////////
       // check if this view has a transform step - if so, set it up now and run its pre-run //
@@ -382,7 +273,7 @@ public class GenerateReportAction extends AbstractQActionFunction<ReportInput, R
       RunBackendStepOutput  transformStepOutput = null;
       if(tableView != null && tableView.getRecordTransformStep() != null)
       {
-         transformStep = QCodeLoader.getAdHoc(AbstractTransformStep.class, tableView.getRecordTransformStep());
+         transformStep = QCodeLoader.getBackendStep(AbstractTransformStep.class, tableView.getRecordTransformStep());
 
          transformStepInput = new RunBackendStepInput();
          transformStepInput.setValues(reportInput.getInputValues());
@@ -399,9 +290,6 @@ public class GenerateReportAction extends AbstractQActionFunction<ReportInput, R
       RunBackendStepInput   finalTransformStepInput  = transformStepInput;
       RunBackendStepOutput  finalTransformStepOutput = transformStepOutput;
 
-      String        tableLabel    = ObjectUtils.tryElse(() -> QContext.getQInstance().getTable(dataSource.getSourceTable()).getLabel(), Objects.requireNonNullElse(dataSource.getSourceTable(), ""));
-      AtomicInteger consumedCount = new AtomicInteger(0);
-
       /////////////////////////////////////////////////////////////////
       // run a record pipe loop, over the query for this data source //
       /////////////////////////////////////////////////////////////////
@@ -418,8 +306,6 @@ public class GenerateReportAction extends AbstractQActionFunction<ReportInput, R
             queryInput.setTableName(dataSource.getSourceTable());
             queryInput.setFilter(queryFilter);
             queryInput.setQueryJoins(dataSource.getQueryJoins());
-            queryInput.withQueryHint(QueryHint.POTENTIALLY_LARGE_NUMBER_OF_RESULTS);
-            queryInput.withQueryHint(QueryHint.MAY_USE_READ_ONLY_BACKEND);
 
             queryInput.setShouldTranslatePossibleValues(true);
             queryInput.setFieldsToTranslatePossibleValues(setupFieldsToTranslatePossibleValues(reportInput, dataSource, new JoinsContext(reportInput.getInstance(), dataSource.getSourceTable(), dataSource.getQueryJoins(), queryInput.getFilter())));
@@ -459,21 +345,10 @@ public class GenerateReportAction extends AbstractQActionFunction<ReportInput, R
          if(finalTransformStep != null)
          {
             finalTransformStepInput.setRecords(records);
-            finalTransformStep.runOnePage(finalTransformStepInput, finalTransformStepOutput);
+            finalTransformStep.run(finalTransformStepInput, finalTransformStepOutput);
             records = finalTransformStepOutput.getRecords();
          }
 
-         Integer total = countByDataSource.get(dataSource.getName());
-         if(total != null)
-         {
-            reportInput.getAsyncJobCallback().updateStatus("Processing " + tableLabel + " records", consumedCount.get() + 1, total);
-         }
-         else
-         {
-            reportInput.getAsyncJobCallback().updateStatus("Processing " + tableLabel + " records (" + String.format("%,d", consumedCount.get() + 1) + ")");
-         }
-         consumedCount.getAndAdd(records.size());
-
          return (consumeRecords(reportInput, dataSource, records, tableView, summaryViews, variantViews));
       });
 
@@ -484,8 +359,6 @@ public class GenerateReportAction extends AbstractQActionFunction<ReportInput, R
       {
          transformStep.postRun(new BackendStepPostRunInput(transformStepInput), new BackendStepPostRunOutput(transformStepOutput));
       }
-
-      return consumedCount.get();
    }
 
 
@@ -493,11 +366,11 @@ public class GenerateReportAction extends AbstractQActionFunction<ReportInput, R
    /*******************************************************************************
     **
     *******************************************************************************/
-   private Set<String> setupFieldsToTranslatePossibleValues(ReportInput reportInput, QReportDataSource dataSource, JoinsContext joinsContext) throws QException
+   private Set<String> setupFieldsToTranslatePossibleValues(ReportInput reportInput, QReportDataSource dataSource, JoinsContext joinsContext)
    {
       Set<String> fieldsToTranslatePossibleValues = new HashSet<>();
 
-      for(QReportView view : views)
+      for(QReportView view : report.getViews())
       {
          for(QReportField column : CollectionUtils.nonNullList(view.getColumns()))
          {
@@ -511,16 +384,15 @@ public class GenerateReportAction extends AbstractQActionFunction<ReportInput, R
             }
          }
 
-         for(String summaryFieldName : CollectionUtils.nonNullList(view.getSummaryFields()))
+         for(String summaryField : CollectionUtils.nonNullList(view.getPivotFields()))
          {
             ///////////////////////////////////////////////////////////////////////////////
             // all pivotFields that are possible value sources are implicitly translated //
             ///////////////////////////////////////////////////////////////////////////////
-            QTableMetaData    mainTable         = QContext.getQInstance().getTable(dataSource.getSourceTable());
-            FieldAndJoinTable fieldAndJoinTable = getFieldAndJoinTable(mainTable, summaryFieldName);
-            if(fieldAndJoinTable.field().getPossibleValueSourceName() != null)
+            QTableMetaData table = reportInput.getInstance().getTable(dataSource.getSourceTable());
+            if(table.getField(summaryField).getPossibleValueSourceName() != null)
             {
-               fieldsToTranslatePossibleValues.add(summaryFieldName);
+               fieldsToTranslatePossibleValues.add(summaryField);
             }
          }
       }
@@ -533,33 +405,7 @@ public class GenerateReportAction extends AbstractQActionFunction<ReportInput, R
    /*******************************************************************************
     **
     *******************************************************************************/
-   public static FieldAndJoinTable getFieldAndJoinTable(QTableMetaData mainTable, String fieldName) throws QException
-   {
-      if(fieldName.indexOf('.') > -1)
-      {
-         String joinTableName = fieldName.replaceAll("\\..*", "");
-         String joinFieldName = fieldName.replaceAll(".*\\.", "");
-
-         QTableMetaData joinTable = QContext.getQInstance().getTable(joinTableName);
-         if(joinTable == null)
-         {
-            throw (new QException("Unrecognized join table name: " + joinTableName));
-         }
-
-         return new FieldAndJoinTable(joinTable.getField(joinFieldName), joinTable);
-      }
-      else
-      {
-         return new FieldAndJoinTable(mainTable.getField(fieldName), mainTable);
-      }
-   }
-
-
-
-   /*******************************************************************************
-    **
-    *******************************************************************************/
-   private void setInputValuesInQueryFilter(ReportInput reportInput, QQueryFilter queryFilter) throws QException
+   private void setInputValuesInQueryFilter(ReportInput reportInput, QQueryFilter queryFilter)
    {
       if(queryFilter == null || queryFilter.getCriteria() == null)
       {
@@ -648,27 +494,26 @@ public class GenerateReportAction extends AbstractQActionFunction<ReportInput, R
    /*******************************************************************************
     **
     *******************************************************************************/
-   private void addRecordsToSummaryAggregates(QReportView view, QTableMetaData table, List<QRecord> records, Map<String, Map<SummaryKey, Map<String, AggregatesInterface<?, ?>>>> aggregatesMap) throws QException
+   private void addRecordsToSummaryAggregates(QReportView view, QTableMetaData table, List<QRecord> records, Map<String, Map<SummaryKey, Map<String, AggregatesInterface<?>>>> aggregatesMap)
    {
-      Map<SummaryKey, Map<String, AggregatesInterface<?, ?>>> viewAggregates = aggregatesMap.computeIfAbsent(view.getName(), (name) -> new HashMap<>());
+      Map<SummaryKey, Map<String, AggregatesInterface<?>>> viewAggregates = aggregatesMap.computeIfAbsent(view.getName(), (name) -> new HashMap<>());
 
       for(QRecord record : records)
       {
          SummaryKey key = new SummaryKey();
-         for(String summaryFieldName : view.getSummaryFields())
+         for(String summaryField : view.getPivotFields())
          {
-            FieldAndJoinTable fieldAndJoinTable = getFieldAndJoinTable(table, summaryFieldName);
-            Serializable      summaryValue      = record.getValue(summaryFieldName);
-            if(fieldAndJoinTable.field().getPossibleValueSourceName() != null)
+            Serializable summaryValue = record.getValue(summaryField);
+            if(table.getField(summaryField).getPossibleValueSourceName() != null)
             {
                //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
                // so, this is kinda a thing - where we implicitly use possible-value labels (e.g., display values) for pivot fields... //
                //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-               summaryValue = record.getDisplayValue(summaryFieldName);
+               summaryValue = record.getDisplayValue(summaryField);
             }
-            key.add(summaryFieldName, summaryValue);
+            key.add(summaryField, summaryValue);
 
-            if(view.getIncludeSummarySubTotals() && key.getKeys().size() < view.getSummaryFields().size())
+            if(view.getIncludePivotSubTotals() && key.getKeys().size() < view.getPivotFields().size())
             {
                /////////////////////////////////////////////////////////////////////////////////////////
                // be careful here, with these key objects, and their identity, being used as map keys //
@@ -687,9 +532,9 @@ public class GenerateReportAction extends AbstractQActionFunction<ReportInput, R
    /*******************************************************************************
     **
     *******************************************************************************/
-   private void addRecordToSummaryKeyAggregates(QTableMetaData table, QRecord record, Map<SummaryKey, Map<String, AggregatesInterface<?, ?>>> viewAggregates, SummaryKey key) throws QException
+   private void addRecordToSummaryKeyAggregates(QTableMetaData table, QRecord record, Map<SummaryKey, Map<String, AggregatesInterface<?>>> viewAggregates, SummaryKey key)
    {
-      Map<String, AggregatesInterface<?, ?>> keyAggregates = viewAggregates.computeIfAbsent(key, (name) -> new HashMap<>());
+      Map<String, AggregatesInterface<?>> keyAggregates = viewAggregates.computeIfAbsent(key, (name) -> new HashMap<>());
       addRecordToAggregatesMap(table, record, keyAggregates);
    }
 
@@ -698,74 +543,23 @@ public class GenerateReportAction extends AbstractQActionFunction<ReportInput, R
    /*******************************************************************************
     **
     *******************************************************************************/
-   private void addRecordToAggregatesMap(QTableMetaData table, QRecord record, Map<String, AggregatesInterface<?, ?>> aggregatesMap) throws QException
+   private void addRecordToAggregatesMap(QTableMetaData table, QRecord record, Map<String, AggregatesInterface<?>> aggregatesMap)
    {
-      //////////////////////////////////////////////////////////////////////////////////////
-      // todo - an optimization could be, to only compute aggregates that we'll need...   //
-      // Only if we measure and see this to be slow - it may be, lots of BigDecimal math? //
-      //////////////////////////////////////////////////////////////////////////////////////
-      for(String fieldName : record.getValues().keySet())
+      for(QFieldMetaData field : table.getFields().values())
       {
-         QFieldMetaData field = null;
-         try
-         {
-            //////////////////////////////////////////////////////
-            // todo - memoize this, if we ever need to optimize //
-            //////////////////////////////////////////////////////
-            FieldAndJoinTable fieldAndJoinTable = getFieldAndJoinTable(table, fieldName);
-            field = fieldAndJoinTable.field();
-         }
-         catch(Exception e)
-         {
-            //////////////////////////////////////////////////////////////////////////////////////
-            // for non-real-fields... let's skip for now - but maybe treat as string in future? //
-            //////////////////////////////////////////////////////////////////////////////////////
-            LOG.debug("Couldn't find field in table qInstance - won't compute aggregates for it", logPair("fieldName", fieldName));
-            continue;
-         }
-
-         if(StringUtils.hasContent(field.getPossibleValueSourceName()))
+         if(field.getType().equals(QFieldType.INTEGER))
          {
             @SuppressWarnings("unchecked")
-            AggregatesInterface<String, ?> fieldAggregates = (AggregatesInterface<String, ?>) aggregatesMap.computeIfAbsent(fieldName, (name) -> new StringAggregates());
-            fieldAggregates.add(record.getDisplayValue(fieldName));
-         }
-         else if(field.getType().equals(QFieldType.INTEGER))
-         {
-            @SuppressWarnings("unchecked")
-            AggregatesInterface<Integer, ?> fieldAggregates = (AggregatesInterface<Integer, ?>) aggregatesMap.computeIfAbsent(fieldName, (name) -> new IntegerAggregates());
-            fieldAggregates.add(record.getValueInteger(fieldName));
-         }
-         else if(field.getType().equals(QFieldType.LONG))
-         {
-            @SuppressWarnings("unchecked")
-            AggregatesInterface<Long, ?> fieldAggregates = (AggregatesInterface<Long, ?>) aggregatesMap.computeIfAbsent(fieldName, (name) -> new LongAggregates());
-            fieldAggregates.add(record.getValueLong(fieldName));
+            AggregatesInterface<Integer> fieldAggregates = (AggregatesInterface<Integer>) aggregatesMap.computeIfAbsent(field.getName(), (name) -> new IntegerAggregates());
+            fieldAggregates.add(record.getValueInteger(field.getName()));
          }
          else if(field.getType().equals(QFieldType.DECIMAL))
          {
             @SuppressWarnings("unchecked")
-            AggregatesInterface<BigDecimal, ?> fieldAggregates = (AggregatesInterface<BigDecimal, ?>) aggregatesMap.computeIfAbsent(fieldName, (name) -> new BigDecimalAggregates());
-            fieldAggregates.add(record.getValueBigDecimal(fieldName));
-         }
-         else if(field.getType().equals(QFieldType.DATE_TIME))
-         {
-            @SuppressWarnings("unchecked")
-            AggregatesInterface<Instant, ?> fieldAggregates = (AggregatesInterface<Instant, ?>) aggregatesMap.computeIfAbsent(fieldName, (name) -> new InstantAggregates());
-            fieldAggregates.add(record.getValueInstant(fieldName));
-         }
-         else if(field.getType().equals(QFieldType.DATE))
-         {
-            @SuppressWarnings("unchecked")
-            AggregatesInterface<LocalDate, ?> fieldAggregates = (AggregatesInterface<LocalDate, ?>) aggregatesMap.computeIfAbsent(fieldName, (name) -> new LocalDateAggregates());
-            fieldAggregates.add(record.getValueLocalDate(fieldName));
-         }
-         if(field.getType().isStringLike())
-         {
-            @SuppressWarnings("unchecked")
-            AggregatesInterface<String, ?> fieldAggregates = (AggregatesInterface<String, ?>) aggregatesMap.computeIfAbsent(fieldName, (name) -> new StringAggregates());
-            fieldAggregates.add(record.getValueString(fieldName));
+            AggregatesInterface<BigDecimal> fieldAggregates = (AggregatesInterface<BigDecimal>) aggregatesMap.computeIfAbsent(field.getName(), (name) -> new BigDecimalAggregates());
+            fieldAggregates.add(record.getValueBigDecimal(field.getName()));
          }
+         // todo - more types (dates, at least?)
       }
    }
 
@@ -774,22 +568,24 @@ public class GenerateReportAction extends AbstractQActionFunction<ReportInput, R
    /*******************************************************************************
     **
     *******************************************************************************/
-   private void outputSummaries(ReportInput reportInput) throws QException
+   private void outputSummaries(ReportInput reportInput) throws QReportingException, QFormulaException
    {
-      List<QReportView> reportViews = views.stream().filter(v -> v.getType().equals(ReportType.SUMMARY)).toList();
+      List<QReportView> reportViews = report.getViews().stream().filter(v -> v.getType().equals(ReportType.SUMMARY)).toList();
       for(QReportView view : reportViews)
       {
-         QReportDataSource dataSource    = getDataSource(view.getDataSourceName());
+         QReportDataSource dataSource    = report.getDataSource(view.getDataSourceName());
          QTableMetaData    table         = reportInput.getInstance().getTable(dataSource.getSourceTable());
          SummaryOutput     summaryOutput = computeSummaryRowsForView(reportInput, view, table);
 
          ExportInput exportInput = new ExportInput();
-         exportInput.setReportDestination(reportInput.getReportDestination());
+         exportInput.setReportFormat(reportFormat);
+         exportInput.setFilename(reportInput.getFilename());
          exportInput.setTitleRow(summaryOutput.titleRow);
          exportInput.setIncludeHeaderRow(view.getIncludeHeaderRow());
+         exportInput.setReportOutputStream(reportInput.getReportOutputStream());
 
          reportStreamer.setDisplayFormats(getDisplayFormatMap(view));
-         reportStreamer.start(exportInput, getFields(table, view), view.getLabel(), view);
+         reportStreamer.start(exportInput, getFields(table, view), view.getLabel());
 
          reportStreamer.addRecords(summaryOutput.summaryRows); // todo - what if this set is huge?
 
@@ -802,24 +598,6 @@ public class GenerateReportAction extends AbstractQActionFunction<ReportInput, R
 
 
 
-   /*******************************************************************************
-    **
-    *******************************************************************************/
-   private QReportDataSource getDataSource(String dataSourceName)
-   {
-      for(QReportDataSource dataSource : CollectionUtils.nonNullList(dataSources))
-      {
-         if(dataSource.getName().equals(dataSourceName))
-         {
-            return (dataSource);
-         }
-      }
-
-      return (null);
-   }
-
-
-
    /*******************************************************************************
     **
     *******************************************************************************/
@@ -847,13 +625,13 @@ public class GenerateReportAction extends AbstractQActionFunction<ReportInput, R
    /*******************************************************************************
     **
     *******************************************************************************/
-   private List<QFieldMetaData> getFields(QTableMetaData table, QReportView view) throws QException
+   private List<QFieldMetaData> getFields(QTableMetaData table, QReportView view)
    {
       List<QFieldMetaData> fields = new ArrayList<>();
-      for(String summaryFieldName : view.getSummaryFields())
+      for(String pivotField : view.getPivotFields())
       {
-         FieldAndJoinTable fieldAndJoinTable = getFieldAndJoinTable(table, summaryFieldName);
-         fields.add(new QFieldMetaData(summaryFieldName, fieldAndJoinTable.field().getType()).withLabel(fieldAndJoinTable.field().getLabel())); // todo do we need the type?  if so need table as input here
+         QFieldMetaData field = table.getField(pivotField);
+         fields.add(new QFieldMetaData(pivotField, field.getType()).withLabel(field.getLabel())); // todo do we need the type?  if so need table as input here
       }
       for(QReportField column : view.getColumns())
       {
@@ -883,11 +661,11 @@ public class GenerateReportAction extends AbstractQActionFunction<ReportInput, R
       // create summary rows //
       /////////////////////////
       List<QRecord> summaryRows = new ArrayList<>();
-      for(Map.Entry<SummaryKey, Map<String, AggregatesInterface<?, ?>>> entry : summaryAggregates.getOrDefault(view.getName(), Collections.emptyMap()).entrySet())
+      for(Map.Entry<SummaryKey, Map<String, AggregatesInterface<?>>> entry : summaryAggregates.getOrDefault(view.getName(), Collections.emptyMap()).entrySet())
       {
-         SummaryKey                             summaryKey      = entry.getKey();
-         Map<String, AggregatesInterface<?, ?>> fieldAggregates = entry.getValue();
-         Map<String, Serializable>              summaryValues   = getSummaryValuesForInterpreter(fieldAggregates);
+         SummaryKey                          summaryKey      = entry.getKey();
+         Map<String, AggregatesInterface<?>> fieldAggregates = entry.getValue();
+         Map<String, Serializable>           summaryValues   = getSummaryValuesForInterpreter(fieldAggregates);
          variableInterpreter.addValueMap("pivot", summaryValues);
          variableInterpreter.addValueMap("summary", summaryValues);
 
@@ -896,9 +674,9 @@ public class GenerateReportAction extends AbstractQActionFunction<ReportInput, R
 
          if(!varianceAggregates.isEmpty())
          {
-            Map<SummaryKey, Map<String, AggregatesInterface<?, ?>>> varianceMap    = varianceAggregates.getOrDefault(view.getName(), Collections.emptyMap());
-            Map<String, AggregatesInterface<?, ?>>                  varianceSubMap = varianceMap.getOrDefault(summaryKey, Collections.emptyMap());
-            Map<String, Serializable>                               varianceValues = getSummaryValuesForInterpreter(varianceSubMap);
+            Map<SummaryKey, Map<String, AggregatesInterface<?>>> varianceMap    = varianceAggregates.getOrDefault(view.getName(), Collections.emptyMap());
+            Map<String, AggregatesInterface<?>>                  varianceSubMap = varianceMap.getOrDefault(summaryKey, Collections.emptyMap());
+            Map<String, Serializable>                            varianceValues = getSummaryValuesForInterpreter(varianceSubMap);
             variableInterpreter.addValueMap("variancePivot", varianceValues);
             variableInterpreter.addValueMap("variance", varianceValues);
          }
@@ -917,7 +695,7 @@ public class GenerateReportAction extends AbstractQActionFunction<ReportInput, R
          ///////////////////////////////////////////////////////////////////////////////
          // for summary subtotals, add the text "Total" to the last field in this key //
          ///////////////////////////////////////////////////////////////////////////////
-         if(summaryKey.getKeys().size() < view.getSummaryFields().size())
+         if(summaryKey.getKeys().size() < view.getPivotFields().size())
          {
             String fieldName = summaryKey.getKeys().get(summaryKey.getKeys().size() - 1).getA();
             summaryRow.setValue(fieldName, summaryRow.getValueString(fieldName) + " Total");
@@ -955,11 +733,11 @@ public class GenerateReportAction extends AbstractQActionFunction<ReportInput, R
       {
          totalRow = new QRecord();
 
-         for(String summaryField : view.getSummaryFields())
+         for(String pivotField : view.getPivotFields())
          {
             if(totalRow.getValues().isEmpty())
             {
-               totalRow.setValue(summaryField, "Totals");
+               totalRow.setValue(pivotField, "Totals");
             }
          }
 
@@ -1079,24 +857,18 @@ public class GenerateReportAction extends AbstractQActionFunction<ReportInput, R
    /*******************************************************************************
     **
     *******************************************************************************/
-   private Map<String, Serializable> getSummaryValuesForInterpreter(Map<String, AggregatesInterface<?, ?>> fieldAggregates)
+   private Map<String, Serializable> getSummaryValuesForInterpreter(Map<String, AggregatesInterface<?>> fieldAggregates)
    {
       Map<String, Serializable> summaryValuesForInterpreter = new HashMap<>();
-      for(Map.Entry<String, AggregatesInterface<?, ?>> subEntry : fieldAggregates.entrySet())
+      for(Map.Entry<String, AggregatesInterface<?>> subEntry : fieldAggregates.entrySet())
       {
-         String                    fieldName  = subEntry.getKey();
-         AggregatesInterface<?, ?> aggregates = subEntry.getValue();
+         String                 fieldName  = subEntry.getKey();
+         AggregatesInterface<?> aggregates = subEntry.getValue();
          summaryValuesForInterpreter.put("sum." + fieldName, aggregates.getSum());
          summaryValuesForInterpreter.put("count." + fieldName, aggregates.getCount());
-         summaryValuesForInterpreter.put("count_nums." + fieldName, aggregates.getCount());
          summaryValuesForInterpreter.put("min." + fieldName, aggregates.getMin());
          summaryValuesForInterpreter.put("max." + fieldName, aggregates.getMax());
          summaryValuesForInterpreter.put("average." + fieldName, aggregates.getAverage());
-         summaryValuesForInterpreter.put("product." + fieldName, aggregates.getProduct());
-         summaryValuesForInterpreter.put("var." + fieldName, aggregates.getVariance());
-         summaryValuesForInterpreter.put("varp." + fieldName, aggregates.getVarP());
-         summaryValuesForInterpreter.put("std_dev." + fieldName, aggregates.getStandardDeviation());
-         summaryValuesForInterpreter.put("std_devp." + fieldName, aggregates.getStdDevP());
       }
       return summaryValuesForInterpreter;
    }
@@ -1110,27 +882,4 @@ public class GenerateReportAction extends AbstractQActionFunction<ReportInput, R
    {
    }
 
-
-
-   /*******************************************************************************
-    **
-    *******************************************************************************/
-   public record FieldAndJoinTable(QFieldMetaData field, QTableMetaData joinTable)
-   {
-
-      /*******************************************************************************
-       **
-       *******************************************************************************/
-      public String getLabel(QTableMetaData mainTable)
-      {
-         if(mainTable.getName().equals(joinTable.getName()))
-         {
-            return (field.getLabel());
-         }
-         else
-         {
-            return (joinTable.getLabel() + ": " + field.getLabel());
-         }
-      }
-   }
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/JsonExportStreamer.java

@@ -26,23 +26,17 @@ import java.io.IOException;
 import java.io.OutputStream;
 import java.io.Serializable;
 import java.nio.charset.StandardCharsets;
-import java.util.Arrays;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
-import java.util.Optional;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
 import com.kingsrook.qqq.backend.core.exceptions.QReportingException;
 import com.kingsrook.qqq.backend.core.logging.QLogger;
 import com.kingsrook.qqq.backend.core.model.actions.reporting.ExportInput;
-import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportDestination;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
-import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportView;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 import com.kingsrook.qqq.backend.core.utils.JsonUtils;
-import com.kingsrook.qqq.backend.core.utils.memoization.Memoization;
+import com.kingsrook.qqq.backend.core.utils.StringUtils;
 
 
 /*******************************************************************************
@@ -52,23 +46,13 @@ public class JsonExportStreamer implements ExportStreamerInterface
 {
    private static final QLogger LOG = QLogger.getLogger(JsonExportStreamer.class);
 
-   private boolean prettyPrint = true;
-
    private ExportInput          exportInput;
    private QTableMetaData       table;
    private List<QFieldMetaData> fields;
    private OutputStream         outputStream;
 
-   private boolean multipleViews       = false;
-   private boolean haveStartedAnyViews = false;
-
-   private boolean needCommaBeforeRecord = false;
-
-   private byte[] indent       = new byte[0];
-   private String indentString = "";
-
-   private Pattern                     colonLetterPattern    = Pattern.compile(":([A-Z]+)($|[A-Z][a-z])");
-   private Memoization<String, String> fieldLabelMemoization = new Memoization<>();
+   private boolean needComma   = false;
+   private boolean prettyPrint = true;
 
 
 
@@ -85,124 +69,21 @@ public class JsonExportStreamer implements ExportStreamerInterface
     **
     *******************************************************************************/
    @Override
-   public void preRun(ReportDestination reportDestination, List<QReportView> views) throws QReportingException
-   {
-      outputStream = reportDestination.getReportOutputStream();
-
-      if(views.size() > 1)
-      {
-         multipleViews = true;
-      }
-
-      if(multipleViews)
-      {
-         try
-         {
-            indentIfPretty(outputStream);
-            outputStream.write('[');
-            newlineIfPretty(outputStream);
-            increaseIndent();
-         }
-         catch(IOException e)
-         {
-            throw (new QReportingException("Error starting report output", e));
-         }
-      }
-   }
-
-
-
-   /*******************************************************************************
-    **
-    *******************************************************************************/
-   @Override
-   public void start(ExportInput exportInput, List<QFieldMetaData> fields, String label, QReportView view) throws QReportingException
+   public void start(ExportInput exportInput, List<QFieldMetaData> fields, String label) throws QReportingException
    {
       this.exportInput = exportInput;
       this.fields = fields;
       table = exportInput.getTable();
-
-      needCommaBeforeRecord = false;
+      outputStream = this.exportInput.getReportOutputStream();
 
       try
       {
-         if(multipleViews)
-         {
-            if(haveStartedAnyViews)
-            {
-               /////////////////////////
-               // close the last view //
-               /////////////////////////
-               newlineIfPretty(outputStream);
-
-               decreaseIndent();
-               indentIfPretty(outputStream);
-               outputStream.write(']');
-               newlineIfPretty(outputStream);
-
-               decreaseIndent();
-               indentIfPretty(outputStream);
-               outputStream.write('}');
-               outputStream.write(',');
-               newlineIfPretty(outputStream);
-            }
-
-            /////////////////////////////////////////////////////////////
-            // open a new view, as an object, with a name & data entry //
-            /////////////////////////////////////////////////////////////
-            indentIfPretty(outputStream);
-            outputStream.write('{');
-            newlineIfPretty(outputStream);
-            increaseIndent();
-
-            indentIfPretty(outputStream);
-            outputStream.write(String.format("""
-               "name":"%s",""", label).getBytes(StandardCharsets.UTF_8));
-            newlineIfPretty(outputStream);
-
-            indentIfPretty(outputStream);
-            outputStream.write("""
-               "data":""".getBytes(StandardCharsets.UTF_8));
-            newlineIfPretty(outputStream);
-         }
-
-         //////////////////////////////////////////////
-         // start the array of entries for this view //
-         //////////////////////////////////////////////
-         indentIfPretty(outputStream);
          outputStream.write('[');
-         increaseIndent();
       }
       catch(IOException e)
       {
          throw (new QReportingException("Error starting report output", e));
       }
-
-      haveStartedAnyViews = true;
-   }
-
-
-
-   /*******************************************************************************
-    **
-    *******************************************************************************/
-   private void increaseIndent()
-   {
-      indent = new byte[indent.length + 3];
-      Arrays.fill(indent, (byte) ' ');
-      indentString = new String(indent);
-   }
-
-
-
-   /*******************************************************************************
-    **
-    *******************************************************************************/
-   private void decreaseIndent()
-   {
-      indent = new byte[Math.max(0, indent.length - 3)];
-      Arrays.fill(indent, (byte) ' ');
-      indentString = new String(indent);
    }
 
 
@@ -230,7 +111,7 @@ public class JsonExportStreamer implements ExportStreamerInterface
    {
       try
       {
-         if(needCommaBeforeRecord)
+         if(needComma)
          {
             outputStream.write(',');
          }
@@ -238,25 +119,21 @@ public class JsonExportStreamer implements ExportStreamerInterface
          Map<String, Serializable> mapForJson = new LinkedHashMap<>();
          for(QFieldMetaData field : fields)
          {
-            mapForJson.put(getLabelForJson(field), qRecord.getValue(field.getName()));
+            String labelForJson = StringUtils.lcFirst(field.getLabel().replace(" ", ""));
+            mapForJson.put(labelForJson, qRecord.getValue(field.getName()));
          }
 
          String json = prettyPrint ? JsonUtils.toPrettyJson(mapForJson) : JsonUtils.toJson(mapForJson);
-         if(prettyPrint)
-         {
-            json = json.replaceAll("(?s)\n", "\n" + indentString);
-         }
 
          if(prettyPrint)
          {
             outputStream.write('\n');
          }
 
-         indentIfPretty(outputStream);
          outputStream.write(json.getBytes(StandardCharsets.UTF_8));
 
          outputStream.flush(); // todo - less often?
-         needCommaBeforeRecord = true;
+         needComma = true;
       }
       catch(Exception e)
       {
@@ -266,73 +143,6 @@ public class JsonExportStreamer implements ExportStreamerInterface
 
 
 
-   /*******************************************************************************
-    **
-    *******************************************************************************/
-   String getLabelForJson(QFieldMetaData field)
-   {
-      //////////////////////////////////////////////////////////////////////////
-      // memoize, to avoid running these regex/replacements millions of times //
-      //////////////////////////////////////////////////////////////////////////
-      Optional<String> result = fieldLabelMemoization.getResult(field.getName(), fieldName ->
-      {
-         String labelForJson = field.getLabel().replace(" ", "");
-
-         /////////////////////////////////////////////////////////////////////////////
-         // now fix up any field-name-parts after the table: portion of a join name //
-         // lineItem:SKU to become lineItem:sku                                     //
-         // parcel:SLAStatus to become parcel:slaStatus                             //
-         // order:Client to become order:client                                     //
-         /////////////////////////////////////////////////////////////////////////////
-         Matcher allCaps                          = Pattern.compile("^[A-Z]+$").matcher(labelForJson);
-         Matcher startsAllCapsThenNextWordMatcher = Pattern.compile("([A-Z]+)([A-Z][a-z].*)").matcher(labelForJson);
-         Matcher startsOneCapMatcher              = Pattern.compile("([A-Z])(.*)").matcher(labelForJson);
-
-         if(allCaps.matches())
-         {
-            labelForJson = allCaps.replaceAll(m -> m.group().toLowerCase());
-         }
-         else if(startsAllCapsThenNextWordMatcher.matches())
-         {
-            labelForJson = startsAllCapsThenNextWordMatcher.replaceAll(m -> m.group(1).toLowerCase() + m.group(2));
-         }
-         else if(startsOneCapMatcher.matches())
-         {
-            labelForJson = startsOneCapMatcher.replaceAll(m -> m.group(1).toLowerCase() + m.group(2));
-         }
-
-         /////////////////////////////////////////////////////////////////////////////
-         // now fix up any field-name-parts after the table: portion of a join name //
-         // lineItem:SKU to become lineItem:sku                                     //
-         // parcel:SLAStatus to become parcel:slaStatus                             //
-         // order:Client to become order:client                                     //
-         /////////////////////////////////////////////////////////////////////////////
-         Matcher colonThenAllCapsThenEndMatcher      = Pattern.compile("(.*:)([A-Z]+)$").matcher(labelForJson);
-         Matcher colonThenAllCapsThenNextWordMatcher = Pattern.compile("(.*:)([A-Z]+)([A-Z][a-z].*)").matcher(labelForJson);
-         Matcher colonThenOneCapMatcher              = Pattern.compile("(.*:)([A-Z])(.*)").matcher(labelForJson);
-
-         if(colonThenAllCapsThenEndMatcher.matches())
-         {
-            labelForJson = colonThenAllCapsThenEndMatcher.replaceAll(m -> m.group(1) + m.group(2).toLowerCase());
-         }
-         else if(colonThenAllCapsThenNextWordMatcher.matches())
-         {
-            labelForJson = colonThenAllCapsThenNextWordMatcher.replaceAll(m -> m.group(1) + m.group(2).toLowerCase() + m.group(3));
-         }
-         else if(colonThenOneCapMatcher.matches())
-         {
-            labelForJson = colonThenOneCapMatcher.replaceAll(m -> m.group(1) + m.group(2).toLowerCase() + m.group(3));
-         }
-
-         System.out.println("Label: " + labelForJson);
-         return (labelForJson);
-      });
-
-      return result.orElse(field.getName());
-   }
-
-
-
    /*******************************************************************************
     **
     *******************************************************************************/
@@ -352,34 +162,11 @@ public class JsonExportStreamer implements ExportStreamerInterface
    {
       try
       {
-         //////////////////////////////////////////////
-         // close the array of entries for this view //
-         //////////////////////////////////////////////
-         newlineIfPretty(outputStream);
-
-         decreaseIndent();
-         indentIfPretty(outputStream);
-         outputStream.write(']');
-         newlineIfPretty(outputStream);
-
-         if(multipleViews)
+         if(prettyPrint)
          {
-            ////////////////////////////////////////////
-            // close this view, if there are multiple //
-            ////////////////////////////////////////////
-            decreaseIndent();
-            indentIfPretty(outputStream);
-            outputStream.write('}');
-            newlineIfPretty(outputStream);
-
-            /////////////////////////////
-            // close the list of views //
-            /////////////////////////////
-            decreaseIndent();
-            indentIfPretty(outputStream);
-            outputStream.write(']');
-            newlineIfPretty(outputStream);
+            outputStream.write('\n');
          }
+         outputStream.write(']');
       }
       catch(IOException e)
       {
@@ -387,30 +174,4 @@ public class JsonExportStreamer implements ExportStreamerInterface
       }
    }
 
-
-
-   /*******************************************************************************
-    **
-    *******************************************************************************/
-   private void newlineIfPretty(OutputStream outputStream) throws IOException
-   {
-      if(prettyPrint)
-      {
-         outputStream.write('\n');
-      }
-   }
-
-
-
-   /*******************************************************************************
-    **
-    *******************************************************************************/
-   private void indentIfPretty(OutputStream outputStream) throws IOException
-   {
-      if(prettyPrint)
-      {
-         outputStream.write(indent);
-      }
-   }
-
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/ListOfMapsExportStreamer.java

@@ -31,7 +31,6 @@ import com.kingsrook.qqq.backend.core.logging.QLogger;
 import com.kingsrook.qqq.backend.core.model.actions.reporting.ExportInput;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
-import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportView;
 
 
 /*******************************************************************************
@@ -88,7 +87,7 @@ public class ListOfMapsExportStreamer implements ExportStreamerInterface
     **
     *******************************************************************************/
    @Override
-   public void start(ExportInput exportInput, List<QFieldMetaData> fields, String label, QReportView view) throws QReportingException
+   public void start(ExportInput exportInput, List<QFieldMetaData> fields, String label) throws QReportingException
    {
       this.exportInput = exportInput;
       this.fields = fields;

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/RecordPipe.java

@@ -24,7 +24,6 @@ package com.kingsrook.qqq.backend.core.actions.reporting;
 
 import java.util.ArrayList;
 import java.util.List;
-import java.util.Objects;
 import java.util.concurrent.ArrayBlockingQueue;
 import java.util.concurrent.TimeUnit;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
@@ -44,10 +43,8 @@ public class RecordPipe
 
    private static final long BLOCKING_SLEEP_MILLIS = 100;
    private static final long MAX_SLEEP_LOOP_MILLIS = 300_000; // 5 minutes
-   private static final int  DEFAULT_CAPACITY      = 1_000;
 
-   private int capacity = DEFAULT_CAPACITY;
-   private ArrayBlockingQueue<QRecord> queue = new ArrayBlockingQueue<>(capacity);
+   private ArrayBlockingQueue<QRecord> queue = new ArrayBlockingQueue<>(1_000);
 
    private boolean isTerminated = false;
 
@@ -72,13 +69,10 @@ public class RecordPipe
 
    /*******************************************************************************
     ** Construct a record pipe, with an alternative capacity for the internal queue.
-    **
-    ** overrideCapacity is allowed to be null - in which case, DEFAULT_CAPACITY is used.
     *******************************************************************************/
    public RecordPipe(Integer overrideCapacity)
    {
-      this.capacity = Objects.requireNonNullElse(overrideCapacity, DEFAULT_CAPACITY);
-      queue = new ArrayBlockingQueue<>(this.capacity);
+      queue = new ArrayBlockingQueue<>(overrideCapacity);
    }
 
 
@@ -142,7 +136,7 @@ public class RecordPipe
          {
             if(now - sleepLoopStartTime > MAX_SLEEP_LOOP_MILLIS)
             {
-               LOG.warn("Giving up adding record to pipe, due to pipe being full for more than " + MAX_SLEEP_LOOP_MILLIS + " millis");
+               LOG.warn("Giving up adding record to pipe, due to pipe being full for more than {} millis", MAX_SLEEP_LOOP_MILLIS);
                throw (new IllegalStateException("Giving up adding record to pipe, due to pipe staying full too long."));
             }
             LOG.trace("Record pipe.add failed (due to full pipe).  Blocking.");
@@ -219,14 +213,4 @@ public class RecordPipe
       this.postRecordActions = postRecordActions;
    }
 
-
-
-   /*******************************************************************************
-    ** Getter for capacity
-    **
-    *******************************************************************************/
-   public int getCapacity()
-   {
-      return capacity;
-   }
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/ReportUtils.java

@@ -1,51 +0,0 @@
-/*
- * QQQ - Low-code Application Framework for Engineers.
- * Copyright (C) 2021-2024.  Kingsrook, LLC
- * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
- * contact@kingsrook.com
- * https://github.com/Kingsrook/
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program.  If not, see <https://www.gnu.org/licenses/>.
- */
-
-package com.kingsrook.qqq.backend.core.actions.reporting;
-
-
-import java.util.List;
-import java.util.Optional;
-import com.kingsrook.qqq.backend.core.exceptions.QReportingException;
-import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportView;
-
-
-/*******************************************************************************
- **
- *******************************************************************************/
-public class ReportUtils
-{
-
-   /*******************************************************************************
-    **
-    *******************************************************************************/
-   public static QReportView getSourceViewForPivotTableView(List<QReportView> views, QReportView pivotTableView) throws QReportingException
-   {
-      Optional<QReportView> sourceView = views.stream().filter(v -> v.getName().equals(pivotTableView.getPivotTableSourceViewName())).findFirst();
-      if(sourceView.isEmpty())
-      {
-         throw (new QReportingException("Could not find data view [" + pivotTableView.getPivotTableSourceViewName() + "] for pivot table view [" + pivotTableView.getName() + "]"));
-      }
-
-      return sourceView.get();
-   }
-
-}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/excel/fastexcel/PlainFastExcelStyler.java

@@ -1,43 +0,0 @@
-/*
- * QQQ - Low-code Application Framework for Engineers.
- * Copyright (C) 2021-2024.  Kingsrook, LLC
- * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
- * contact@kingsrook.com
- * https://github.com/Kingsrook/
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program.  If not, see <https://www.gnu.org/licenses/>.
- */
-
-package com.kingsrook.qqq.backend.core.actions.reporting.excel.fastexcel;
-
-
-import org.dhatim.fastexcel.StyleSetter;
-
-
-/*******************************************************************************
- ** Excel styler that does nothing - just takes defaults (which are all no-op) from the interface.
- *******************************************************************************/
-public class PlainFastExcelStyler implements FastExcelStylerInterface
-{
-
-   /*******************************************************************************
-    ** ... sorry, but adding this gives us test coverage on this class, even though
-    ** we're just deferring to super...
-    *******************************************************************************/
-   @Override
-   public void styleHeaderRow(StyleSetter headerRowStyle)
-   {
-      FastExcelStylerInterface.super.styleHeaderRow(headerRowStyle);
-   }
-}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/excel/poi/BoldHeaderAndFooterPoiExcelStyler.java

@@ -1,93 +0,0 @@
-/*
- * QQQ - Low-code Application Framework for Engineers.
- * Copyright (C) 2021-2024.  Kingsrook, LLC
- * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
- * contact@kingsrook.com
- * https://github.com/Kingsrook/
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program.  If not, see <https://www.gnu.org/licenses/>.
- */
-
-package com.kingsrook.qqq.backend.core.actions.reporting.excel.poi;
-
-
-import org.apache.poi.ss.usermodel.BorderStyle;
-import org.apache.poi.ss.usermodel.CreationHelper;
-import org.apache.poi.ss.usermodel.Font;
-import org.apache.poi.ss.usermodel.HorizontalAlignment;
-import org.apache.poi.xssf.usermodel.XSSFCellStyle;
-import org.apache.poi.xssf.usermodel.XSSFWorkbook;
-
-
-/*******************************************************************************
- ** Version of POI excel styler that does bold headers and footers, with basic borders.
- *******************************************************************************/
-public class BoldHeaderAndFooterPoiExcelStyler implements PoiExcelStylerInterface
-{
-
-   /*******************************************************************************
-    **
-    *******************************************************************************/
-   @Override
-   public XSSFCellStyle createStyleForTitle(XSSFWorkbook workbook, CreationHelper createHelper)
-   {
-      Font font = workbook.createFont();
-      font.setFontHeightInPoints((short) 14);
-      font.setBold(true);
-
-      XSSFCellStyle cellStyle = workbook.createCellStyle();
-      cellStyle.setFont(font);
-      cellStyle.setAlignment(HorizontalAlignment.CENTER);
-
-      return (cellStyle);
-   }
-
-
-
-   /*******************************************************************************
-    **
-    *******************************************************************************/
-   @Override
-   public XSSFCellStyle createStyleForHeader(XSSFWorkbook workbook, CreationHelper createHelper)
-   {
-      Font font = workbook.createFont();
-      font.setBold(true);
-
-      XSSFCellStyle cellStyle = workbook.createCellStyle();
-      cellStyle.setFont(font);
-      cellStyle.setBorderBottom(BorderStyle.THIN);
-
-      return (cellStyle);
-   }
-
-
-
-   /*******************************************************************************
-    **
-    *******************************************************************************/
-   @Override
-   public XSSFCellStyle createStyleForFooter(XSSFWorkbook workbook, CreationHelper createHelper)
-   {
-      Font font = workbook.createFont();
-      font.setBold(true);
-
-      XSSFCellStyle cellStyle = workbook.createCellStyle();
-      cellStyle.setFont(font);
-      cellStyle.setBorderTop(BorderStyle.THIN);
-      cellStyle.setBorderBottom(BorderStyle.DOUBLE);
-
-      return (cellStyle);
-   }
-
-}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/excel/poi/ExcelPoiBasedStreamingExportStreamer.java

@@ -1,819 +0,0 @@
-/*
- * QQQ - Low-code Application Framework for Engineers.
- * Copyright (C) 2021-2024.  Kingsrook, LLC
- * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
- * contact@kingsrook.com
- * https://github.com/Kingsrook/
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program.  If not, see <https://www.gnu.org/licenses/>.
- */
-
-package com.kingsrook.qqq.backend.core.actions.reporting.excel.poi;
-
-
-import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
-import java.io.IOException;
-import java.io.OutputStream;
-import java.io.OutputStreamWriter;
-import java.io.Serializable;
-import java.io.Writer;
-import java.time.Instant;
-import java.time.LocalDate;
-import java.time.LocalDateTime;
-import java.time.ZoneId;
-import java.time.ZonedDateTime;
-import java.util.ArrayList;
-import java.util.Date;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.Objects;
-import java.util.Optional;
-import java.util.zip.ZipEntry;
-import java.util.zip.ZipInputStream;
-import java.util.zip.ZipOutputStream;
-import com.kingsrook.qqq.backend.core.actions.reporting.ExportStreamerInterface;
-import com.kingsrook.qqq.backend.core.actions.reporting.ReportUtils;
-import com.kingsrook.qqq.backend.core.exceptions.QReportingException;
-import com.kingsrook.qqq.backend.core.instances.QInstanceEnricher;
-import com.kingsrook.qqq.backend.core.logging.QLogger;
-import com.kingsrook.qqq.backend.core.model.actions.reporting.ExportInput;
-import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportDestination;
-import com.kingsrook.qqq.backend.core.model.actions.reporting.pivottable.PivotTableGroupBy;
-import com.kingsrook.qqq.backend.core.model.actions.reporting.pivottable.PivotTableValue;
-import com.kingsrook.qqq.backend.core.model.data.QRecord;
-import com.kingsrook.qqq.backend.core.model.metadata.fields.DisplayFormat;
-import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
-import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
-import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportField;
-import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportView;
-import com.kingsrook.qqq.backend.core.model.metadata.reporting.ReportType;
-import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
-import com.kingsrook.qqq.backend.core.utils.StringUtils;
-import com.kingsrook.qqq.backend.core.utils.ValueUtils;
-import org.apache.poi.ss.SpreadsheetVersion;
-import org.apache.poi.ss.usermodel.CreationHelper;
-import org.apache.poi.ss.usermodel.DataConsolidateFunction;
-import org.apache.poi.ss.usermodel.DateUtil;
-import org.apache.poi.ss.util.AreaReference;
-import org.apache.poi.ss.util.CellReference;
-import org.apache.poi.ss.util.WorkbookUtil;
-import org.apache.poi.xssf.usermodel.XSSFCell;
-import org.apache.poi.xssf.usermodel.XSSFCellStyle;
-import org.apache.poi.xssf.usermodel.XSSFPivotTable;
-import org.apache.poi.xssf.usermodel.XSSFRow;
-import org.apache.poi.xssf.usermodel.XSSFSheet;
-import org.apache.poi.xssf.usermodel.XSSFWorkbook;
-
-
-/*******************************************************************************
- ** Excel export format implementation using POI library, but with modifications
- ** to actually stream output rather than use any temp files.
- **
- ** For a rough outline:
- ** - create a basically empty Excel workbook using POI - empty meaning, without
- ** data rows.
- ** - have POI write that workbook out into a byte[] - which will be a zip full
- ** of xml (e.g., xlsx).
- ** - then open a new ZipOutputStream wrapper around the OutputStream we took in
- ** as the report destination (e.g., streamed into storage or http output)
- ** - Copy over all entries from the xlsx into our new zip-output-stream, other than
- ** ones that are the actual sheets that we want to put data into.
- ** - For the sheet entries, use the StreamedPoiSheetWriter class to write the
- ** report's data directly out as Excel XML (not using POI).
- ** - Pivot tables require a bit of an additional hack - to write a "pivot cache
- ** definition", which, while we won't put all the data in it (because we'll tell
- ** it refreshOnLoad="true"), we will at least need to know how many cols & rows
- ** are in the data-sheet (which we wouldn't know until we streamed that sheet!)
- *******************************************************************************/
-public class ExcelPoiBasedStreamingExportStreamer implements ExportStreamerInterface
-{
-   private static final QLogger LOG = QLogger.getLogger(ExcelPoiBasedStreamingExportStreamer.class);
-
-   private List<QReportView>    views;
-   private ExportInput          exportInput;
-   private List<QFieldMetaData> fields;
-   private OutputStream         outputStream;
-   private ZipOutputStream      zipOutputStream;
-
-   public static final String EXCEL_DATE_FORMAT      = "yyyy-MM-dd";
-   public static final String EXCEL_DATE_TIME_FORMAT = "yyyy-MM-dd H:mm:ss";
-
-   private PoiExcelStylerInterface    poiExcelStylerInterface = getStylerInterface();
-   private Map<String, String>        excelCellFormats;
-   private Map<String, XSSFCellStyle> styles                  = new HashMap<>();
-
-   private int rowNo      = 0;
-   private int sheetIndex = 1;
-
-   private Map<String, String> pivotViewToCacheDefinitionReferenceMap = new HashMap<>();
-
-   private Writer              activeSheetWriter = null;
-   private StreamedSheetWriter sheetWriter       = null;
-
-   private QReportView                       currentView      = null;
-   private Map<String, List<QFieldMetaData>> fieldsPerView    = new HashMap<>();
-   private Map<String, Integer>              rowsPerView      = new HashMap<>();
-   private Map<String, String>               labelViewsByName = new HashMap<>();
-
-
-
-   /*******************************************************************************
-    **
-    *******************************************************************************/
-   public ExcelPoiBasedStreamingExportStreamer()
-   {
-   }
-
-
-
-   /*******************************************************************************
-    **
-    *******************************************************************************/
-   @Override
-   public void preRun(ReportDestination reportDestination, List<QReportView> views) throws QReportingException
-   {
-      try
-      {
-         this.outputStream = reportDestination.getReportOutputStream();
-         this.views = views;
-
-         ///////////////////////////////////////////////////////////////////////////////
-         // create 'template' workbook through poi - with sheets corresponding to our //
-         // actual file this will be a zip file (stream), with entries for all of the //
-         // files in the final xlsx but without any data, so it'll be small           //
-         ///////////////////////////////////////////////////////////////////////////////
-         XSSFWorkbook workbook = new XSSFWorkbook();
-         createStyles(workbook);
-
-         //////////////////////////////////////////////////////////////////////////////////////////////////
-         // for each of the sheets, create it in the workbook, and put a reference to it in the sheetMap //
-         //////////////////////////////////////////////////////////////////////////////////////////////////
-         Map<String, XSSFSheet> sheetMapByExcelReference = new HashMap<>();
-         Map<String, XSSFSheet> sheetMapByViewName       = new HashMap<>();
-
-         int sheetCounter = 1;
-         for(QReportView view : views)
-         {
-            String label = Objects.requireNonNullElse(view.getLabel(), "Sheet " + sheetCounter);
-            label = WorkbookUtil.createSafeSheetName(label);
-
-            /////////////////////////////////////////////////////////////////////////////////////////////
-            // track the actually-used sheet labels (needed for referencing in pivot table generation) //
-            /////////////////////////////////////////////////////////////////////////////////////////////
-            labelViewsByName.put(view.getName(), label);
-
-            XSSFSheet sheet          = workbook.createSheet(label);
-            String    sheetReference = sheet.getPackagePart().getPartName().getName().substring(1);
-            sheetMapByExcelReference.put(sheetReference, sheet);
-            sheetMapByViewName.put(view.getName(), sheet);
-            sheetCounter++;
-         }
-
-         ////////////////////////////////////////////////////
-         // if any views are pivot tables, create them now //
-         ////////////////////////////////////////////////////
-         List<String> pivotViewNames = new ArrayList<>();
-         for(QReportView view : views)
-         {
-            if(ReportType.PIVOT.equals(view.getType()))
-            {
-               pivotViewNames.add(view.getName());
-
-               XSSFSheet   pivotTableSheet = Objects.requireNonNull(sheetMapByViewName.get(view.getName()), "Could not get pivot table sheet view by name: " + view.getName());
-               XSSFSheet   dataSheet       = Objects.requireNonNull(sheetMapByViewName.get(view.getPivotTableSourceViewName()), "Could not get pivot table source sheet by view name: " + view.getPivotTableSourceViewName());
-               QReportView dataView        = ReportUtils.getSourceViewForPivotTableView(views, view);
-               createPivotTableTemplate(pivotTableSheet, view, dataSheet, dataView);
-            }
-         }
-         Iterator<String> pivotViewNameIterator = pivotViewNames.iterator();
-
-         /////////////////////////////////////////////////////////
-         // write that template worksheet zip out to byte array //
-         /////////////////////////////////////////////////////////
-         ByteArrayOutputStream templateBAOS = new ByteArrayOutputStream();
-         workbook.write(templateBAOS);
-         templateBAOS.close();
-         byte[] templateBytes = templateBAOS.toByteArray();
-
-         /////////////////////////////////////////////////////////////////////////////////////////////
-         // open up a zipOutputStream around the output stream that the report is to be written to. //
-         // note:  this stream is closed in the finish method - see more comments there.            //
-         /////////////////////////////////////////////////////////////////////////////////////////////
-         this.zipOutputStream = new ZipOutputStream(this.outputStream);
-
-         /////////////////////////////////////////////////////////////////////////////////////////////////
-         // copy over all the entries in the template zip that aren't the sheets into the output stream //
-         /////////////////////////////////////////////////////////////////////////////////////////////////
-         ZipInputStream zipInputStream   = new ZipInputStream(new ByteArrayInputStream(templateBytes));
-         ZipEntry       zipTemplateEntry = null;
-         byte[]         buffer           = new byte[2048];
-         while((zipTemplateEntry = zipInputStream.getNextEntry()) != null)
-         {
-            if(zipTemplateEntry.getName().matches(".*/pivotCacheDefinition.*.xml"))
-            {
-               //////////////////////////////////////////////////////////////////////////////////////////////////////
-               // if this zip entry is a pivotCacheDefinition, then don't write it to the output stream right now. //
-               // instead, just map the pivot view's name to the zipTemplateEntry name                             //
-               //////////////////////////////////////////////////////////////////////////////////////////////////////
-               if(!pivotViewNameIterator.hasNext())
-               {
-                  throw new QReportingException("Found a pivot cache definition [" + zipTemplateEntry.getName() + "] in the template ZIP, but no (more) corresponding pivot view names");
-               }
-
-               String pivotViewName = pivotViewNameIterator.next();
-               LOG.info("Holding on a pivot cache definition zip template entry [" + pivotViewName + "] [" + zipTemplateEntry.getName() + "]...");
-               pivotViewToCacheDefinitionReferenceMap.put(pivotViewName, zipTemplateEntry.getName());
-            }
-            else if(!sheetMapByExcelReference.containsKey(zipTemplateEntry.getName()))
-            {
-               ////////////////////////////////////////////////////////////////////////////////////////////////////////////
-               // else, if we don't have this zipTemplateEntry name in our map of sheets, then this is a kinda "meta"    //
-               // file that we don't really care about (e.g., not our sheet data), so just copy it to the output stream. //
-               ////////////////////////////////////////////////////////////////////////////////////////////////////////////
-               LOG.info("Copying zip template entry [" + zipTemplateEntry.getName() + "] to output stream");
-               zipOutputStream.putNextEntry(new ZipEntry(zipTemplateEntry.getName()));
-
-               int length;
-               while((length = zipInputStream.read(buffer)) > 0)
-               {
-                  zipOutputStream.write(buffer, 0, length);
-               }
-
-               zipInputStream.closeEntry();
-            }
-            else
-            {
-               ////////////////////////////////////////////////////////////////////////////////////
-               // else - this is a sheet - so again, don't write it yet - stream its data below. //
-               ////////////////////////////////////////////////////////////////////////////////////
-               LOG.info("Skipping presumed sheet zip template entry [" + zipTemplateEntry.getName() + "] to output stream");
-            }
-         }
-
-         zipInputStream.close();
-      }
-      catch(Exception e)
-      {
-         throw (new QReportingException("Error preparing to generate spreadsheet", e));
-      }
-   }
-
-
-
-   /*******************************************************************************
-    **
-    *******************************************************************************/
-   private void createPivotTableTemplate(XSSFSheet pivotTableSheet, QReportView pivotView, XSSFSheet dataSheet, QReportView dataView) throws QReportingException
-   {
-      /////////////////////////////////////////////////////////////////////////////////////////////////////////////
-      // write just enough data to the dataView's sheet so that we can refer to it for creating the pivot table. //
-      // we need to do this, because POI will try to create the pivotCache referring to the data sheet, and if   //
-      // there isn't any data there, it'll crash.                                                                //
-      /////////////////////////////////////////////////////////////////////////////////////////////////////////////
-      XSSFRow headerRow = dataSheet.createRow(0);
-      int     columnNo  = 0;
-      for(QReportField column : dataView.getColumns())
-      {
-         XSSFCell cell = headerRow.createCell(columnNo++);
-         cell.setCellValue(QInstanceEnricher.nameToLabel(column.getName()));
-      }
-
-      XSSFRow valuesRow = dataSheet.createRow(1);
-      columnNo = 0;
-      for(QReportField column : dataView.getColumns())
-      {
-         XSSFCell cell = valuesRow.createCell(columnNo++);
-         cell.setCellValue("Value " + columnNo);
-      }
-
-      /////////////////////////////////////////////////////////////////////////////////////////////////////
-      // for this template version of the pivot table, tell it there are only 2 rows in the source sheet //
-      // as that's all that we wrote above (a header and 1 fake value row)                               //
-      /////////////////////////////////////////////////////////////////////////////////////////////////////
-      int           rows       = 2;
-      String        colsLetter = CellReference.convertNumToColString(dataView.getColumns().size() - 1);
-      AreaReference source     = new AreaReference("A1:" + colsLetter + rows, SpreadsheetVersion.EXCEL2007);
-      CellReference position   = new CellReference("A1");
-
-      //////////////////////////////////////////////////////////////////
-      // tell poi all about our pivot table - rows, cols, and columns //
-      //////////////////////////////////////////////////////////////////
-      XSSFPivotTable pivotTable = pivotTableSheet.createPivotTable(source, position, dataSheet);
-
-      for(PivotTableGroupBy row : CollectionUtils.nonNullList(pivotView.getPivotTableDefinition().getRows()))
-      {
-         int rowLabelColumnIndex = getColumnIndex(dataView.getColumns(), row.getFieldName());
-         pivotTable.addRowLabel(rowLabelColumnIndex);
-      }
-
-      for(PivotTableGroupBy column : CollectionUtils.nonNullList(pivotView.getPivotTableDefinition().getColumns()))
-      {
-         int colLabelColumnIndex = getColumnIndex(dataView.getColumns(), column.getFieldName());
-         pivotTable.addColLabel(colLabelColumnIndex);
-      }
-
-      for(PivotTableValue value : CollectionUtils.nonNullList(pivotView.getPivotTableDefinition().getValues()))
-      {
-         int columnLabelColumnIndex = getColumnIndex(dataView.getColumns(), value.getFieldName());
-
-         String labelPrefix = value.getFunction().name() + " of ";
-         String label       = labelPrefix + QInstanceEnricher.nameToLabel(value.getFieldName());
-         String valueFormat = null;
-
-         Optional<QReportField> optSourceField = dataView.getColumns().stream().filter(c -> c.getName().equals(value.getFieldName())).findFirst();
-         if(optSourceField.isPresent())
-         {
-            QReportField sourceField = optSourceField.get();
-
-            if(StringUtils.hasContent(sourceField.getLabel()))
-            {
-               label = labelPrefix + sourceField.getLabel();
-            }
-
-            if(StringUtils.hasContent(sourceField.getDisplayFormat()))
-            {
-               valueFormat = DisplayFormat.getExcelFormat(sourceField.getDisplayFormat());
-            }
-            else
-            {
-               if(QFieldType.DATE.equals(sourceField.getType()))
-               {
-                  valueFormat = EXCEL_DATE_FORMAT;
-               }
-               else if(QFieldType.DATE_TIME.equals(sourceField.getType()))
-               {
-                  valueFormat = EXCEL_DATE_TIME_FORMAT;
-               }
-            }
-         }
-
-         pivotTable.addColumnLabel(DataConsolidateFunction.valueOf(value.getFunction().name()), columnLabelColumnIndex, label, valueFormat);
-      }
-   }
-
-
-
-   /*******************************************************************************
-    **
-    *******************************************************************************/
-   private int getColumnIndex(List<QReportField> columns, String fieldName) throws QReportingException
-   {
-      for(int i = 0; i < columns.size(); i++)
-      {
-         if(columns.get(i).getName().equals(fieldName))
-         {
-            return (i);
-         }
-      }
-
-      throw (new QReportingException("Could not find column by name [" + fieldName + "]"));
-   }
-
-
-
-   /*******************************************************************************
-    **
-    *******************************************************************************/
-   private void createStyles(XSSFWorkbook workbook)
-   {
-      CreationHelper createHelper = workbook.getCreationHelper();
-
-      XSSFCellStyle dateStyle = workbook.createCellStyle();
-      dateStyle.setDataFormat(createHelper.createDataFormat().getFormat(EXCEL_DATE_FORMAT));
-      styles.put("date", dateStyle);
-
-      XSSFCellStyle dateTimeStyle = workbook.createCellStyle();
-      dateTimeStyle.setDataFormat(createHelper.createDataFormat().getFormat(EXCEL_DATE_TIME_FORMAT));
-      styles.put("datetime", dateTimeStyle);
-
-      styles.put("title", poiExcelStylerInterface.createStyleForTitle(workbook, createHelper));
-      styles.put("header", poiExcelStylerInterface.createStyleForHeader(workbook, createHelper));
-      styles.put("footer", poiExcelStylerInterface.createStyleForFooter(workbook, createHelper));
-
-      XSSFCellStyle footerDateStyle = poiExcelStylerInterface.createStyleForFooter(workbook, createHelper);
-      footerDateStyle.setDataFormat(createHelper.createDataFormat().getFormat(EXCEL_DATE_FORMAT));
-      styles.put("footer-date", footerDateStyle);
-
-      XSSFCellStyle footerDateTimeStyle = poiExcelStylerInterface.createStyleForFooter(workbook, createHelper);
-      footerDateTimeStyle.setDataFormat(createHelper.createDataFormat().getFormat(EXCEL_DATE_TIME_FORMAT));
-      styles.put("footer-datetime", footerDateTimeStyle);
-   }
-
-
-
-   /*******************************************************************************
-    ** Starts a new worksheet in the current workbook.  Can be called multiple times.
-    *******************************************************************************/
-   @Override
-   public void start(ExportInput exportInput, List<QFieldMetaData> fields, String label, QReportView view) throws QReportingException
-   {
-      try
-      {
-         /////////////////////////////////////////
-         // close previous sheet if one is open //
-         /////////////////////////////////////////
-         closeLastSheetIfOpen();
-
-         if(currentView != null)
-         {
-            this.rowsPerView.put(currentView.getName(), rowNo);
-         }
-
-         this.currentView = view;
-         this.exportInput = exportInput;
-         this.fields = fields;
-         this.rowNo = 0;
-
-         this.fieldsPerView.put(view.getName(), fields);
-
-         //////////////////////////////////////////
-         // start the new sheet as:              //
-         // - a new entry in the zipOutputStream //
-         // - with a new output stream writer    //
-         // - and with a SpreadsheetWriter       //
-         //////////////////////////////////////////
-         zipOutputStream.putNextEntry(new ZipEntry("xl/worksheets/sheet" + this.sheetIndex++ + ".xml"));
-         activeSheetWriter = new OutputStreamWriter(zipOutputStream);
-         sheetWriter = new StreamedSheetWriter(activeSheetWriter);
-
-         if(ReportType.PIVOT.equals(view.getType()))
-         {
-            writePivotTable(view, ReportUtils.getSourceViewForPivotTableView(views, view));
-         }
-         else
-         {
-            sheetWriter.beginSheet();
-
-            ////////////////////////////////////////////////
-            // put the title and header rows in the sheet //
-            ////////////////////////////////////////////////
-            writeTitleAndHeader();
-         }
-      }
-      catch(Exception e)
-      {
-         throw (new QReportingException("Error starting worksheet", e));
-      }
-   }
-
-
-
-   /*******************************************************************************
-    **
-    *******************************************************************************/
-   private void writeTitleAndHeader() throws QReportingException
-   {
-      try
-      {
-         ///////////////
-         // title row //
-         ///////////////
-         if(StringUtils.hasContent(exportInput.getTitleRow()))
-         {
-            sheetWriter.insertRow(rowNo++);
-            sheetWriter.createCell(0, exportInput.getTitleRow(), styles.get("title").getIndex());
-            sheetWriter.endRow();
-         }
-
-         ////////////////
-         // header row //
-         ////////////////
-         if(exportInput.getIncludeHeaderRow())
-         {
-            sheetWriter.insertRow(rowNo++);
-            XSSFCellStyle headerStyle = styles.get("header");
-
-            int col = 0;
-            for(QFieldMetaData column : fields)
-            {
-               sheetWriter.createCell(col, column.getLabel(), headerStyle.getIndex());
-               col++;
-            }
-
-            sheetWriter.endRow();
-         }
-      }
-      catch(Exception e)
-      {
-         throw (new QReportingException("Error starting Excel report"));
-      }
-   }
-
-
-
-   /*******************************************************************************
-    **
-    *******************************************************************************/
-   @Override
-   public void addRecords(List<QRecord> qRecords) throws QReportingException
-   {
-      LOG.info("Consuming [" + qRecords.size() + "] records from the pipe");
-
-      try
-      {
-         for(QRecord qRecord : qRecords)
-         {
-            writeRecord(qRecord);
-         }
-      }
-      catch(Exception e)
-      {
-         LOG.error("Exception generating excel file", e);
-         try
-         {
-            outputStream.close();
-         }
-         catch(IOException ex)
-         {
-            LOG.warn("Secondary error closing excel output stream", e);
-         }
-
-         throw (new QReportingException("Error generating Excel report", e));
-      }
-   }
-
-
-
-   /*******************************************************************************
-    **
-    *******************************************************************************/
-   private void writeRecord(QRecord qRecord) throws IOException
-   {
-      writeRecord(qRecord, false);
-   }
-
-
-
-   /*******************************************************************************
-    **
-    *******************************************************************************/
-   private void writeRecord(QRecord qRecord, boolean isFooter) throws IOException
-   {
-      sheetWriter.insertRow(rowNo++);
-
-      int styleIndex         = -1;
-      int dateStyleIndex     = styles.get("date").getIndex();
-      int dateTimeStyleIndex = styles.get("datetime").getIndex();
-      if(isFooter)
-      {
-         styleIndex = styles.get("footer").getIndex();
-         dateStyleIndex = styles.get("footer-date").getIndex();
-         dateTimeStyleIndex = styles.get("footer-datetime").getIndex();
-      }
-
-      int col = 0;
-      for(QFieldMetaData field : fields)
-      {
-         Serializable value = qRecord.getValue(field.getName());
-
-         if(value != null)
-         {
-            if(value instanceof String s)
-            {
-               sheetWriter.createCell(col, s, styleIndex);
-            }
-            else if(value instanceof Number n)
-            {
-               sheetWriter.createCell(col, n.doubleValue(), styleIndex);
-
-               if(excelCellFormats != null)
-               {
-                  String format = excelCellFormats.get(field.getName());
-                  if(format != null)
-                  {
-                     ////////////////////////////////////////////////////////////////////////////////////////////
-                     // todo - so - for this streamed/zip approach, we need to know all styles before we start //
-                     // any sheets.  but, right now Report action only calls us with per-sheet styles when     //
-                     // it's starting individual sheets.  so, we can't quite support this at this time.        //
-                     // "just" need to change GenerateReportAction to look up all cell formats for all sheets  //
-                     // before preRun is called... and change all existing streamer classes to handle that too //
-                     ////////////////////////////////////////////////////////////////////////////////////////////
-                     // worksheet.style(rowNo, col).format(format).set();
-                  }
-               }
-            }
-            else if(value instanceof Boolean b)
-            {
-               sheetWriter.createCell(col, b, styleIndex);
-            }
-            else if(value instanceof Date d)
-            {
-               sheetWriter.createCell(col, DateUtil.getExcelDate(d), dateStyleIndex);
-            }
-            else if(value instanceof LocalDate d)
-            {
-               sheetWriter.createCell(col, DateUtil.getExcelDate(d), dateStyleIndex);
-            }
-            else if(value instanceof LocalDateTime d)
-            {
-               sheetWriter.createCell(col, DateUtil.getExcelDate(d), dateStyleIndex);
-            }
-            else if(value instanceof ZonedDateTime d)
-            {
-               sheetWriter.createCell(col, DateUtil.getExcelDate(d.toLocalDateTime()), dateTimeStyleIndex);
-            }
-            else if(value instanceof Instant i)
-            {
-               sheetWriter.createCell(col, DateUtil.getExcelDate(i.atZone(ZoneId.systemDefault()).toLocalDateTime()), dateTimeStyleIndex);
-            }
-            else
-            {
-               sheetWriter.createCell(col, ValueUtils.getValueAsString(value), styleIndex);
-            }
-         }
-
-         col++;
-      }
-
-      sheetWriter.endRow();
-   }
-
-
-
-   /*******************************************************************************
-    **
-    *******************************************************************************/
-   public void addTotalsRow(QRecord record) throws QReportingException
-   {
-      try
-      {
-         writeRecord(record, true);
-      }
-      catch(Exception e)
-      {
-         throw (new QReportingException("Error adding totals row", e));
-      }
-   }
-
-
-
-   /*******************************************************************************
-    **
-    *******************************************************************************/
-   @Override
-   public void finish() throws QReportingException
-   {
-      try
-      {
-         //////////////////////////////////////////////
-         // close the last open sheet if one is open //
-         //////////////////////////////////////////////
-         closeLastSheetIfOpen();
-
-         /////////////////////////////////////////////////////////////////////////////////////
-         // so, we DO need a close here, on the zipOutputStream, to finish its "zippiness". //
-         // even though, doing so also closes the outputStream from the caller that this    //
-         // zipOutputStream is wrapping (and the caller will likely call close on too)...   //
-         /////////////////////////////////////////////////////////////////////////////////////
-         zipOutputStream.close();
-      }
-      catch(Exception e)
-      {
-         throw (new QReportingException("Error finishing Excel report", e));
-      }
-   }
-
-
-
-   /*******************************************************************************
-    **
-    *******************************************************************************/
-   private void closeLastSheetIfOpen() throws IOException
-   {
-      //////////////////////////////////////////////////////////////////////////////////////////////////////
-      // if we have an active sheet writer:                                                               //
-      // - end the current sheet in the spreadsheet writer (write some closing xml, unless it's a pivot!) //
-      // - flush the contents through the activeSheetWriter                                               //
-      // - close the zip entry in the output stream.                                                      //
-      //////////////////////////////////////////////////////////////////////////////////////////////////////
-      if(activeSheetWriter != null)
-      {
-         if(!ReportType.PIVOT.equals(currentView.getType()))
-         {
-            sheetWriter.endSheet();
-         }
-
-         activeSheetWriter.flush();
-         zipOutputStream.closeEntry();
-      }
-   }
-
-
-
-   /*******************************************************************************
-    ** display formats is a map of field name to Excel format strings (e.g., $#,##0.00)
-    *******************************************************************************/
-   @Override
-   public void setDisplayFormats(Map<String, String> displayFormats)
-   {
-      this.excelCellFormats = new HashMap<>();
-      for(Map.Entry<String, String> entry : displayFormats.entrySet())
-      {
-         String excelFormat = DisplayFormat.getExcelFormat(entry.getValue());
-         if(excelFormat != null)
-         {
-            excelCellFormats.put(entry.getKey(), excelFormat);
-         }
-      }
-   }
-
-
-
-   /*******************************************************************************
-    **
-    *******************************************************************************/
-   private void writePivotTable(QReportView pivotTableView, QReportView dataView) throws QReportingException
-   {
-      try
-      {
-         //////////////////////////////////////////////////////////////////////////////////
-         // write the xml file that is the pivot table sheet.                            //
-         // note that the ZipEntry here will have been started above in the start method //
-         //////////////////////////////////////////////////////////////////////////////////
-         activeSheetWriter.write("""
-            <?xml version="1.0" encoding="UTF-8" standalone="yes"?>
-              <worksheet xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main" xmlns:r="http://schemas.openxmlformats.org/officeDocument/2006/relationships" xmlns:mx="http://schemas.microsoft.com/office/mac/excel/2008/main" xmlns:mc="http://schemas.openxmlformats.org/markup-compatibility/2006" xmlns:mv="urn:schemas-microsoft-com:mac:vml" xmlns:x14="http://schemas.microsoft.com/office/spreadsheetml/2009/9/main" xmlns:x15="http://schemas.microsoft.com/office/spreadsheetml/2010/11/main" xmlns:x14ac="http://schemas.microsoft.com/office/spreadsheetml/2009/9/ac" xmlns:xm="http://schemas.microsoft.com/office/excel/2006/main">
-                 <sheetPr>
-                   <outlinePr summaryBelow="0" summaryRight="0"/>
-                 </sheetPr>
-                 <sheetViews>
-                   <sheetView workbookViewId="0"/>
-                 </sheetViews>
-                 <sheetFormatPr customHeight="1" defaultColWidth="14.43" defaultRowHeight="15.0"/>
-                 <sheetData>
-                   <row r="1"/>
-                 </sheetData>
-              </worksheet>
-            """);
-         activeSheetWriter.flush();
-
-         ////////////////////////////////////////////////////////////////////////////
-         // start a new zip entry, for this pivot view's cacheDefinition reference //
-         ////////////////////////////////////////////////////////////////////////////
-         zipOutputStream.putNextEntry(new ZipEntry(pivotViewToCacheDefinitionReferenceMap.get(pivotTableView.getName())));
-
-         /////////////////////////////////////////////////////////
-         // prepare the xml for each field (e.g., w/ its label) //
-         /////////////////////////////////////////////////////////
-         List<String> cachedFieldElements = new ArrayList<>();
-         for(QFieldMetaData column : this.fieldsPerView.get(dataView.getName()))
-         {
-            cachedFieldElements.add(String.format("""
-               <cacheField numFmtId="0" name="%s">
-                  <sharedItems/>
-               </cacheField>
-               """, column.getLabel()));
-         }
-
-         /////////////////////////////////////////////////////////////////////////////////////
-         // write the xml file that is the pivot cache definition (structure only, no data) //
-         /////////////////////////////////////////////////////////////////////////////////////
-         activeSheetWriter = new OutputStreamWriter(zipOutputStream);
-         activeSheetWriter.write(String.format("""
-               <?xml version="1.0" encoding="UTF-8"?>
-               <pivotCacheDefinition xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main" xmlns:r="http://schemas.openxmlformats.org/officeDocument/2006/relationships" createdVersion="3" minRefreshableVersion="3" refreshedVersion="3" refreshedBy="Apache POI" refreshedDate="1.711395767702E12" refreshOnLoad="true" r:id="rId1">
-                 <cacheSource type="worksheet">
-                   <worksheetSource sheet="%s" ref="A1:%s%d"/>
-                 </cacheSource>
-                 <cacheFields count="%d">
-                   %s
-                 </cacheFields>
-               </pivotCacheDefinition>
-               """,
-            StreamedSheetWriter.cleanseValue(labelViewsByName.get(dataView.getName())),
-            CellReference.convertNumToColString(dataView.getColumns().size() - 1),
-            rowsPerView.get(dataView.getName()),
-            dataView.getColumns().size(),
-            StringUtils.join("\n", cachedFieldElements)));
-      }
-      catch(Exception e)
-      {
-         throw (new QReportingException("Error writing pivot table", e));
-      }
-   }
-
-
-
-   /*******************************************************************************
-    **
-    *******************************************************************************/
-   protected PoiExcelStylerInterface getStylerInterface()
-   {
-      return (new PlainPoiExcelStyler());
-   }
-
-}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/excel/poi/PlainPoiExcelStyler.java

@@ -1,46 +0,0 @@
-/*
- * QQQ - Low-code Application Framework for Engineers.
- * Copyright (C) 2021-2024.  Kingsrook, LLC
- * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
- * contact@kingsrook.com
- * https://github.com/Kingsrook/
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program.  If not, see <https://www.gnu.org/licenses/>.
- */
-
-package com.kingsrook.qqq.backend.core.actions.reporting.excel.poi;
-
-
-import org.apache.poi.ss.usermodel.CreationHelper;
-import org.apache.poi.xssf.usermodel.XSSFCellStyle;
-import org.apache.poi.xssf.usermodel.XSSFWorkbook;
-
-
-/*******************************************************************************
- ** Excel styler that does nothing - just takes defaults (which are all no-op) from the interface.
- *******************************************************************************/
-public class PlainPoiExcelStyler implements PoiExcelStylerInterface
-{
-
-   /*******************************************************************************
-    ** ... sorry, but adding this gives us test coverage on this class, even though
-    ** we're just deferring to super...
-    *******************************************************************************/
-   @Override
-   public XSSFCellStyle createStyleForHeader(XSSFWorkbook workbook, CreationHelper createHelper)
-   {
-      return PoiExcelStylerInterface.super.createStyleForHeader(workbook, createHelper);
-   }
-
-}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/excel/poi/PoiExcelStylerInterface.java

@@ -1,59 +0,0 @@
-/*
- * QQQ - Low-code Application Framework for Engineers.
- * Copyright (C) 2021-2024.  Kingsrook, LLC
- * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
- * contact@kingsrook.com
- * https://github.com/Kingsrook/
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program.  If not, see <https://www.gnu.org/licenses/>.
- */
-
-package com.kingsrook.qqq.backend.core.actions.reporting.excel.poi;
-
-
-import org.apache.poi.ss.usermodel.CreationHelper;
-import org.apache.poi.xssf.usermodel.XSSFCellStyle;
-import org.apache.poi.xssf.usermodel.XSSFWorkbook;
-
-
-/*******************************************************************************
- ** Interface for classes that know how to apply styles to an Excel stream being
- ** built by POI.
- *******************************************************************************/
-public interface PoiExcelStylerInterface
-{
-   /*******************************************************************************
-    **
-    *******************************************************************************/
-   default XSSFCellStyle createStyleForTitle(XSSFWorkbook workbook, CreationHelper createHelper)
-   {
-      return (workbook.createCellStyle());
-   }
-
-   /*******************************************************************************
-    **
-    *******************************************************************************/
-   default XSSFCellStyle createStyleForHeader(XSSFWorkbook workbook, CreationHelper createHelper)
-   {
-      return (workbook.createCellStyle());
-   }
-
-   /*******************************************************************************
-    **
-    *******************************************************************************/
-   default XSSFCellStyle createStyleForFooter(XSSFWorkbook workbook, CreationHelper createHelper)
-   {
-      return (workbook.createCellStyle());
-   }
-}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/excel/poi/StreamedSheetWriter.java

@@ -1,242 +0,0 @@
-/*
- * QQQ - Low-code Application Framework for Engineers.
- * Copyright (C) 2021-2024.  Kingsrook, LLC
- * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
- * contact@kingsrook.com
- * https://github.com/Kingsrook/
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program.  If not, see <https://www.gnu.org/licenses/>.
- */
-
-package com.kingsrook.qqq.backend.core.actions.reporting.excel.poi;
-
-
-import java.io.IOException;
-import java.io.Writer;
-import java.util.HashMap;
-import java.util.Map;
-import org.apache.poi.ss.util.CellReference;
-
-
-/*******************************************************************************
- ** Write excel formatted XML to a Writer.
- ** Originally from https://coderanch.com/t/548897/java/Generate-large-excel-POI
- *******************************************************************************/
-public class StreamedSheetWriter
-{
-   private final Writer writer;
-   private       int    rowNo;
-
-
-
-   /*******************************************************************************
-    **
-    *******************************************************************************/
-   public StreamedSheetWriter(Writer writer)
-   {
-      this.writer = writer;
-   }
-
-
-
-   /*******************************************************************************
-    **
-    *******************************************************************************/
-   public void beginSheet() throws IOException
-   {
-      writer.write("""
-         <?xml version="1.0" encoding="UTF-8"?>
-            <worksheet xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main">
-               <sheetData>""");
-
-   }
-
-
-
-   /*******************************************************************************
-    **
-    *******************************************************************************/
-   public void endSheet() throws IOException
-   {
-      writer.write("""
-            </sheetData>
-         </worksheet>""");
-   }
-
-
-
-   /*******************************************************************************
-    ** Insert a new row
-    **
-    ** @param rowNo 0-based row number
-    *******************************************************************************/
-   public void insertRow(int rowNo) throws IOException
-   {
-      writer.write("<row r=\"" + (rowNo + 1) + "\">\n");
-      this.rowNo = rowNo;
-   }
-
-
-
-   /*******************************************************************************
-    ** Insert row end marker
-    *******************************************************************************/
-   public void endRow() throws IOException
-   {
-      writer.write("</row>\n");
-   }
-
-
-
-   /*******************************************************************************
-    **
-    *******************************************************************************/
-   public void createCell(int columnIndex, String value, int styleIndex) throws IOException
-   {
-      String ref = new CellReference(rowNo, columnIndex).formatAsString();
-      writer.write("<c r=\"" + ref + "\" t=\"inlineStr\"");
-      if(styleIndex != -1)
-      {
-         writer.write(" s=\"" + styleIndex + "\"");
-      }
-
-      String cleanValue = cleanseValue(value);
-
-      writer.write(">");
-      writer.write("<is><t>" + cleanValue + "</t></is>");
-      writer.write("</c>");
-   }
-
-
-
-   /*******************************************************************************
-    **
-    *******************************************************************************/
-   public static String cleanseValue(String value)
-   {
-      if(value != null)
-      {
-         StringBuilder rs = new StringBuilder();
-         for(int i = 0; i < value.length(); i++)
-         {
-            char c = value.charAt(i);
-            if(c == '&')
-            {
-               rs.append("&amp;");
-            }
-            else if(c == '<')
-            {
-               rs.append("&lt;");
-            }
-            else if(c == '>')
-            {
-               rs.append("&gt;");
-            }
-            else if(c == '\'')
-            {
-               rs.append("&apos;");
-            }
-            else if(c == '"')
-            {
-               rs.append("&quot;");
-            }
-            else if (c < 32 && c != '\t' && c != '\n')
-            {
-               rs.append(' ');
-            }
-            else
-            {
-               rs.append(c);
-            }
-         }
-
-         Map<String, Integer> m = new HashMap();
-         m.computeIfAbsent("s", (s) -> 3);
-
-         value = rs.toString();
-      }
-
-      return (value);
-   }
-
-
-
-   /*******************************************************************************
-    **
-    *******************************************************************************/
-   public void createCell(int columnIndex, String value) throws IOException
-   {
-      createCell(columnIndex, value, -1);
-   }
-
-
-
-   /*******************************************************************************
-    **
-    *******************************************************************************/
-   public void createCell(int columnIndex, double value, int styleIndex) throws IOException
-   {
-      String ref = new CellReference(rowNo, columnIndex).formatAsString();
-      writer.write("<c r=\"" + ref + "\" t=\"n\"");
-      if(styleIndex != -1)
-      {
-         writer.write(" s=\"" + styleIndex + "\"");
-      }
-      writer.write(">");
-      writer.write("<v>" + value + "</v>");
-      writer.write("</c>");
-   }
-
-
-
-   /*******************************************************************************
-    **
-    *******************************************************************************/
-   public void createCell(int columnIndex, double value) throws IOException
-   {
-      createCell(columnIndex, value, -1);
-   }
-
-
-
-   /*******************************************************************************
-    **
-    *******************************************************************************/
-   public void createCell(int columnIndex, Boolean value) throws IOException
-   {
-      createCell(columnIndex, value, -1);
-   }
-
-
-
-   /*******************************************************************************
-    **
-    *******************************************************************************/
-   public void createCell(int columnIndex, Boolean value, int styleIndex) throws IOException
-   {
-      String ref = new CellReference(rowNo, columnIndex).formatAsString();
-      writer.write("<c r=\"" + ref + "\" t=\"b\"");
-      if(styleIndex != -1)
-      {
-         writer.write(" s=\"" + styleIndex + "\"");
-      }
-      writer.write(">");
-      if(value != null)
-      {
-         writer.write("<v>" + (value ? 1 : 0) + "</v>");
-      }
-      writer.write("</c>");
-   }
-
-}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/excelformatting/BoldHeaderAndFooterExcelStyler.java

@@ -1,6 +1,6 @@
 /*
  * QQQ - Low-code Application Framework for Engineers.
- * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
  * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
  * contact@kingsrook.com
  * https://github.com/Kingsrook/
@@ -19,7 +19,7 @@
  * along with this program.  If not, see <https://www.gnu.org/licenses/>.
  */
 
-package com.kingsrook.qqq.backend.core.actions.reporting.excel.fastexcel;
+package com.kingsrook.qqq.backend.core.actions.reporting.excelformatting;
 
 
 import org.dhatim.fastexcel.BorderSide;
@@ -30,7 +30,7 @@ import org.dhatim.fastexcel.StyleSetter;
 /*******************************************************************************
  ** Version of excel styler that does bold headers and footers, with basic borders.
  *******************************************************************************/
-public class BoldHeaderAndFooterFastExcelStyler implements FastExcelStylerInterface
+public class BoldHeaderAndFooterExcelStyler implements ExcelStylerInterface
 {
 
    /*******************************************************************************
@@ -60,9 +60,6 @@ public class BoldHeaderAndFooterFastExcelStyler implements FastExcelStylerInterf
 
 
 
-   /*******************************************************************************
-    **
-    *******************************************************************************/
    @Override
    public void styleTotalsRow(StyleSetter totalsRowStyle)
    {

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/excelformatting/ExcelStylerInterface.java

@@ -1,6 +1,6 @@
 /*
  * QQQ - Low-code Application Framework for Engineers.
- * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
  * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
  * contact@kingsrook.com
  * https://github.com/Kingsrook/
@@ -19,7 +19,7 @@
  * along with this program.  If not, see <https://www.gnu.org/licenses/>.
  */
 
-package com.kingsrook.qqq.backend.core.actions.reporting.excel.fastexcel;
+package com.kingsrook.qqq.backend.core.actions.reporting.excelformatting;
 
 
 import org.dhatim.fastexcel.StyleSetter;
@@ -29,7 +29,7 @@ import org.dhatim.fastexcel.StyleSetter;
  ** Interface for classes that know how to apply styles to an Excel stream being
  ** built by fastexcel.
  *******************************************************************************/
-public interface FastExcelStylerInterface
+public interface ExcelStylerInterface
 {
 
    /*******************************************************************************

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/excelformatting/PlainExcelStyler.java

@@ -19,15 +19,13 @@
  * along with this program.  If not, see <https://www.gnu.org/licenses/>.
  */
 
-package com.kingsrook.qqq.backend.core.model.metadata.dashboard;
+package com.kingsrook.qqq.backend.core.actions.reporting.excelformatting;
 
 
 /*******************************************************************************
- ** Possible types for widget dropdowns
- **
+ ** Excel styler that does nothing - just takes defaults (which are all no-op) from the interface.
  *******************************************************************************/
-public enum WidgetDropdownType
+public class PlainExcelStyler implements ExcelStylerInterface
 {
-   POSSIBLE_VALUE_SOURCE,
-   DATE_PICKER
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/scripts/ExecuteCodeAction.java

@@ -77,6 +77,7 @@ public class ExecuteCodeAction
    /*******************************************************************************
     **
     *******************************************************************************/
+   @SuppressWarnings("checkstyle:indentation")
    public void run(ExecuteCodeInput input, ExecuteCodeOutput output) throws QException, QCodeException
    {
       QCodeReference codeReference = input.getCodeReference();

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/scripts/RunAdHocRecordScriptAction.java

@@ -30,7 +30,6 @@ import java.util.List;
 import java.util.Map;
 import java.util.Optional;
 import com.kingsrook.qqq.backend.core.actions.ActionHelper;
-import com.kingsrook.qqq.backend.core.actions.audits.DMLAuditAction;
 import com.kingsrook.qqq.backend.core.actions.tables.GetAction;
 import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
 import com.kingsrook.qqq.backend.core.context.QContext;
@@ -48,14 +47,12 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryJoin;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput;
-import com.kingsrook.qqq.backend.core.model.data.QRecord;
 import com.kingsrook.qqq.backend.core.model.metadata.code.AdHocScriptCodeReference;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 import com.kingsrook.qqq.backend.core.model.scripts.Script;
 import com.kingsrook.qqq.backend.core.model.scripts.ScriptRevision;
 import com.kingsrook.qqq.backend.core.model.scripts.ScriptsMetaDataProvider;
 import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
-import com.kingsrook.qqq.backend.core.utils.memoization.Memoization;
 
 
 /*******************************************************************************
@@ -68,8 +65,6 @@ public class RunAdHocRecordScriptAction
    private Map<Integer, ScriptRevision> scriptRevisionCacheByScriptRevisionId = new HashMap<>();
    private Map<Integer, ScriptRevision> scriptRevisionCacheByScriptId         = new HashMap<>();
 
-   private static Memoization<Integer, Script> scriptMemoizationById = new Memoization<>();
-
 
 
    /*******************************************************************************
@@ -90,12 +85,6 @@ public class RunAdHocRecordScriptAction
             throw (new QException("Script revision was not found."));
          }
 
-         Optional<Script> script = getScript(scriptRevision);
-
-         QContext.getQSession().setValue(DMLAuditAction.AUDIT_CONTEXT_FIELD_NAME, script.isPresent()
-            ? "via Script \"%s\"".formatted(script.get().getName())
-            : "via Script id " + scriptRevision.getScriptId());
-
          ////////////////////////////
          // figure out the records //
          ////////////////////////////
@@ -135,10 +124,6 @@ public class RunAdHocRecordScriptAction
       {
          output.setException(Optional.of(e));
       }
-      finally
-      {
-         QContext.getQSession().removeValue(DMLAuditAction.AUDIT_CONTEXT_FIELD_NAME);
-      }
    }
 
 
@@ -232,44 +217,4 @@ public class RunAdHocRecordScriptAction
       throw (new QException("Code reference did not contain a scriptRevision, scriptRevisionId, or scriptId"));
    }
 
-
-
-   /*******************************************************************************
-    **
-    *******************************************************************************/
-   private Optional<Script> getScript(ScriptRevision scriptRevision)
-   {
-      if(scriptRevision == null || scriptRevision.getScriptId() == null)
-      {
-         return (Optional.empty());
-      }
-
-      try
-      {
-         return scriptMemoizationById.getResult(scriptRevision.getScriptId(), scriptId ->
-         {
-            try
-            {
-               QRecord scriptRecord = new GetAction().executeForRecord(new GetInput(Script.TABLE_NAME).withPrimaryKey(scriptRevision.getScriptId()));
-               if(scriptRecord != null)
-               {
-                  Script script = new Script(scriptRecord);
-                  scriptMemoizationById.storeResult(scriptRevision.getScriptId(), script);
-                  return (script);
-               }
-            }
-            catch(Exception e)
-            {
-               LOG.info("");
-            }
-
-            return (null);
-         });
-      }
-      catch(Exception e)
-      {
-         return (Optional.empty());
-      }
-   }
-
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/DeleteAction.java

@@ -35,8 +35,9 @@ import java.util.Set;
 import java.util.stream.Collectors;
 import com.kingsrook.qqq.backend.core.actions.ActionHelper;
 import com.kingsrook.qqq.backend.core.actions.audits.DMLAuditAction;
+import com.kingsrook.qqq.backend.core.actions.customizers.AbstractPostDeleteCustomizer;
+import com.kingsrook.qqq.backend.core.actions.customizers.AbstractPreDeleteCustomizer;
 import com.kingsrook.qqq.backend.core.actions.customizers.QCodeLoader;
-import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizerInterface;
 import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizers;
 import com.kingsrook.qqq.backend.core.actions.interfaces.DeleteInterface;
 import com.kingsrook.qqq.backend.core.actions.tables.helpers.ValidateRecordSecurityLockHelper;
@@ -249,7 +250,7 @@ public class DeleteAction
       //////////////////////////////////////////////////////////////
       // finally, run the post-delete customizer, if there is one //
       //////////////////////////////////////////////////////////////
-      Optional<TableCustomizerInterface> postDeleteCustomizer = QCodeLoader.getTableCustomizer(table, TableCustomizers.POST_DELETE_RECORD.getRole());
+      Optional<AbstractPostDeleteCustomizer> postDeleteCustomizer = QCodeLoader.getTableCustomizer(AbstractPostDeleteCustomizer.class, table, TableCustomizers.POST_DELETE_RECORD.getRole());
       if(postDeleteCustomizer.isPresent() && oldRecordList.isPresent())
       {
          ////////////////////////////////////////////////////////////////////////////
@@ -259,7 +260,8 @@ public class DeleteAction
 
          try
          {
-            List<QRecord> postCustomizerResult = postDeleteCustomizer.get().postDelete(deleteInput, recordsForCustomizer);
+            postDeleteCustomizer.get().setDeleteInput(deleteInput);
+            List<QRecord> postCustomizerResult = postDeleteCustomizer.get().apply(recordsForCustomizer);
 
             ///////////////////////////////////////////////////////
             // check if any records got errors in the customizer //
@@ -325,11 +327,13 @@ public class DeleteAction
       ///////////////////////////////////////////////////////////////////////////
       // after all validations, run the pre-delete customizer, if there is one //
       ///////////////////////////////////////////////////////////////////////////
-      Optional<TableCustomizerInterface> preDeleteCustomizer = QCodeLoader.getTableCustomizer(table, TableCustomizers.PRE_DELETE_RECORD.getRole());
+      Optional<AbstractPreDeleteCustomizer> preDeleteCustomizer = QCodeLoader.getTableCustomizer(AbstractPreDeleteCustomizer.class, table, TableCustomizers.PRE_DELETE_RECORD.getRole());
       List<QRecord>                         customizerResult    = oldRecordList.get();
       if(preDeleteCustomizer.isPresent())
       {
-         customizerResult = preDeleteCustomizer.get().preDelete(deleteInput, oldRecordList.get(), isPreview);
+         preDeleteCustomizer.get().setDeleteInput(deleteInput);
+         preDeleteCustomizer.get().setIsPreview(isPreview);
+         customizerResult = preDeleteCustomizer.get().apply(oldRecordList.get());
       }
 
       /////////////////////////////////////////////////////////////////////////
@@ -470,7 +474,7 @@ public class DeleteAction
                QRecord recordWithError = new QRecord();
                recordsWithErrors.add(recordWithError);
                recordWithError.setValue(primaryKeyField.getName(), primaryKeyValue);
-               recordWithError.addError(new NotFoundStatusMessage("No record was found to delete for " + Objects.requireNonNullElse(primaryKeyField.getLabel(), primaryKeyField.getName()) + " = " + primaryKeyValue));
+               recordWithError.addError(new NotFoundStatusMessage("No record was found to delete for " + primaryKeyField.getLabel() + " = " + primaryKeyValue));
             }
          }
       }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/GetAction.java

@@ -27,8 +27,8 @@ import java.util.List;
 import java.util.Map;
 import java.util.Optional;
 import com.kingsrook.qqq.backend.core.actions.ActionHelper;
+import com.kingsrook.qqq.backend.core.actions.customizers.AbstractPostQueryCustomizer;
 import com.kingsrook.qqq.backend.core.actions.customizers.QCodeLoader;
-import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizerInterface;
 import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizers;
 import com.kingsrook.qqq.backend.core.actions.interfaces.GetInterface;
 import com.kingsrook.qqq.backend.core.actions.tables.helpers.GetActionCacheHelper;
@@ -49,7 +49,6 @@ import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleDispatcher;
 import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleInterface;
-import com.kingsrook.qqq.backend.core.utils.ObjectUtils;
 
 
 /*******************************************************************************
@@ -58,13 +57,23 @@ import com.kingsrook.qqq.backend.core.utils.ObjectUtils;
  *******************************************************************************/
 public class GetAction
 {
-   private Optional<TableCustomizerInterface> postGetRecordCustomizer;
+   private Optional<AbstractPostQueryCustomizer> postGetRecordCustomizer;
 
    private GetInput                 getInput;
    private QPossibleValueTranslator qPossibleValueTranslator;
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public QRecord executeForRecord(GetInput getInput) throws QException
+   {
+      return (execute(getInput).getRecord());
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/
@@ -78,7 +87,7 @@ public class GetAction
          throw (new QException("Requested to Get a record from an unrecognized table: " + getInput.getTableName()));
       }
 
-      postGetRecordCustomizer = QCodeLoader.getTableCustomizer(table, TableCustomizers.POST_QUERY_RECORD.getRole());
+      postGetRecordCustomizer = QCodeLoader.getTableCustomizer(AbstractPostQueryCustomizer.class, table, TableCustomizers.POST_QUERY_RECORD.getRole());
       this.getInput = getInput;
 
       QBackendModuleDispatcher qBackendModuleDispatcher = new QBackendModuleDispatcher();
@@ -98,11 +107,9 @@ public class GetAction
       }
 
       GetOutput getOutput;
-      boolean   usingDefaultGetInterface = false;
       if(getInterface == null)
       {
          getInterface = new DefaultGetInterface();
-         usingDefaultGetInterface = true;
       }
 
       getInterface.validateInput(getInput);
@@ -116,11 +123,10 @@ public class GetAction
          new GetActionCacheHelper().handleCaching(getInput, getOutput);
       }
 
-      ////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-      // if the record is found, perform post-actions on it                                                         //
-      // unless the defaultGetInterface was used - as it just does a query, and the query will do the post-actions. //
-      ////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-      if(getOutput.getRecord() != null && !usingDefaultGetInterface)
+      ////////////////////////////////////////////////////////
+      // if the record is found, perform post-actions on it //
+      ////////////////////////////////////////////////////////
+      if(getOutput.getRecord() != null)
       {
          getOutput.setRecord(postRecordActions(getOutput.getRecord()));
       }
@@ -130,43 +136,6 @@ public class GetAction
 
 
 
-   /*******************************************************************************
-    ** shorthand way to call for the most common use-case, when you just want the
-    ** output record to be returned.
-    *******************************************************************************/
-   public QRecord executeForRecord(GetInput getInput) throws QException
-   {
-      return (execute(getInput).getRecord());
-   }
-
-
-
-   /*******************************************************************************
-    ** more shorthand way to call for the most common use-case, when you just want the
-    ** output record to be returned, and you just want to pass in a table name and primary key.
-    *******************************************************************************/
-   public static QRecord execute(String tableName, Serializable primaryKey) throws QException
-   {
-      GetAction getAction = new GetAction();
-      GetInput  getInput  = new GetInput(tableName).withPrimaryKey(primaryKey);
-      return getAction.executeForRecord(getInput);
-   }
-
-
-
-   /*******************************************************************************
-    ** more shorthand way to call for the most common use-case, when you just want the
-    ** output record to be returned, and you just want to pass in a table name and unique key
-    *******************************************************************************/
-   public static QRecord execute(String tableName, Map<String, Serializable> uniqueKey) throws QException
-   {
-      GetAction getAction = new GetAction();
-      GetInput  getInput  = new GetInput(tableName).withUniqueKey(uniqueKey);
-      return getAction.executeForRecord(getInput);
-   }
-
-
-
    /*******************************************************************************
     ** Run a GetAction by using the QueryAction instead (e.g., with a filter made
     ** from the pkey/ukey, and returning the single record if found).
@@ -233,7 +202,7 @@ public class GetAction
       }
       else
       {
-         throw (new QException("Unable to get " + ObjectUtils.tryElse(() -> queryInput.getTable().getLabel(), queryInput.getTableName()) + ".  Missing required input."));
+         throw (new QException("No primaryKey or uniqueKey was passed to Get"));
       }
 
       queryInput.setFilter(filter);
@@ -247,12 +216,12 @@ public class GetAction
     ** Run the necessary actions on a record.  This may include setting display values,
     ** translating possible values, and running post-record customizations.
     *******************************************************************************/
-   public QRecord postRecordActions(QRecord record) throws QException
+   public QRecord postRecordActions(QRecord record)
    {
       QRecord returnRecord = record;
       if(this.postGetRecordCustomizer.isPresent())
       {
-         returnRecord = postGetRecordCustomizer.get().postQuery(getInput, List.of(record)).get(0);
+         returnRecord = postGetRecordCustomizer.get().apply(List.of(record)).get(0);
       }
 
       if(getInput.getShouldTranslatePossibleValues())

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/InsertAction.java

@@ -28,7 +28,6 @@ import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
-import java.util.Objects;
 import java.util.Optional;
 import java.util.Set;
 import java.util.stream.Collectors;
@@ -38,9 +37,9 @@ import com.kingsrook.qqq.backend.core.actions.QBackendTransaction;
 import com.kingsrook.qqq.backend.core.actions.audits.DMLAuditAction;
 import com.kingsrook.qqq.backend.core.actions.automation.AutomationStatus;
 import com.kingsrook.qqq.backend.core.actions.automation.RecordAutomationStatusUpdater;
+import com.kingsrook.qqq.backend.core.actions.customizers.AbstractPostInsertCustomizer;
 import com.kingsrook.qqq.backend.core.actions.customizers.AbstractPreInsertCustomizer;
 import com.kingsrook.qqq.backend.core.actions.customizers.QCodeLoader;
-import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizerInterface;
 import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizers;
 import com.kingsrook.qqq.backend.core.actions.interfaces.InsertInterface;
 import com.kingsrook.qqq.backend.core.actions.tables.helpers.UniqueKeyHelper;
@@ -62,7 +61,6 @@ import com.kingsrook.qqq.backend.core.model.metadata.tables.Association;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.UniqueKey;
 import com.kingsrook.qqq.backend.core.model.statusmessages.BadInputStatusMessage;
-import com.kingsrook.qqq.backend.core.model.statusmessages.DuplicateKeyBadInputStatusMessage;
 import com.kingsrook.qqq.backend.core.model.statusmessages.QWarningMessage;
 import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleDispatcher;
 import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleInterface;
@@ -170,12 +168,13 @@ public class InsertAction extends AbstractQActionFunction<InsertInput, InsertOut
       //////////////////////////////////////////////////////////////
       // finally, run the post-insert customizer, if there is one //
       //////////////////////////////////////////////////////////////
-      Optional<TableCustomizerInterface> postInsertCustomizer = QCodeLoader.getTableCustomizer(table, TableCustomizers.POST_INSERT_RECORD.getRole());
+      Optional<AbstractPostInsertCustomizer> postInsertCustomizer = QCodeLoader.getTableCustomizer(AbstractPostInsertCustomizer.class, table, TableCustomizers.POST_INSERT_RECORD.getRole());
       if(postInsertCustomizer.isPresent())
       {
          try
          {
-            insertOutput.setRecords(postInsertCustomizer.get().postInsert(insertInput, insertOutput.getRecords()));
+            postInsertCustomizer.get().setInsertInput(insertInput);
+            insertOutput.setRecords(postInsertCustomizer.get().apply(insertOutput.getRecords()));
          }
          catch(Exception e)
          {
@@ -233,29 +232,31 @@ public class InsertAction extends AbstractQActionFunction<InsertInput, InsertOut
       // load the pre-insert customizer and set it up, if there is one //
       // then we'll run it based on its WhenToRun value                //
       ///////////////////////////////////////////////////////////////////
-      Optional<TableCustomizerInterface> preInsertCustomizer = QCodeLoader.getTableCustomizer(table, TableCustomizers.PRE_INSERT_RECORD.getRole());
+      Optional<AbstractPreInsertCustomizer> preInsertCustomizer = QCodeLoader.getTableCustomizer(AbstractPreInsertCustomizer.class, table, TableCustomizers.PRE_INSERT_RECORD.getRole());
       if(preInsertCustomizer.isPresent())
       {
-         runPreInsertCustomizerIfItIsTime(insertInput, isPreview, preInsertCustomizer, AbstractPreInsertCustomizer.WhenToRun.BEFORE_ALL_VALIDATIONS);
+         preInsertCustomizer.get().setInsertInput(insertInput);
+         preInsertCustomizer.get().setIsPreview(isPreview);
+         runPreInsertCustomizerIfItIsTime(insertInput, preInsertCustomizer, AbstractPreInsertCustomizer.WhenToRun.BEFORE_ALL_VALIDATIONS);
       }
 
       setDefaultValuesInRecords(table, insertInput.getRecords());
 
-      ValueBehaviorApplier.applyFieldBehaviors(ValueBehaviorApplier.Action.INSERT, insertInput.getInstance(), table, insertInput.getRecords(), null);
+      ValueBehaviorApplier.applyFieldBehaviors(ValueBehaviorApplier.Action.INSERT, insertInput.getInstance(), table, insertInput.getRecords());
 
-      runPreInsertCustomizerIfItIsTime(insertInput, isPreview, preInsertCustomizer, AbstractPreInsertCustomizer.WhenToRun.BEFORE_UNIQUE_KEY_CHECKS);
+      runPreInsertCustomizerIfItIsTime(insertInput, preInsertCustomizer, AbstractPreInsertCustomizer.WhenToRun.BEFORE_UNIQUE_KEY_CHECKS);
       setErrorsIfUniqueKeyErrors(insertInput, table);
 
-      runPreInsertCustomizerIfItIsTime(insertInput, isPreview, preInsertCustomizer, AbstractPreInsertCustomizer.WhenToRun.BEFORE_REQUIRED_FIELD_CHECKS);
+      runPreInsertCustomizerIfItIsTime(insertInput, preInsertCustomizer, AbstractPreInsertCustomizer.WhenToRun.BEFORE_REQUIRED_FIELD_CHECKS);
       if(insertInput.getInputSource().shouldValidateRequiredFields())
       {
          validateRequiredFields(insertInput);
       }
 
-      runPreInsertCustomizerIfItIsTime(insertInput, isPreview, preInsertCustomizer, AbstractPreInsertCustomizer.WhenToRun.BEFORE_SECURITY_CHECKS);
+      runPreInsertCustomizerIfItIsTime(insertInput, preInsertCustomizer, AbstractPreInsertCustomizer.WhenToRun.BEFORE_SECURITY_CHECKS);
       ValidateRecordSecurityLockHelper.validateSecurityFields(insertInput.getTable(), insertInput.getRecords(), ValidateRecordSecurityLockHelper.Action.INSERT);
 
-      runPreInsertCustomizerIfItIsTime(insertInput, isPreview, preInsertCustomizer, AbstractPreInsertCustomizer.WhenToRun.AFTER_ALL_VALIDATIONS);
+      runPreInsertCustomizerIfItIsTime(insertInput, preInsertCustomizer, AbstractPreInsertCustomizer.WhenToRun.AFTER_ALL_VALIDATIONS);
    }
 
 
@@ -289,13 +290,13 @@ public class InsertAction extends AbstractQActionFunction<InsertInput, InsertOut
    /*******************************************************************************
     **
     *******************************************************************************/
-   private void runPreInsertCustomizerIfItIsTime(InsertInput insertInput, boolean isPreview, Optional<TableCustomizerInterface> preInsertCustomizer, AbstractPreInsertCustomizer.WhenToRun whenToRun) throws QException
+   private void runPreInsertCustomizerIfItIsTime(InsertInput insertInput, Optional<AbstractPreInsertCustomizer> preInsertCustomizer, AbstractPreInsertCustomizer.WhenToRun whenToRun) throws QException
    {
       if(preInsertCustomizer.isPresent())
       {
-         if(whenToRun.equals(preInsertCustomizer.get().whenToRunPreInsert(insertInput, isPreview)))
+         if(whenToRun.equals(preInsertCustomizer.get().getWhenToRun()))
          {
-            insertInput.setRecords(preInsertCustomizer.get().preInsert(insertInput, insertInput.getRecords(), isPreview));
+            insertInput.setRecords(preInsertCustomizer.get().apply(insertInput.getRecords()));
          }
       }
    }
@@ -320,7 +321,7 @@ public class InsertAction extends AbstractQActionFunction<InsertInput, InsertOut
             {
                if(record.getValue(requiredField.getName()) == null || (requiredField.getType().isStringLike() && record.getValueString(requiredField.getName()).trim().equals("")))
                {
-                  record.addError(new BadInputStatusMessage("Missing value in required field: " + Objects.requireNonNullElse(requiredField.getLabel(), requiredField.getName())));
+                  record.addError(new BadInputStatusMessage("Missing value in required field: " + requiredField.getLabel()));
                }
             }
          }
@@ -415,7 +416,7 @@ public class InsertAction extends AbstractQActionFunction<InsertInput, InsertOut
                Optional<List<Serializable>> keyValues = UniqueKeyHelper.getKeyValues(table, uniqueKey, record);
                if(keyValues.isPresent() && (existingKeys.get(uniqueKey).contains(keyValues.get()) || keysInThisList.get(uniqueKey).contains(keyValues.get())))
                {
-                  record.addError(new DuplicateKeyBadInputStatusMessage("Another record already exists with this " + uniqueKey.getDescription(table)));
+                  record.addError(new BadInputStatusMessage("Another record already exists with this " + uniqueKey.getDescription(table)));
                   foundDupe = true;
                   break;
                }
@@ -443,7 +444,7 @@ public class InsertAction extends AbstractQActionFunction<InsertInput, InsertOut
     *******************************************************************************/
    private void setAutomationStatusField(InsertInput insertInput)
    {
-      RecordAutomationStatusUpdater.setAutomationStatusInRecords(insertInput.getTable(), insertInput.getRecords(), AutomationStatus.PENDING_INSERT_AUTOMATIONS, insertInput.getTransaction(), null);
+      RecordAutomationStatusUpdater.setAutomationStatusInRecords(insertInput.getSession(), insertInput.getTable(), insertInput.getRecords(), AutomationStatus.PENDING_INSERT_AUTOMATIONS);
    }
 
 
@@ -454,7 +455,7 @@ public class InsertAction extends AbstractQActionFunction<InsertInput, InsertOut
    private QBackendModuleInterface getBackendModuleInterface(QBackendMetaData backend) throws QException
    {
       QBackendModuleDispatcher qBackendModuleDispatcher = new QBackendModuleDispatcher();
-      QBackendModuleInterface  qModule                  = qBackendModuleDispatcher.getQBackendModule(backend);
+      QBackendModuleInterface qModule = qBackendModuleDispatcher.getQBackendModule(backend);
       return (qModule);
    }
 

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/QueryAction.java

@@ -31,8 +31,8 @@ import java.util.Map;
 import java.util.Optional;
 import java.util.Set;
 import com.kingsrook.qqq.backend.core.actions.ActionHelper;
+import com.kingsrook.qqq.backend.core.actions.customizers.AbstractPostQueryCustomizer;
 import com.kingsrook.qqq.backend.core.actions.customizers.QCodeLoader;
-import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizerInterface;
 import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizers;
 import com.kingsrook.qqq.backend.core.actions.interfaces.QueryInterface;
 import com.kingsrook.qqq.backend.core.actions.reporting.BufferedRecordPipe;
@@ -73,7 +73,7 @@ public class QueryAction
 {
    private static final QLogger LOG = QLogger.getLogger(QueryAction.class);
 
-   private Optional<TableCustomizerInterface> postQueryRecordCustomizer;
+   private Optional<AbstractPostQueryCustomizer> postQueryRecordCustomizer;
 
    private QueryInput               queryInput;
    private QueryInterface           queryInterface;
@@ -100,7 +100,7 @@ public class QueryAction
       }
 
       QBackendMetaData backend = queryInput.getBackend();
-      postQueryRecordCustomizer = QCodeLoader.getTableCustomizer(table, TableCustomizers.POST_QUERY_RECORD.getRole());
+      postQueryRecordCustomizer = QCodeLoader.getTableCustomizer(AbstractPostQueryCustomizer.class, table, TableCustomizers.POST_QUERY_RECORD.getRole());
       this.queryInput = queryInput;
 
       if(queryInput.getRecordPipe() != null)
@@ -151,22 +151,6 @@ public class QueryAction
 
 
 
-   /*******************************************************************************
-    ** shorthand way to call for the most common use-case, when you just want the
-    ** records to be returned, and you just want to pass in a table name and filter.
-    *******************************************************************************/
-   public static List<QRecord> execute(String tableName, QQueryFilter filter) throws QException
-   {
-      QueryAction queryAction = new QueryAction();
-      QueryInput  queryInput  = new QueryInput();
-      queryInput.setTableName(tableName);
-      queryInput.setFilter(filter);
-      QueryOutput queryOutput = queryAction.execute(queryInput);
-      return (queryOutput.getRecords());
-   }
-
-
-
    /*******************************************************************************
     **
     *******************************************************************************/
@@ -185,7 +169,6 @@ public class QueryAction
             nextLevelQueryInput.setTableName(association.getAssociatedTableName());
             nextLevelQueryInput.setIncludeAssociations(true);
             nextLevelQueryInput.setAssociationNamesToInclude(buildNextLevelAssociationNamesToInclude(association.getName(), queryInput.getAssociationNamesToInclude()));
-            nextLevelQueryInput.setTransaction(queryInput.getTransaction());
 
             QQueryFilter filter = new QQueryFilter();
             nextLevelQueryInput.setFilter(filter);
@@ -281,7 +264,7 @@ public class QueryAction
    {
       if(this.postQueryRecordCustomizer.isPresent())
       {
-         records = postQueryRecordCustomizer.get().postQuery(queryInput, records);
+         records = postQueryRecordCustomizer.get().apply(records);
       }
 
       if(queryInput.getShouldTranslatePossibleValues())

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/ReplaceAction.java

@@ -47,7 +47,6 @@ import com.kingsrook.qqq.backend.core.model.data.QRecord;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.UniqueKey;
 import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
-import org.apache.commons.lang.BooleanUtils;
 
 
 /*******************************************************************************
@@ -80,11 +79,9 @@ public class ReplaceAction extends AbstractQActionFunction<ReplaceInput, Replace
 
       try
       {
-         QTableMetaData table                     = input.getTable();
-         UniqueKey      uniqueKey                 = input.getKey();
-         String         primaryKeyField           = table.getPrimaryKeyField();
-         boolean        allowNullKeyValuesToEqual = BooleanUtils.isTrue(input.getAllowNullKeyValuesToEqual());
-
+         QTableMetaData table           = input.getTable();
+         UniqueKey      uniqueKey       = input.getKey();
+         String         primaryKeyField = table.getPrimaryKeyField();
          if(transaction == null)
          {
             transaction = QBackendTransaction.openFor(new InsertInput(input.getTableName()));
@@ -101,11 +98,10 @@ public class ReplaceAction extends AbstractQActionFunction<ReplaceInput, Replace
             // originally it was thought that we'd need to pass the filter in here           //
             // but, it's been decided not to.  the filter only applies to what we can delete //
             ///////////////////////////////////////////////////////////////////////////////////
-            Map<List<Serializable>, Serializable> existingKeys = UniqueKeyHelper.getExistingKeys(transaction, table, page, uniqueKey, allowNullKeyValuesToEqual);
-
+            Map<List<Serializable>, Serializable> existingKeys = UniqueKeyHelper.getExistingKeys(transaction, table, page, uniqueKey);
             for(QRecord record : page)
             {
-               Optional<List<Serializable>> keyValues = UniqueKeyHelper.getKeyValues(table, uniqueKey, record, allowNullKeyValuesToEqual);
+               Optional<List<Serializable>> keyValues = UniqueKeyHelper.getKeyValues(table, uniqueKey, record);
                if(keyValues.isPresent())
                {
                   if(existingKeys.containsKey(keyValues.get()))
@@ -140,33 +136,19 @@ public class ReplaceAction extends AbstractQActionFunction<ReplaceInput, Replace
          UpdateOutput updateOutput = new UpdateAction().execute(updateInput);
          output.setUpdateOutput(updateOutput);
 
-         if(input.getPerformDeletes())
+         QQueryFilter deleteFilter = new QQueryFilter(new QFilterCriteria(primaryKeyField, QCriteriaOperator.NOT_IN, primaryKeysToKeep));
+         if(input.getFilter() != null)
          {
-            QQueryFilter deleteFilter = new QQueryFilter(new QFilterCriteria(primaryKeyField, QCriteriaOperator.NOT_IN, primaryKeysToKeep));
-            if(input.getFilter() != null)
-            {
-               deleteFilter.addSubFilter(input.getFilter());
-            }
-
-            DeleteInput deleteInput = new DeleteInput();
-            deleteInput.setTableName(table.getName());
-            deleteInput.setQueryFilter(deleteFilter);
-            deleteInput.setTransaction(transaction);
-            deleteInput.setOmitDmlAudit(input.getOmitDmlAudit());
-            DeleteOutput deleteOutput = new DeleteAction().execute(deleteInput);
-            output.setDeleteOutput(deleteOutput);
+            deleteFilter.addSubFilter(input.getFilter());
          }
 
-         if(input.getSetPrimaryKeyInInsertedRecords())
-         {
-            for(int i = 0; i < insertList.size(); i++)
-            {
-               if(i < insertOutput.getRecords().size())
-               {
-                  insertList.get(i).setValue(primaryKeyField, insertOutput.getRecords().get(i).getValue(primaryKeyField));
-               }
-            }
-         }
+         DeleteInput deleteInput = new DeleteInput();
+         deleteInput.setTableName(table.getName());
+         deleteInput.setQueryFilter(deleteFilter);
+         deleteInput.setTransaction(transaction);
+         deleteInput.setOmitDmlAudit(input.getOmitDmlAudit());
+         DeleteOutput deleteOutput = new DeleteAction().execute(deleteInput);
+         output.setDeleteOutput(deleteOutput);
 
          if(weOwnTheTransaction)
          {

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/StorageAction.java

@@ -1,120 +0,0 @@
-/*
- * QQQ - Low-code Application Framework for Engineers.
- * Copyright (C) 2021-2024.  Kingsrook, LLC
- * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
- * contact@kingsrook.com
- * https://github.com/Kingsrook/
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program.  If not, see <https://www.gnu.org/licenses/>.
- */
-
-package com.kingsrook.qqq.backend.core.actions.tables;
-
-
-import java.io.InputStream;
-import java.io.OutputStream;
-import com.kingsrook.qqq.backend.core.actions.ActionHelper;
-import com.kingsrook.qqq.backend.core.actions.interfaces.QStorageInterface;
-import com.kingsrook.qqq.backend.core.exceptions.QException;
-import com.kingsrook.qqq.backend.core.model.actions.tables.storage.StorageInput;
-import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
-import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
-import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleDispatcher;
-import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleInterface;
-
-
-/*******************************************************************************
- ** Action to do (generally, "mass") storage operations in a backend.
- **
- ** e.g., store a (potentially large) file - specifically - by working with it
- ** as either an InputStream or OutputStream.
- **
- ** May not be implemented in all backends.
- **
- *******************************************************************************/
-public class StorageAction
-{
-
-   /*******************************************************************************
-    **
-    *******************************************************************************/
-   public OutputStream createOutputStream(StorageInput storageInput) throws QException
-   {
-      QBackendModuleInterface qBackendModuleInterface = preAction(storageInput);
-      QStorageInterface       storageInterface        = qBackendModuleInterface.getStorageInterface();
-      return (storageInterface.createOutputStream(storageInput));
-   }
-
-
-
-   /*******************************************************************************
-    **
-    *******************************************************************************/
-   public InputStream getInputStream(StorageInput storageInput) throws QException
-   {
-      QBackendModuleInterface qBackendModuleInterface = preAction(storageInput);
-      QStorageInterface       storageInterface        = qBackendModuleInterface.getStorageInterface();
-      return (storageInterface.getInputStream(storageInput));
-   }
-
-
-
-   /*******************************************************************************
-    **
-    *******************************************************************************/
-   private QBackendModuleInterface preAction(StorageInput storageInput) throws QException
-   {
-      ActionHelper.validateSession(storageInput);
-
-      if(storageInput.getTableName() == null)
-      {
-         throw (new QException("Table name was not specified in storage input"));
-      }
-
-      QTableMetaData table = storageInput.getTable();
-      if(table == null)
-      {
-         throw (new QException("A table named [" + storageInput.getTableName() + "] was not found in the active QInstance"));
-      }
-
-      QBackendMetaData         backend                  = storageInput.getBackend();
-      QBackendModuleDispatcher qBackendModuleDispatcher = new QBackendModuleDispatcher();
-      QBackendModuleInterface  qModule                  = qBackendModuleDispatcher.getQBackendModule(backend);
-      return (qModule);
-   }
-
-
-
-   /*******************************************************************************
-    **
-    *******************************************************************************/
-   public void makePublic(StorageInput storageInput) throws QException
-   {
-      QBackendModuleInterface qBackendModuleInterface = preAction(storageInput);
-      QStorageInterface       storageInterface        = qBackendModuleInterface.getStorageInterface();
-      storageInterface.makePublic(storageInput);
-   }
-
-
-
-   /*******************************************************************************
-    **
-    *******************************************************************************/
-   public String getDownloadURL(StorageInput storageInput) throws QException
-   {
-      QBackendModuleInterface qBackendModuleInterface = preAction(storageInput);
-      QStorageInterface       storageInterface        = qBackendModuleInterface.getStorageInterface();
-      return (storageInterface.getDownloadURL(storageInput));
-   }
-}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/UpdateAction.java

@@ -24,7 +24,6 @@ package com.kingsrook.qqq.backend.core.actions.tables;
 
 import java.io.Serializable;
 import java.util.ArrayList;
-import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -35,8 +34,9 @@ import com.kingsrook.qqq.backend.core.actions.ActionHelper;
 import com.kingsrook.qqq.backend.core.actions.audits.DMLAuditAction;
 import com.kingsrook.qqq.backend.core.actions.automation.AutomationStatus;
 import com.kingsrook.qqq.backend.core.actions.automation.RecordAutomationStatusUpdater;
+import com.kingsrook.qqq.backend.core.actions.customizers.AbstractPostUpdateCustomizer;
+import com.kingsrook.qqq.backend.core.actions.customizers.AbstractPreUpdateCustomizer;
 import com.kingsrook.qqq.backend.core.actions.customizers.QCodeLoader;
-import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizerInterface;
 import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizers;
 import com.kingsrook.qqq.backend.core.actions.interfaces.UpdateInterface;
 import com.kingsrook.qqq.backend.core.actions.tables.helpers.ValidateRecordSecurityLockHelper;
@@ -57,8 +57,6 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateOutput;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
-import com.kingsrook.qqq.backend.core.model.metadata.fields.DynamicDefaultValueBehavior;
-import com.kingsrook.qqq.backend.core.model.metadata.fields.FieldBehavior;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
 import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinOn;
@@ -69,13 +67,11 @@ import com.kingsrook.qqq.backend.core.model.metadata.tables.Association;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 import com.kingsrook.qqq.backend.core.model.statusmessages.BadInputStatusMessage;
 import com.kingsrook.qqq.backend.core.model.statusmessages.NotFoundStatusMessage;
-import com.kingsrook.qqq.backend.core.model.statusmessages.QErrorMessage;
 import com.kingsrook.qqq.backend.core.model.statusmessages.QWarningMessage;
 import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleDispatcher;
 import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleInterface;
 import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
 import com.kingsrook.qqq.backend.core.utils.ValueUtils;
-import org.apache.commons.lang.BooleanUtils;
 import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
 
 
@@ -117,6 +113,7 @@ public class UpdateAction
    public UpdateOutput execute(UpdateInput updateInput) throws QException
    {
       ActionHelper.validateSession(updateInput);
+      setAutomationStatusField(updateInput);
 
       QTableMetaData table = updateInput.getTable();
 
@@ -133,16 +130,6 @@ public class UpdateAction
       ////////////////////////////////////////////////////////////////////////////////
       Optional<List<QRecord>> oldRecordList = fetchOldRecords(updateInput, updateInterface);
 
-      ///////////////////////////////////////////////////////////////////////////////////////
-      // allow caller to specify that we don't want to trigger automations. this isn't     //
-      // isn't expected to be used much - by design, only for the process that is meant to //
-      // heal automation status, so that it can force us into status=Pending-inserts       //
-      ///////////////////////////////////////////////////////////////////////////////////////
-      if(!updateInput.getOmitTriggeringAutomations())
-      {
-         setAutomationStatusField(updateInput, oldRecordList);
-      }
-
       performValidations(updateInput, oldRecordList, false);
 
       ////////////////////////////////////
@@ -193,12 +180,14 @@ public class UpdateAction
       //////////////////////////////////////////////////////////////
       // finally, run the post-update customizer, if there is one //
       //////////////////////////////////////////////////////////////
-      Optional<TableCustomizerInterface> postUpdateCustomizer = QCodeLoader.getTableCustomizer(table, TableCustomizers.POST_UPDATE_RECORD.getRole());
+      Optional<AbstractPostUpdateCustomizer> postUpdateCustomizer = QCodeLoader.getTableCustomizer(AbstractPostUpdateCustomizer.class, table, TableCustomizers.POST_UPDATE_RECORD.getRole());
       if(postUpdateCustomizer.isPresent())
       {
          try
          {
-            updateOutput.setRecords(postUpdateCustomizer.get().postUpdate(updateInput, updateOutput.getRecords(), oldRecordList));
+            postUpdateCustomizer.get().setUpdateInput(updateInput);
+            oldRecordList.ifPresent(l -> postUpdateCustomizer.get().setOldRecordList(l));
+            updateOutput.setRecords(postUpdateCustomizer.get().apply(updateOutput.getRecords()));
          }
          catch(Exception e)
          {
@@ -246,13 +235,7 @@ public class UpdateAction
       /////////////////////////////
       // run standard validators //
       /////////////////////////////
-      Set<FieldBehavior<?>> behaviorsToOmit = null;
-      if(BooleanUtils.isTrue(updateInput.getOmitModifyDateUpdate()))
-      {
-         behaviorsToOmit = Set.of(DynamicDefaultValueBehavior.MODIFY_DATE);
-      }
-
-      ValueBehaviorApplier.applyFieldBehaviors(ValueBehaviorApplier.Action.UPDATE, updateInput.getInstance(), table, updateInput.getRecords(), behaviorsToOmit);
+      ValueBehaviorApplier.applyFieldBehaviors(ValueBehaviorApplier.Action.UPDATE, updateInput.getInstance(), table, updateInput.getRecords());
       validatePrimaryKeysAreGiven(updateInput);
 
       if(oldRecordList.isPresent())
@@ -272,10 +255,13 @@ public class UpdateAction
       ///////////////////////////////////////////////////////////////////////////
       // after all validations, run the pre-update customizer, if there is one //
       ///////////////////////////////////////////////////////////////////////////
-      Optional<TableCustomizerInterface> preUpdateCustomizer = QCodeLoader.getTableCustomizer(table, TableCustomizers.PRE_UPDATE_RECORD.getRole());
+      Optional<AbstractPreUpdateCustomizer> preUpdateCustomizer = QCodeLoader.getTableCustomizer(AbstractPreUpdateCustomizer.class, table, TableCustomizers.PRE_UPDATE_RECORD.getRole());
       if(preUpdateCustomizer.isPresent())
       {
-         updateInput.setRecords(preUpdateCustomizer.get().preUpdate(updateInput, updateInput.getRecords(), isPreview, oldRecordList));
+         preUpdateCustomizer.get().setUpdateInput(updateInput);
+         preUpdateCustomizer.get().setIsPreview(isPreview);
+         oldRecordList.ifPresent(l -> preUpdateCustomizer.get().setOldRecordList(l));
+         updateInput.setRecords(preUpdateCustomizer.get().apply(updateInput.getRecords()));
       }
    }
 
@@ -336,9 +322,6 @@ public class UpdateAction
       QTableMetaData table           = updateInput.getTable();
       QFieldMetaData primaryKeyField = table.getField(table.getPrimaryKeyField());
 
-      /////////////////////////////////////////////////////////////
-      // todo - evolve to use lock tree (e.g., from multi-locks) //
-      /////////////////////////////////////////////////////////////
       List<RecordSecurityLock> onlyWriteLocks = RecordSecurityLockFilters.filterForOnlyWriteLocks(CollectionUtils.nonNullList(table.getRecordSecurityLocks()));
 
       for(List<QRecord> page : CollectionUtils.getPages(updateInput.getRecords(), 1000))
@@ -398,12 +381,7 @@ public class UpdateAction
                   QRecord      oldRecord = lookedUpRecords.get(value);
                   QFieldType   fieldType = table.getField(lock.getFieldName()).getType();
                   Serializable lockValue = ValueUtils.getValueAsFieldType(fieldType, oldRecord.getValue(lock.getFieldName()));
-
-                  List<QErrorMessage> errors = ValidateRecordSecurityLockHelper.validateRecordSecurityValue(table, lock, lockValue, fieldType, ValidateRecordSecurityLockHelper.Action.UPDATE, Collections.emptyMap());
-                  if(CollectionUtils.nullSafeHasContents(errors))
-                  {
-                     errors.forEach(e -> record.addError(e));
-                  }
+                  ValidateRecordSecurityLockHelper.validateRecordSecurityValue(table, record, lock, lockValue, fieldType, ValidateRecordSecurityLockHelper.Action.UPDATE);
                }
             }
          }
@@ -583,9 +561,9 @@ public class UpdateAction
    /*******************************************************************************
     ** If the table being updated uses an automation-status field, populate it now.
     *******************************************************************************/
-   private void setAutomationStatusField(UpdateInput updateInput, Optional<List<QRecord>> oldRecordList)
+   private void setAutomationStatusField(UpdateInput updateInput)
    {
-      RecordAutomationStatusUpdater.setAutomationStatusInRecords(updateInput.getTable(), updateInput.getRecords(), AutomationStatus.PENDING_UPDATE_AUTOMATIONS, updateInput.getTransaction(), oldRecordList.orElse(null));
+      RecordAutomationStatusUpdater.setAutomationStatusInRecords(updateInput.getSession(), updateInput.getTable(), updateInput.getRecords(), AutomationStatus.PENDING_UPDATE_AUTOMATIONS);
    }
 
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/helpers/ActionTimeoutHelper.java

@@ -23,10 +23,8 @@ package com.kingsrook.qqq.backend.core.actions.tables.helpers;
 
 
 import java.util.concurrent.Executors;
-import java.util.concurrent.ScheduledExecutorService;
 import java.util.concurrent.ScheduledFuture;
 import java.util.concurrent.TimeUnit;
-import com.kingsrook.qqq.backend.core.utils.PrefixedDefaultThreadFactory;
 
 
 /*******************************************************************************
@@ -52,9 +50,6 @@ public class ActionTimeoutHelper
 
    private boolean didTimeout = false;
 
-   private static Integer                  CORE_THREADS             = 10;
-   private static ScheduledExecutorService scheduledExecutorService = Executors.newScheduledThreadPool(CORE_THREADS, new PrefixedDefaultThreadFactory(ActionTimeoutHelper.class));
-
 
 
    /*******************************************************************************
@@ -80,7 +75,7 @@ public class ActionTimeoutHelper
          return;
       }
 
-      future = scheduledExecutorService.schedule(() ->
+      future = Executors.newSingleThreadScheduledExecutor().schedule(() ->
       {
          didTimeout = true;
          runnable.run();

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/helpers/QueryStatManager.java

@@ -57,7 +57,6 @@ import com.kingsrook.qqq.backend.core.model.session.QSession;
 import com.kingsrook.qqq.backend.core.model.tables.QQQTable;
 import com.kingsrook.qqq.backend.core.model.tables.QQQTablesMetaDataProvider;
 import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
-import com.kingsrook.qqq.backend.core.utils.PrefixedDefaultThreadFactory;
 import com.kingsrook.qqq.backend.core.utils.StringUtils;
 import com.kingsrook.qqq.backend.core.utils.collections.MapBuilder;
 import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
@@ -177,7 +176,7 @@ public class QueryStatManager
       active = true;
       queryStats = new ArrayList<>();
 
-      executorService = Executors.newSingleThreadScheduledExecutor(new PrefixedDefaultThreadFactory(this));
+      executorService = Executors.newSingleThreadScheduledExecutor();
       executorService.scheduleAtFixedRate(new QueryStatManagerInsertJob(), jobInitialDelay, jobPeriodSeconds, TimeUnit.SECONDS);
    }
 
@@ -215,78 +214,71 @@ public class QueryStatManager
     *******************************************************************************/
    public void add(QueryStat queryStat)
    {
-      try
+      if(queryStat == null)
       {
-         if(queryStat == null)
+         return;
+      }
+
+      if(active)
+      {
+         ////////////////////////////////////////////////////////////////////////////////////////
+         // set fields that we need to capture now (rather than when the thread to store runs) //
+         ////////////////////////////////////////////////////////////////////////////////////////
+         if(queryStat.getFirstResultTimestamp() == null)
          {
+            queryStat.setFirstResultTimestamp(Instant.now());
+         }
+
+         if(queryStat.getStartTimestamp() != null && queryStat.getFirstResultTimestamp() != null && queryStat.getFirstResultMillis() == null)
+         {
+            long millis = queryStat.getFirstResultTimestamp().toEpochMilli() - queryStat.getStartTimestamp().toEpochMilli();
+            queryStat.setFirstResultMillis((int) millis);
+         }
+
+         if(queryStat.getFirstResultMillis() != null && queryStat.getFirstResultMillis() < minMillisToStore)
+         {
+            //////////////////////////////////////////////////////////////
+            // discard this record if it's under the min millis setting //
+            //////////////////////////////////////////////////////////////
             return;
          }
 
-         if(active)
+         if(queryStat.getSessionId() == null && QContext.getQSession() != null)
          {
-            ////////////////////////////////////////////////////////////////////////////////////////
-            // set fields that we need to capture now (rather than when the thread to store runs) //
-            ////////////////////////////////////////////////////////////////////////////////////////
-            if(queryStat.getFirstResultTimestamp() == null)
-            {
-               queryStat.setFirstResultTimestamp(Instant.now());
-            }
+            queryStat.setSessionId(QContext.getQSession().getUuid());
+         }
 
-            if(queryStat.getStartTimestamp() != null && queryStat.getFirstResultTimestamp() != null && queryStat.getFirstResultMillis() == null)
+         if(queryStat.getAction() == null)
+         {
+            if(!QContext.getActionStack().isEmpty())
             {
-               long millis = queryStat.getFirstResultTimestamp().toEpochMilli() - queryStat.getStartTimestamp().toEpochMilli();
-               queryStat.setFirstResultMillis((int) millis);
+               queryStat.setAction(QContext.getActionStack().peek().getActionIdentity());
             }
-
-            if(queryStat.getFirstResultMillis() != null && queryStat.getFirstResultMillis() < minMillisToStore)
+            else
             {
-               //////////////////////////////////////////////////////////////
-               // discard this record if it's under the min millis setting //
-               //////////////////////////////////////////////////////////////
-               return;
-            }
-
-            if(queryStat.getSessionId() == null && QContext.getQSession() != null)
-            {
-               queryStat.setSessionId(QContext.getQSession().getUuid());
-            }
-
-            if(queryStat.getAction() == null)
-            {
-               if(QContext.getActionStack() != null && !QContext.getActionStack().isEmpty())
+               boolean   expected = false;
+               Exception e        = new Exception("Unexpected empty action stack");
+               for(StackTraceElement stackTraceElement : e.getStackTrace())
                {
-                  queryStat.setAction(QContext.getActionStack().peek().getActionIdentity());
-               }
-               else
-               {
-                  boolean   expected = false;
-                  Exception e        = new Exception("Unexpected empty action stack");
-                  for(StackTraceElement stackTraceElement : e.getStackTrace())
+                  String className = stackTraceElement.getClassName();
+                  if(className.contains(QueryStatManagerInsertJob.class.getName()))
                   {
-                     String className = stackTraceElement.getClassName();
-                     if(className.contains(QueryStatManagerInsertJob.class.getName()))
-                     {
-                        expected = true;
-                        break;
-                     }
-                  }
-
-                  if(!expected)
-                  {
-                     LOG.debug(e);
+                     expected = true;
+                     break;
                   }
                }
-            }
 
-            synchronized(this)
-            {
-               queryStats.add(queryStat);
+               if(!expected)
+               {
+                  LOG.debug(e);
+               }
             }
          }
-      }
-      catch(Exception e)
-      {
-         LOG.debug("Error adding query stat", e);
+
+         synchronized(this)
+         {
+            queryStats.add(queryStat);
+         }
       }
    }
 

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/helpers/UniqueKeyHelper.java

@@ -54,7 +54,7 @@ public class UniqueKeyHelper
    /*******************************************************************************
     **
     *******************************************************************************/
-   public static Map<List<Serializable>, Serializable> getExistingKeys(QBackendTransaction transaction, QTableMetaData table, List<QRecord> recordList, UniqueKey uniqueKey, boolean allowNullKeyValuesToEqual) throws QException
+   public static Map<List<Serializable>, Serializable> getExistingKeys(QBackendTransaction transaction, QTableMetaData table, List<QRecord> recordList, UniqueKey uniqueKey) throws QException
    {
       List<String>                          ukFieldNames    = uniqueKey.getFieldNames();
       Map<List<Serializable>, Serializable> existingRecords = new HashMap<>();
@@ -112,7 +112,7 @@ public class UniqueKeyHelper
          QueryOutput queryOutput = new QueryAction().execute(queryInput);
          for(QRecord record : queryOutput.getRecords())
          {
-            Optional<List<Serializable>> keyValues = getKeyValues(table, uniqueKey, record, allowNullKeyValuesToEqual);
+            Optional<List<Serializable>> keyValues = getKeyValues(table, uniqueKey, record);
             if(keyValues.isPresent())
             {
                existingRecords.put(keyValues.get(), record.getValue(table.getPrimaryKeyField()));
@@ -128,17 +128,7 @@ public class UniqueKeyHelper
    /*******************************************************************************
     **
     *******************************************************************************/
-   public static Map<List<Serializable>, Serializable> getExistingKeys(QBackendTransaction transaction, QTableMetaData table, List<QRecord> recordList, UniqueKey uniqueKey) throws QException
-   {
-      return (getExistingKeys(transaction, table, recordList, uniqueKey, false));
-   }
-
-
-
-   /*******************************************************************************
-    **
-    *******************************************************************************/
-   public static Optional<List<Serializable>> getKeyValues(QTableMetaData table, UniqueKey uniqueKey, QRecord record, boolean allowNullKeyValuesToEqual)
+   public static Optional<List<Serializable>> getKeyValues(QTableMetaData table, UniqueKey uniqueKey, QRecord record)
    {
       try
       {
@@ -148,19 +138,7 @@ public class UniqueKeyHelper
             QFieldMetaData field      = table.getField(fieldName);
             Serializable   value      = record.getValue(fieldName);
             Serializable   typedValue = ValueUtils.getValueAsFieldType(field.getType(), value);
-
-            ///////////////////////////////////////////////////////////////////////////////////
-            // if null value, look at flag to determine if a null should be used (which will //
-            // allow keys to match), or a NullUniqueKeyValue, (which will never match)       //
-            ///////////////////////////////////////////////////////////////////////////////////
-            if(typedValue == null)
-            {
-               keyValues.add(allowNullKeyValuesToEqual ? null : new NullUniqueKeyValue());
-            }
-            else
-            {
-               keyValues.add(typedValue);
-            }
+            keyValues.add(typedValue == null ? new NullUniqueKeyValue() : typedValue);
          }
          return (Optional.of(keyValues));
       }
@@ -172,16 +150,6 @@ public class UniqueKeyHelper
 
 
 
-   /*******************************************************************************
-    **
-    *******************************************************************************/
-   public static Optional<List<Serializable>> getKeyValues(QTableMetaData table, UniqueKey uniqueKey, QRecord record)
-   {
-      return (getKeyValues(table, uniqueKey, record, false));
-   }
-
-
-
    /*******************************************************************************
     ** To make a list of unique key values here behave like they do in an RDBMS
     ** (which is what we're trying to mimic - which is - 2 null values in a field

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/helpers/ValidateRecordSecurityLockHelper.java

@@ -24,7 +24,6 @@ package com.kingsrook.qqq.backend.core.actions.tables.helpers;
 
 import java.io.Serializable;
 import java.util.ArrayList;
-import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -43,16 +42,12 @@ import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
 import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinOn;
 import com.kingsrook.qqq.backend.core.model.metadata.joins.QJoinMetaData;
-import com.kingsrook.qqq.backend.core.model.metadata.security.MultiRecordSecurityLock;
-import com.kingsrook.qqq.backend.core.model.metadata.security.NullValueBehaviorUtil;
 import com.kingsrook.qqq.backend.core.model.metadata.security.QSecurityKeyType;
 import com.kingsrook.qqq.backend.core.model.metadata.security.RecordSecurityLock;
 import com.kingsrook.qqq.backend.core.model.metadata.security.RecordSecurityLockFilters;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 import com.kingsrook.qqq.backend.core.model.statusmessages.PermissionDeniedMessage;
-import com.kingsrook.qqq.backend.core.model.statusmessages.QErrorMessage;
 import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
-import com.kingsrook.qqq.backend.core.utils.JsonUtils;
 import com.kingsrook.qqq.backend.core.utils.ListingHash;
 import com.kingsrook.qqq.backend.core.utils.StringUtils;
 import com.kingsrook.qqq.backend.core.utils.ValueUtils;
@@ -85,273 +80,160 @@ public class ValidateRecordSecurityLockHelper
     *******************************************************************************/
    public static void validateSecurityFields(QTableMetaData table, List<QRecord> records, Action action) throws QException
    {
-      MultiRecordSecurityLock locksToCheck = getRecordSecurityLocks(table, action);
-      if(locksToCheck == null || CollectionUtils.nullSafeIsEmpty(locksToCheck.getLocks()))
+      List<RecordSecurityLock> locksToCheck = getRecordSecurityLocks(table, action);
+      if(CollectionUtils.nullSafeIsEmpty(locksToCheck))
       {
          return;
       }
 
-      //////////////////////////////////////////////////////////////////////////////////////////
-      // we will be relying on primary keys being set in records - but (at least for inserts) //
-      // we might not have pkeys - so make them up (and clear them out at the end)            //
-      //////////////////////////////////////////////////////////////////////////////////////////
-      Map<Serializable, QRecord> madeUpPrimaryKeys = makeUpPrimaryKeysIfNeeded(records, table);
-
       ////////////////////////////////
       // actually check lock values //
       ////////////////////////////////
-      Map<Serializable, RecordWithErrors> errorRecords = new HashMap<>();
-      evaluateRecordLocks(table, records, action, locksToCheck, errorRecords, new ArrayList<>(), madeUpPrimaryKeys);
-
-      /////////////////////////////////
-      // propagate errors to records //
-      /////////////////////////////////
-      for(RecordWithErrors recordWithErrors : errorRecords.values())
+      for(RecordSecurityLock recordSecurityLock : locksToCheck)
       {
-         recordWithErrors.propagateErrorsToRecord(locksToCheck);
-      }
-
-      /////////////////////////////////
-      // remove made-up primary keys //
-      /////////////////////////////////
-      String primaryKeyField = table.getPrimaryKeyField();
-      for(QRecord record : madeUpPrimaryKeys.values())
-      {
-         record.setValue(primaryKeyField, null);
-      }
-   }
-
-
-
-   /*******************************************************************************
-    ** For a list of `records` from a `table`, and a given `action`, evaluate a
-    ** `recordSecurityLock` (which may be a multi-lock) - populating the input map
-    ** of `errorRecords` - key'ed by primary key value (real or made up), with
-    ** error messages existing in a tree, with positions matching the multi-lock
-    ** tree that we're navigating, as tracked by `treePosition`.
-    **
-    ** Recursively processes multi-locks (and the top-level call is always with a
-    ** multi-lock - as the table's recordLocks list converted to an AND-multi-lock).
-    **
-    ** Of note - for the case of READ_WRITE locks, we're only evaluating the values
-    ** on the record, to see if they're allowed for us to store (because if we didn't
-    ** have the key, we wouldn't have been able to read the value (which is verified
-    ** outside of here, in UpdateAction/DeleteAction).
-    **
-    ** BUT - WRITE locks - in their case, we read the record no matter what, and in
-    ** here we need to verify we have a key that allows us to WRITE the record.
-    *******************************************************************************/
-   private static void evaluateRecordLocks(QTableMetaData table, List<QRecord> records, Action action, RecordSecurityLock recordSecurityLock, Map<Serializable, RecordWithErrors> errorRecords, List<Integer> treePosition, Map<Serializable, QRecord> madeUpPrimaryKeys) throws QException
-   {
-      if(recordSecurityLock instanceof MultiRecordSecurityLock multiRecordSecurityLock)
-      {
-         /////////////////////////////////////////////
-         // for multi-locks, make recursive descent //
-         /////////////////////////////////////////////
-         int i = 0;
-         for(RecordSecurityLock childLock : CollectionUtils.nonNullList(multiRecordSecurityLock.getLocks()))
+         if(CollectionUtils.nullSafeIsEmpty(recordSecurityLock.getJoinNameChain()))
          {
-            treePosition.add(i);
-            evaluateRecordLocks(table, records, action, childLock, errorRecords, treePosition, madeUpPrimaryKeys);
-            treePosition.remove(treePosition.size() - 1);
-            i++;
-         }
+            //////////////////////////////////////////////////////////////////////////////////
+            // handle the value being in the table we're inserting/updating (e.g., no join) //
+            //////////////////////////////////////////////////////////////////////////////////
+            QFieldMetaData field = table.getField(recordSecurityLock.getFieldName());
 
-         return;
-      }
-
-      /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-      // if this lock has an all-access key, and the user has that key, then there can't be any errors here, so return early //
-      /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-      QSecurityKeyType securityKeyType = QContext.getQInstance().getSecurityKeyType(recordSecurityLock.getSecurityKeyType());
-      if(StringUtils.hasContent(securityKeyType.getAllAccessKeyName()) && QContext.getQSession().hasSecurityKeyValue(securityKeyType.getAllAccessKeyName(), true, QFieldType.BOOLEAN))
-      {
-         return;
-      }
-
-      ////////////////////////////////
-      // proceed w/ non-multi locks //
-      ////////////////////////////////
-      String primaryKeyField = table.getPrimaryKeyField();
-      if(CollectionUtils.nullSafeIsEmpty(recordSecurityLock.getJoinNameChain()))
-      {
-         //////////////////////////////////////////////////////////////////////////////////
-         // handle the value being in the table we're inserting/updating (e.g., no join) //
-         //////////////////////////////////////////////////////////////////////////////////
-         QFieldMetaData field = table.getField(recordSecurityLock.getFieldName());
-
-         for(QRecord record : records)
-         {
-            if(action.equals(Action.UPDATE) && !record.getValues().containsKey(field.getName()) && RecordSecurityLock.LockScope.READ_AND_WRITE.equals(recordSecurityLock.getLockScope()))
+            for(QRecord record : records)
             {
-               /////////////////////////////////////////////////////////////////////////////////////////////////////////
-               // if this is a read-write lock, then if we have the record, it means we were able to read the record. //
-               // So if we're not updating the security field, then no error can come from it!                        //
-               /////////////////////////////////////////////////////////////////////////////////////////////////////////
-               continue;
-            }
+               if(action.equals(Action.UPDATE) && !record.getValues().containsKey(field.getName()) && RecordSecurityLock.LockScope.READ_AND_WRITE.equals(recordSecurityLock.getLockScope()))
+               {
+                  /////////////////////////////////////////////////////////////////////////////////////////////////////////
+                  // if this is a read-write lock, then if we have the record, it means we were able to read the record. //
+                  // So if we're not updating the security field, then no error can come from it!                        //
+                  /////////////////////////////////////////////////////////////////////////////////////////////////////////
+                  continue;
+               }
 
-            Serializable        recordSecurityValue = record.getValue(field.getName());
-            List<QErrorMessage> recordErrors        = validateRecordSecurityValue(table, recordSecurityLock, recordSecurityValue, field.getType(), action, madeUpPrimaryKeys);
-            if(CollectionUtils.nullSafeHasContents(recordErrors))
-            {
-               errorRecords.computeIfAbsent(record.getValue(primaryKeyField), (k) -> new RecordWithErrors(record)).addAll(recordErrors, treePosition);
+               Serializable recordSecurityValue = record.getValue(field.getName());
+               validateRecordSecurityValue(table, record, recordSecurityLock, recordSecurityValue, field.getType(), action);
             }
          }
-      }
-      else
-      {
-         ////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-         // else look for the joined record - if it isn't found, assume a fail - else validate security value if found //
-         ////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-         QJoinMetaData leftMostJoin  = QContext.getQInstance().getJoin(recordSecurityLock.getJoinNameChain().get(0));
-         QJoinMetaData rightMostJoin = QContext.getQInstance().getJoin(recordSecurityLock.getJoinNameChain().get(recordSecurityLock.getJoinNameChain().size() - 1));
-
-         ////////////////////////////////
-         // todo probably, but more... //
-         ////////////////////////////////
-         // if(leftMostJoin.getLeftTable().equals(table.getName()))
-         // {
-         //    leftMostJoin = leftMostJoin.flip();
-         // }
-
-         QTableMetaData rightMostJoinTable = QContext.getQInstance().getTable(rightMostJoin.getRightTable());
-         QTableMetaData leftMostJoinTable  = QContext.getQInstance().getTable(leftMostJoin.getLeftTable());
-
-         for(List<QRecord> inputRecordPage : CollectionUtils.getPages(records, 500))
+         else
          {
-            ////////////////////////////////////////////////////////////////////////////////////////////////
-            // set up a query for joined records                                                          //
-            // query will be like (fkey1=? and fkey2=?) OR (fkey1=? and fkey2=?) OR (fkey1=? and fkey2=?) //
-            ////////////////////////////////////////////////////////////////////////////////////////////////
-            QueryInput queryInput = new QueryInput();
-            queryInput.setTableName(leftMostJoin.getLeftTable());
-            QQueryFilter filter = new QQueryFilter().withBooleanOperator(QQueryFilter.BooleanOperator.OR);
-            queryInput.setFilter(filter);
+            ////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+            // else look for the joined record - if it isn't found, assume a fail - else validate security value if found //
+            ////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+            QJoinMetaData  leftMostJoin       = QContext.getQInstance().getJoin(recordSecurityLock.getJoinNameChain().get(0));
+            QJoinMetaData  rightMostJoin      = QContext.getQInstance().getJoin(recordSecurityLock.getJoinNameChain().get(recordSecurityLock.getJoinNameChain().size() - 1));
+            QTableMetaData rightMostJoinTable = QContext.getQInstance().getTable(rightMostJoin.getRightTable());
+            QTableMetaData leftMostJoinTable  = QContext.getQInstance().getTable(leftMostJoin.getLeftTable());
 
-            for(String joinName : recordSecurityLock.getJoinNameChain())
+            for(List<QRecord> inputRecordPage : CollectionUtils.getPages(records, 500))
             {
-               ///////////////////////////////////////
-               // we don't need the right-most join //
-               ///////////////////////////////////////
-               if(!joinName.equals(rightMostJoin.getName()))
+               ////////////////////////////////////////////////////////////////////////////////////////////////
+               // set up a query for joined records                                                          //
+               // query will be like (fkey1=? and fkey2=?) OR (fkey1=? and fkey2=?) OR (fkey1=? and fkey2=?) //
+               ////////////////////////////////////////////////////////////////////////////////////////////////
+               QueryInput queryInput = new QueryInput();
+               queryInput.setTableName(leftMostJoin.getLeftTable());
+               QQueryFilter filter = new QQueryFilter().withBooleanOperator(QQueryFilter.BooleanOperator.OR);
+               queryInput.setFilter(filter);
+
+               for(String joinName : recordSecurityLock.getJoinNameChain())
                {
-                  queryInput.withQueryJoin(new QueryJoin().withJoinMetaData(QContext.getQInstance().getJoin(joinName)).withSelect(true));
-               }
-            }
-
-            ///////////////////////////////////////////////////////////////////////////////////////////////////
-            // foreach input record (in this page), put it in a listing hash, with key = list of join-values //
-            // e.g., (17,47)=(QRecord1), (18,48)=(QRecord2,QRecord3)                                         //
-            // also build up the query's sub-filters here (only adding them if they're unique).              //
-            // e.g., 2 order-lines referencing the same orderId don't need to be added to the query twice    //
-            ///////////////////////////////////////////////////////////////////////////////////////////////////
-            ListingHash<List<Serializable>, QRecord> inputRecordMapByJoinFields = new ListingHash<>();
-            for(QRecord inputRecord : inputRecordPage)
-            {
-               List<Serializable> inputRecordJoinValues = new ArrayList<>();
-               QQueryFilter       subFilter             = new QQueryFilter();
-
-               boolean updatingAnyLockJoinFields = false;
-               for(JoinOn joinOn : rightMostJoin.getJoinOns())
-               {
-                  QFieldType   type             = rightMostJoinTable.getField(joinOn.getRightField()).getType();
-                  Serializable inputRecordValue = ValueUtils.getValueAsFieldType(type, inputRecord.getValue(joinOn.getRightField()));
-                  inputRecordJoinValues.add(inputRecordValue);
-
-                  // if we have a value in this field (and it's not the primary key), then it means we're updating part of the lock
-                  if(inputRecordValue != null && !joinOn.getRightField().equals(table.getPrimaryKeyField()))
+                  ///////////////////////////////////////
+                  // we don't need the right-most join //
+                  ///////////////////////////////////////
+                  if(!joinName.equals(rightMostJoin.getName()))
                   {
-                     updatingAnyLockJoinFields = true;
+                     queryInput.withQueryJoin(new QueryJoin().withJoinMetaData(QContext.getQInstance().getJoin(joinName)).withSelect(true));
+                  }
+               }
+
+               ///////////////////////////////////////////////////////////////////////////////////////////////////
+               // foreach input record (in this page), put it in a listing hash, with key = list of join-values //
+               // e.g., (17,47)=(QRecord1), (18,48)=(QRecord2,QRecord3)                                         //
+               // also build up the query's sub-filters here (only adding them if they're unique).              //
+               // e.g., 2 order-lines referencing the same orderId don't need to be added to the query twice    //
+               ///////////////////////////////////////////////////////////////////////////////////////////////////
+               ListingHash<List<Serializable>, QRecord> inputRecordMapByJoinFields = new ListingHash<>();
+               for(QRecord inputRecord : inputRecordPage)
+               {
+                  List<Serializable> inputRecordJoinValues = new ArrayList<>();
+                  QQueryFilter       subFilter             = new QQueryFilter();
+
+                  for(JoinOn joinOn : rightMostJoin.getJoinOns())
+                  {
+                     QFieldType   type             = rightMostJoinTable.getField(joinOn.getRightField()).getType();
+                     Serializable inputRecordValue = ValueUtils.getValueAsFieldType(type, inputRecord.getValue(joinOn.getRightField()));
+                     inputRecordJoinValues.add(inputRecordValue);
+
+                     subFilter.addCriteria(inputRecordValue == null
+                        ? new QFilterCriteria(rightMostJoin.getLeftTable() + "." + joinOn.getLeftField(), QCriteriaOperator.IS_BLANK)
+                        : new QFilterCriteria(rightMostJoin.getLeftTable() + "." + joinOn.getLeftField(), QCriteriaOperator.EQUALS, inputRecordValue));
                   }
 
-                  subFilter.addCriteria(inputRecordValue == null
-                     ? new QFilterCriteria(rightMostJoin.getLeftTable() + "." + joinOn.getLeftField(), QCriteriaOperator.IS_BLANK)
-                     : new QFilterCriteria(rightMostJoin.getLeftTable() + "." + joinOn.getLeftField(), QCriteriaOperator.EQUALS, inputRecordValue));
-               }
-
-               //////////////////////////////////
-               // todo maybe, some version of? //
-               //////////////////////////////////
-               // if(action.equals(Action.UPDATE) && !updatingAnyLockJoinFields && RecordSecurityLock.LockScope.READ_AND_WRITE.equals(recordSecurityLock.getLockScope()))
-               // {
-               //    /////////////////////////////////////////////////////////////////////////////////////////////////////////
-               //    // if this is a read-write lock, then if we have the record, it means we were able to read the record. //
-               //    // So if we're not updating the security field, then no error can come from it!                        //
-               //    /////////////////////////////////////////////////////////////////////////////////////////////////////////
-               //    continue;
-               // }
-
-               if(!inputRecordMapByJoinFields.containsKey(inputRecordJoinValues))
-               {
-                  ////////////////////////////////////////////////////////////////////////////////
-                  // only add this sub-filter if it's for a list of keys we haven't seen before //
-                  ////////////////////////////////////////////////////////////////////////////////
-                  filter.addSubFilter(subFilter);
-               }
-
-               inputRecordMapByJoinFields.add(inputRecordJoinValues, inputRecord);
-            }
-
-            //////////////////////////////////////////////////////////////////////////////////////////////////////////////
-            // execute the query for joined records - then put them in a map with keys corresponding to the join values //
-            // e.g., (17,47)=(JoinRecord), (18,48)=(JoinRecord)                                                         //
-            //////////////////////////////////////////////////////////////////////////////////////////////////////////////
-            QueryOutput                      queryOutput               = new QueryAction().execute(queryInput);
-            Map<List<Serializable>, QRecord> joinRecordMapByJoinFields = new HashMap<>();
-            for(QRecord joinRecord : queryOutput.getRecords())
-            {
-               List<Serializable> joinRecordValues = new ArrayList<>();
-               for(JoinOn joinOn : rightMostJoin.getJoinOns())
-               {
-                  Serializable joinValue = joinRecord.getValue(rightMostJoin.getLeftTable() + "." + joinOn.getLeftField());
-                  if(joinValue == null && joinRecord.getValues().keySet().stream().anyMatch(n -> !n.contains(".")))
+                  if(!inputRecordMapByJoinFields.containsKey(inputRecordJoinValues))
                   {
-                     joinValue = joinRecord.getValue(joinOn.getLeftField());
-                  }
-                  joinRecordValues.add(joinValue);
-               }
-
-               joinRecordMapByJoinFields.put(joinRecordValues, joinRecord);
-            }
-
-            //////////////////////////////////////////////////////////////////////////////////////////////////
-            // now for each input record, look for its joinRecord - if it isn't found, then this insert     //
-            // isn't allowed.  if it is found, then validate its value matches this session's security keys //
-            //////////////////////////////////////////////////////////////////////////////////////////////////
-            for(Map.Entry<List<Serializable>, List<QRecord>> entry : inputRecordMapByJoinFields.entrySet())
-            {
-               List<Serializable> inputRecordJoinValues = entry.getKey();
-               List<QRecord>      inputRecords          = entry.getValue();
-               if(joinRecordMapByJoinFields.containsKey(inputRecordJoinValues))
-               {
-                  QRecord joinRecord = joinRecordMapByJoinFields.get(inputRecordJoinValues);
-
-                  String         fieldName           = recordSecurityLock.getFieldName().replaceFirst(".*\\.", "");
-                  QFieldMetaData field               = leftMostJoinTable.getField(fieldName);
-                  Serializable   recordSecurityValue = joinRecord.getValue(fieldName);
-                  if(recordSecurityValue == null && joinRecord.getValues().keySet().stream().anyMatch(n -> n.contains(".")))
-                  {
-                     recordSecurityValue = joinRecord.getValue(recordSecurityLock.getFieldName());
+                     ////////////////////////////////////////////////////////////////////////////////
+                     // only add this sub-filter if it's for a list of keys we haven't seen before //
+                     ////////////////////////////////////////////////////////////////////////////////
+                     filter.addSubFilter(subFilter);
                   }
 
-                  for(QRecord inputRecord : inputRecords)
+                  inputRecordMapByJoinFields.add(inputRecordJoinValues, inputRecord);
+               }
+
+               //////////////////////////////////////////////////////////////////////////////////////////////////////////////
+               // execute the query for joined records - then put them in a map with keys corresponding to the join values //
+               // e.g., (17,47)=(JoinRecord), (18,48)=(JoinRecord)                                                         //
+               //////////////////////////////////////////////////////////////////////////////////////////////////////////////
+               QueryOutput                      queryOutput               = new QueryAction().execute(queryInput);
+               Map<List<Serializable>, QRecord> joinRecordMapByJoinFields = new HashMap<>();
+               for(QRecord joinRecord : queryOutput.getRecords())
+               {
+                  List<Serializable> joinRecordValues = new ArrayList<>();
+                  for(JoinOn joinOn : rightMostJoin.getJoinOns())
                   {
-                     List<QErrorMessage> recordErrors = validateRecordSecurityValue(table, recordSecurityLock, recordSecurityValue, field.getType(), action, madeUpPrimaryKeys);
-                     if(CollectionUtils.nullSafeHasContents(recordErrors))
+                     Serializable joinValue = joinRecord.getValue(rightMostJoin.getLeftTable() + "." + joinOn.getLeftField());
+                     if(joinValue == null && joinRecord.getValues().keySet().stream().anyMatch(n -> !n.contains(".")))
                      {
-                        errorRecords.computeIfAbsent(inputRecord.getValue(primaryKeyField), (k) -> new RecordWithErrors(inputRecord)).addAll(recordErrors, treePosition);
+                        joinValue = joinRecord.getValue(joinOn.getLeftField());
+                     }
+                     joinRecordValues.add(joinValue);
+                  }
+
+                  joinRecordMapByJoinFields.put(joinRecordValues, joinRecord);
+               }
+
+               //////////////////////////////////////////////////////////////////////////////////////////////////
+               // now for each input record, look for its joinRecord - if it isn't found, then this insert     //
+               // isn't allowed.  if it is found, then validate its value matches this session's security keys //
+               //////////////////////////////////////////////////////////////////////////////////////////////////
+               for(Map.Entry<List<Serializable>, List<QRecord>> entry : inputRecordMapByJoinFields.entrySet())
+               {
+                  List<Serializable> inputRecordJoinValues = entry.getKey();
+                  List<QRecord>      inputRecords          = entry.getValue();
+                  if(joinRecordMapByJoinFields.containsKey(inputRecordJoinValues))
+                  {
+                     QRecord joinRecord = joinRecordMapByJoinFields.get(inputRecordJoinValues);
+
+                     String         fieldName           = recordSecurityLock.getFieldName().replaceFirst(".*\\.", "");
+                     QFieldMetaData field               = leftMostJoinTable.getField(fieldName);
+                     Serializable   recordSecurityValue = joinRecord.getValue(fieldName);
+                     if(recordSecurityValue == null && joinRecord.getValues().keySet().stream().anyMatch(n -> n.contains(".")))
+                     {
+                        recordSecurityValue = joinRecord.getValue(recordSecurityLock.getFieldName());
+                     }
+
+                     for(QRecord inputRecord : inputRecords)
+                     {
+                        validateRecordSecurityValue(table, inputRecord, recordSecurityLock, recordSecurityValue, field.getType(), action);
                      }
                   }
-               }
-               else
-               {
-                  for(QRecord inputRecord : inputRecords)
+                  else
                   {
-                     if(RecordSecurityLock.NullValueBehavior.DENY.equals(NullValueBehaviorUtil.getEffectiveNullValueBehavior(recordSecurityLock)))
+                     for(QRecord inputRecord : inputRecords)
                      {
-                        PermissionDeniedMessage error = new PermissionDeniedMessage("You do not have permission to " + action.name().toLowerCase() + " this record - the referenced " + leftMostJoinTable.getLabel() + " was not found.");
-                        errorRecords.computeIfAbsent(inputRecord.getValue(primaryKeyField), (k) -> new RecordWithErrors(inputRecord)).add(error, treePosition);
+                        if(RecordSecurityLock.NullValueBehavior.DENY.equals(recordSecurityLock.getNullValueBehavior()))
+                        {
+                           inputRecord.addError(new PermissionDeniedMessage("You do not have permission to " + action.name().toLowerCase() + " this record - the referenced " + leftMostJoinTable.getLabel() + " was not found."));
+                        }
                      }
                   }
                }
@@ -363,80 +245,45 @@ public class ValidateRecordSecurityLockHelper
 
 
    /*******************************************************************************
-    ** for tracking errors, we use primary keys.  add "made up" ones to records
-    ** if needed (e.g., insert use-case).
+    **
     *******************************************************************************/
-   private static Map<Serializable, QRecord> makeUpPrimaryKeysIfNeeded(List<QRecord> records, QTableMetaData table)
+   private static List<RecordSecurityLock> getRecordSecurityLocks(QTableMetaData table, Action action)
    {
-      String                     primaryKeyField   = table.getPrimaryKeyField();
-      Map<Serializable, QRecord> madeUpPrimaryKeys = new HashMap<>();
-      Integer                    madeUpPrimaryKey  = Integer.MIN_VALUE / 2;
-      for(QRecord record : records)
+      List<RecordSecurityLock> recordSecurityLocks = CollectionUtils.nonNullList(table.getRecordSecurityLocks());
+      List<RecordSecurityLock> locksToCheck        = new ArrayList<>();
+
+      recordSecurityLocks = switch(action)
       {
-         if(record.getValue(primaryKeyField) == null)
-         {
-            madeUpPrimaryKeys.put(madeUpPrimaryKey, record);
-            record.setValue(primaryKeyField, madeUpPrimaryKey);
-            madeUpPrimaryKey++;
-         }
-      }
-      return madeUpPrimaryKeys;
-   }
-
-
-
-   /*******************************************************************************
-    ** For a given table & action type, convert the table's record locks to a
-    ** MultiRecordSecurityLock, with only the appropriate lock-scopes being included
-    ** (e.g., read-locks for selects, write-locks for insert/update/delete).
-    *******************************************************************************/
-   static MultiRecordSecurityLock getRecordSecurityLocks(QTableMetaData table, Action action)
-   {
-      List<RecordSecurityLock> allLocksOnTable = CollectionUtils.nonNullList(table.getRecordSecurityLocks());
-      MultiRecordSecurityLock locksOfType = switch(action)
-      {
-         case INSERT, UPDATE, DELETE -> RecordSecurityLockFilters.filterForWriteLockTree(allLocksOnTable);
-         case SELECT -> RecordSecurityLockFilters.filterForReadLockTree(allLocksOnTable);
+         case INSERT, UPDATE, DELETE -> RecordSecurityLockFilters.filterForWriteLocks(recordSecurityLocks);
+         case SELECT -> RecordSecurityLockFilters.filterForReadLocks(recordSecurityLocks);
          default -> throw (new IllegalArgumentException("Unsupported action: " + action));
       };
 
-      if(action.equals(Action.UPDATE))
-      {
-         ////////////////////////////////////////////////////////////////////////////
-         // todo at some point this seemed right, but now it doesn't - needs work. //
-         ////////////////////////////////////////////////////////////////////////////
-         // ////////////////////////////////////////////////////////
-         // // when doing an update, convert all OR's to AND's... //
-         // ////////////////////////////////////////////////////////
-         // updateOperators(locksOfType, MultiRecordSecurityLock.BooleanOperator.AND);
-      }
-
       ////////////////////////////////////////
       // if there are no locks, just return //
       ////////////////////////////////////////
-      if(locksOfType == null || CollectionUtils.nullSafeIsEmpty(locksOfType.getLocks()))
+      if(CollectionUtils.nullSafeIsEmpty(recordSecurityLocks))
       {
          return (null);
       }
 
-      return (locksOfType);
-   }
-
-
-
-   /*******************************************************************************
-    ** for a full multi-lock tree, set all of the boolean operators to the specified one.
-    *******************************************************************************/
-   private static void updateOperators(MultiRecordSecurityLock multiLock, MultiRecordSecurityLock.BooleanOperator operator)
-   {
-      multiLock.setOperator(operator);
-      for(RecordSecurityLock childLock : multiLock.getLocks())
+      ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      // decide if any locks need checked - where one may not need checked if it has an all-access key, and the user has all-access //
+      ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      for(RecordSecurityLock recordSecurityLock : recordSecurityLocks)
       {
-         if(childLock instanceof MultiRecordSecurityLock childMultiLock)
+         QSecurityKeyType securityKeyType = QContext.getQInstance().getSecurityKeyType(recordSecurityLock.getSecurityKeyType());
+         if(StringUtils.hasContent(securityKeyType.getAllAccessKeyName()) && QContext.getQSession().hasSecurityKeyValue(securityKeyType.getAllAccessKeyName(), true, QFieldType.BOOLEAN))
          {
-            updateOperators(childMultiLock, operator);
+            LOG.trace("Session has " + securityKeyType.getAllAccessKeyName() + " - not checking this lock.");
+         }
+         else
+         {
+            locksToCheck.add(recordSecurityLock);
          }
       }
+
+      return (locksToCheck);
    }
 
 
@@ -444,17 +291,17 @@ public class ValidateRecordSecurityLockHelper
    /*******************************************************************************
     **
     *******************************************************************************/
-   public static List<QErrorMessage> validateRecordSecurityValue(QTableMetaData table, RecordSecurityLock recordSecurityLock, Serializable recordSecurityValue, QFieldType fieldType, Action action, Map<Serializable, QRecord> madeUpPrimaryKeys)
+   public static void validateRecordSecurityValue(QTableMetaData table, QRecord record, RecordSecurityLock recordSecurityLock, Serializable recordSecurityValue, QFieldType fieldType, Action action)
    {
-      if(recordSecurityValue == null || (madeUpPrimaryKeys != null && madeUpPrimaryKeys.containsKey(recordSecurityValue)))
+      if(recordSecurityValue == null)
       {
          /////////////////////////////////////////////////////////////////
          // handle null values - error if the NullValueBehavior is DENY //
          /////////////////////////////////////////////////////////////////
-         if(RecordSecurityLock.NullValueBehavior.DENY.equals(NullValueBehaviorUtil.getEffectiveNullValueBehavior(recordSecurityLock)))
+         if(RecordSecurityLock.NullValueBehavior.DENY.equals(recordSecurityLock.getNullValueBehavior()))
          {
             String lockLabel = CollectionUtils.nullSafeHasContents(recordSecurityLock.getJoinNameChain()) ? recordSecurityLock.getSecurityKeyType() : table.getField(recordSecurityLock.getFieldName()).getLabel();
-            return (List.of(new PermissionDeniedMessage("You do not have permission to " + action.name().toLowerCase() + " a record without a value in the field: " + lockLabel)));
+            record.addError(new PermissionDeniedMessage("You do not have permission to " + action.name().toLowerCase() + " a record without a value in the field: " + lockLabel));
          }
       }
       else
@@ -466,305 +313,15 @@ public class ValidateRecordSecurityLockHelper
                ///////////////////////////////////////////////////////////////////////////////////////////////
                // avoid telling the user a value from a foreign record that they didn't pass in themselves. //
                ///////////////////////////////////////////////////////////////////////////////////////////////
-               return (List.of(new PermissionDeniedMessage("You do not have permission to " + action.name().toLowerCase() + " this record.")));
+               record.addError(new PermissionDeniedMessage("You do not have permission to " + action.name().toLowerCase() + " this record."));
             }
             else
             {
                QFieldMetaData field = table.getField(recordSecurityLock.getFieldName());
-               return (List.of(new PermissionDeniedMessage("You do not have permission to " + action.name().toLowerCase() + " a record with a value of " + recordSecurityValue + " in the field: " + field.getLabel())));
+               record.addError(new PermissionDeniedMessage("You do not have permission to " + action.name().toLowerCase() + " a record with a value of " + recordSecurityValue + " in the field: " + field.getLabel()));
             }
          }
       }
-      return (Collections.emptyList());
-   }
-
-
-
-   /*******************************************************************************
-    ** Class to track errors that we're associating with a record.
-    **
-    ** More complex than it first seems to be needed, because as we're evaluating
-    ** locks, we might find some, but based on the boolean condition associated with
-    ** them, they might not actually be record-level errors.
-    **
-    ** e.g., two locks with an OR relationship - as long as one passes, the record
-    ** should have no errors.  And so-on through the tree of locks/multi-locks.
-    **
-    ** Stores the errors in a tree of ErrorTreeNode objects.
-    **
-    ** References into that tree are achieved via a List of Integer called "tree positions"
-    ** where each entry in the list denotes the index of the tree node at that level.
-    **
-    ** e.g., given this tree:
-    ** <pre>
-    **   A      B
-    **  / \    /|\
-    ** C   D  E F G
-    **     |
-    **     H
-    ** </pre>
-    **
-    ** The positions of each node would be:
-    ** <pre>
-    ** A: [0]
-    ** B: [1]
-    ** C: [0,0]
-    ** D: [0,1]
-    ** E: [1,0]
-    ** F: [1,1]
-    ** G: [1,2]
-    ** H: [0,1,0]
-    ** </pre>
-    *******************************************************************************/
-   static class RecordWithErrors
-   {
-      private QRecord       record;
-      private ErrorTreeNode errorTree;
-
-
-
-      /*******************************************************************************
-       ** Constructor
-       **
-       *******************************************************************************/
-      public RecordWithErrors(QRecord record)
-      {
-         this.record = record;
-      }
-
-
-
-      /*******************************************************************************
-       ** add a list of errors, for a given list of tree positions
-       *******************************************************************************/
-      public void addAll(List<QErrorMessage> recordErrors, List<Integer> treePositions)
-      {
-         if(errorTree == null)
-         {
-            errorTree = new ErrorTreeNode();
-         }
-
-         ErrorTreeNode node = errorTree;
-         for(Integer treePosition : treePositions)
-         {
-            if(node.children == null)
-            {
-               node.children = new ArrayList<>(treePosition);
-            }
-
-            while(treePosition >= node.children.size())
-            {
-               node.children.add(null);
-            }
-
-            if(node.children.get(treePosition) == null)
-            {
-               node.children.set(treePosition, new ErrorTreeNode());
-            }
-
-            node = node.children.get(treePosition);
-         }
-
-         if(node.errors == null)
-         {
-            node.errors = new ArrayList<>();
-         }
-         node.errors.addAll(recordErrors);
-      }
-
-
-
-      /*******************************************************************************
-       ** add a single error to a given tree-position
-       *******************************************************************************/
-      public void add(QErrorMessage error, List<Integer> treePositions)
-      {
-         addAll(List.of(error), treePositions);
-      }
-
-
-
-      /*******************************************************************************
-       ** after the tree of errors has been built - walk a lock-tree (locksToCheck)
-       ** and resolve boolean operations, to get a final list of errors (possibly empty)
-       ** to put on the record.
-       *******************************************************************************/
-      public void propagateErrorsToRecord(MultiRecordSecurityLock locksToCheck)
-      {
-         List<QErrorMessage> errors = recursivePropagation(locksToCheck, new ArrayList<>());
-
-         if(CollectionUtils.nullSafeHasContents(errors))
-         {
-            errors.forEach(e -> record.addError(e));
-         }
-      }
-
-
-
-      /*******************************************************************************
-       ** recursive implementation of the propagation method - e.g., walk tree applying
-       ** boolean logic.
-       *******************************************************************************/
-      private List<QErrorMessage> recursivePropagation(MultiRecordSecurityLock locksToCheck, List<Integer> treePositions)
-      {
-         //////////////////////////////////////////////////////////////////
-         // build a list of errors at this level (and deeper levels too) //
-         //////////////////////////////////////////////////////////////////
-         List<QErrorMessage> errorsFromThisLevel = new ArrayList<>();
-
-         int i = 0;
-         for(RecordSecurityLock lock : locksToCheck.getLocks())
-         {
-            List<QErrorMessage> errorsFromThisLock;
-
-            treePositions.add(i);
-            if(lock instanceof MultiRecordSecurityLock childMultiLock)
-            {
-               errorsFromThisLock = recursivePropagation(childMultiLock, treePositions);
-            }
-            else
-            {
-               errorsFromThisLock = getErrorsFromTree(treePositions);
-            }
-
-            errorsFromThisLevel.addAll(errorsFromThisLock);
-
-            treePositions.remove(treePositions.size() - 1);
-            i++;
-         }
-
-         if(MultiRecordSecurityLock.BooleanOperator.AND.equals(locksToCheck.getOperator()))
-         {
-            //////////////////////////////////////////////////////////////
-            // for an AND - if there were ANY errors, then return them. //
-            //////////////////////////////////////////////////////////////
-            if(!errorsFromThisLevel.isEmpty())
-            {
-               return (errorsFromThisLevel);
-            }
-         }
-         else // OR
-         {
-            //////////////////////////////////////////////////////////
-            // for an OR - only return if ALL conditions had errors //
-            //////////////////////////////////////////////////////////
-            if(errorsFromThisLevel.size() == locksToCheck.getLocks().size())
-            {
-               return (errorsFromThisLevel); // todo something smarter?
-            }
-         }
-
-         ///////////////////////////////////
-         // else - no errors - empty list //
-         ///////////////////////////////////
-         return Collections.emptyList();
-      }
-
-
-
-      /*******************************************************************************
-       **
-       *******************************************************************************/
-      private List<QErrorMessage> getErrorsFromTree(List<Integer> treePositions)
-      {
-         ErrorTreeNode node = errorTree;
-
-         for(Integer treePosition : treePositions)
-         {
-            if(node.children == null)
-            {
-               return Collections.emptyList();
-            }
-
-            if(treePosition >= node.children.size())
-            {
-               return Collections.emptyList();
-            }
-
-            if(node.children.get(treePosition) == null)
-            {
-               return Collections.emptyList();
-            }
-
-            node = node.children.get(treePosition);
-         }
-
-         if(node.errors == null)
-         {
-            return Collections.emptyList();
-         }
-
-         return node.errors;
-      }
-
-
-
-      /*******************************************************************************
-       **
-       *******************************************************************************/
-      @Override
-      public String toString()
-      {
-         try
-         {
-            return JsonUtils.toPrettyJson(this);
-         }
-         catch(Exception e)
-         {
-            return "error in toString";
-         }
-      }
-   }
-
-
-
-   /*******************************************************************************
-    ** tree node used by RecordWithErrors
-    *******************************************************************************/
-   static class ErrorTreeNode
-   {
-      private List<QErrorMessage>      errors;
-      private ArrayList<ErrorTreeNode> children;
-
-
-
-      /*******************************************************************************
-       **
-       *******************************************************************************/
-      @Override
-      public String toString()
-      {
-         try
-         {
-            return JsonUtils.toPrettyJson(this);
-         }
-         catch(Exception e)
-         {
-            return "error in toString";
-         }
-      }
-
-
-
-      /*******************************************************************************
-       ** Getter for errors - only here for Jackson/toString
-       **
-       *******************************************************************************/
-      public List<QErrorMessage> getErrors()
-      {
-         return errors;
-      }
-
-
-
-      /*******************************************************************************
-       ** Getter for children - only here for Jackson/toString
-       **
-       *******************************************************************************/
-      public ArrayList<ErrorTreeNode> getChildren()
-      {
-         return children;
-      }
    }
 
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/QPossibleValueTranslator.java

@@ -32,12 +32,13 @@ import java.util.List;
 import java.util.Map;
 import java.util.Objects;
 import java.util.Set;
+import com.kingsrook.qqq.backend.core.actions.QBackendTransaction;
 import com.kingsrook.qqq.backend.core.actions.customizers.QCodeLoader;
 import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
 import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.exceptions.QValueException;
 import com.kingsrook.qqq.backend.core.logging.QLogger;
-import com.kingsrook.qqq.backend.core.model.actions.tables.QueryHint;
+import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
@@ -77,6 +78,44 @@ public class QPossibleValueTranslator
 
    private int maxSizePerPvsCache = 50_000;
 
+   private Map<String, QBackendTransaction> transactionsPerTable = new HashMap<>();
+
+   // todo not commit - remove instance & session - use Context
+
+
+   boolean useTransactionsAsConnectionPool = false;
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private QBackendTransaction getTransaction(String tableName)
+   {
+      /////////////////////////////////////////////////////////////
+      // mmm, this does cut down on connections used -           //
+      // especially seems helpful in big exports.                //
+      // but, let's just start using connection pools instead... //
+      /////////////////////////////////////////////////////////////
+      if(useTransactionsAsConnectionPool)
+      {
+         try
+         {
+            if(!transactionsPerTable.containsKey(tableName))
+            {
+               transactionsPerTable.put(tableName, QBackendTransaction.openFor(new InsertInput(tableName)));
+            }
+
+            return (transactionsPerTable.get(tableName));
+         }
+         catch(Exception e)
+         {
+            LOG.warn("Error opening transaction for table", logPair("tableName", tableName));
+         }
+      }
+
+      return null;
+   }
 
 
    /*******************************************************************************
@@ -230,10 +269,6 @@ public class QPossibleValueTranslator
          {
             value = ValueUtils.getValueAsInteger(value);
          }
-         if(field.getType().equals(QFieldType.LONG) && !(value instanceof Long))
-         {
-            value = ValueUtils.getValueAsLong(value);
-         }
       }
       catch(QValueException e)
       {
@@ -331,14 +366,6 @@ public class QPossibleValueTranslator
     *******************************************************************************/
    private String translatePossibleValueCustom(Serializable value, QPossibleValueSource possibleValueSource)
    {
-      /////////////////////////////////
-      // null input gets null output //
-      /////////////////////////////////
-      if(value == null)
-      {
-         return (null);
-      }
-
       try
       {
          QCustomPossibleValueProvider customPossibleValueProvider = QCodeLoader.getCustomPossibleValueProvider(possibleValueSource);
@@ -382,6 +409,7 @@ public class QPossibleValueTranslator
    /*******************************************************************************
     **
     *******************************************************************************/
+   @SuppressWarnings("checkstyle:Indentation")
    private String doFormatPossibleValue(String formatString, List<String> valueFields, Object id, String label)
    {
       List<Object> values = new ArrayList<>();
@@ -520,48 +548,21 @@ public class QPossibleValueTranslator
     *******************************************************************************/
    private void primePvsCache(String tableName, List<QPossibleValueSource> possibleValueSources, Collection<Serializable> values)
    {
-      String idField = null;
       for(QPossibleValueSource possibleValueSource : possibleValueSources)
       {
          possibleValueCache.putIfAbsent(possibleValueSource.getName(), new HashMap<>());
-         String thisPvsIdField;
-         if(StringUtils.hasContent(possibleValueSource.getOverrideIdField()))
-         {
-            thisPvsIdField = possibleValueSource.getOverrideIdField();
-         }
-         else
-         {
-            thisPvsIdField = QContext.getQInstance().getTable(tableName).getPrimaryKeyField();
-         }
-
-         if(idField == null)
-         {
-            idField = thisPvsIdField;
-         }
-         else
-         {
-            /////////////////////////////////////////////////////////////////////////////////////////////////////////////
-            // does this ever happen?  maybe not... because, like, the list of values probably wouldn't make sense for //
-            // more than one field in the table...                                                                     //
-            /////////////////////////////////////////////////////////////////////////////////////////////////////////////
-            if(!idField.equals(thisPvsIdField))
-            {
-               for(QPossibleValueSource valueSource : possibleValueSources)
-               {
-                  primePvsCache(tableName, List.of(valueSource), values);
-               }
-            }
-         }
       }
 
       try
       {
+         String primaryKeyField = QContext.getQInstance().getTable(tableName).getPrimaryKeyField();
+
          for(List<Serializable> page : CollectionUtils.getPages(values, 1000))
          {
             QueryInput queryInput = new QueryInput();
             queryInput.setTableName(tableName);
-            queryInput.setFilter(new QQueryFilter().withCriteria(new QFilterCriteria(idField, QCriteriaOperator.IN, page)));
-            queryInput.hasQueryHint(QueryHint.MAY_USE_READ_ONLY_BACKEND);
+            queryInput.setFilter(new QQueryFilter().withCriteria(new QFilterCriteria(primaryKeyField, QCriteriaOperator.IN, page)));
+            queryInput.setTransaction(getTransaction(tableName));
 
             /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
             // when querying for possible values, we do want to generate their display values, which makes record labels, which are usually used as PVS labels //
@@ -605,7 +606,7 @@ public class QPossibleValueTranslator
             ///////////////////////////////////////////////////////////////////////////////////
             for(QRecord record : queryOutput.getRecords())
             {
-               Serializable pkeyValue = record.getValue(idField);
+               Serializable pkeyValue = record.getValue(primaryKeyField);
                for(QPossibleValueSource possibleValueSource : possibleValueSources)
                {
                   QPossibleValue<?> possibleValue = new QPossibleValue<>(pkeyValue, record.getRecordLabel());

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/QValueFormatter.java

@@ -28,6 +28,7 @@ import java.time.LocalDateTime;
 import java.time.LocalTime;
 import java.time.ZonedDateTime;
 import java.time.format.DateTimeFormatter;
+import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
@@ -363,9 +364,7 @@ public class QValueFormatter
             }
          }
 
-         ValueBehaviorApplier.applyFieldBehaviors(ValueBehaviorApplier.Action.FORMATTING, QContext.getQInstance(), table, records, null);
-
-         setDisplayValuesInRecord(table, fieldMap, record, true);
+         setDisplayValuesInRecord(fieldMap, record);
          record.setRecordLabel(formatRecordLabel(table, record));
       }
    }
@@ -375,49 +374,61 @@ public class QValueFormatter
    /*******************************************************************************
     ** For a list of records, set their recordLabels and display values
     *******************************************************************************/
-   public static void setDisplayValuesInRecords(QTableMetaData table, Map<String, QFieldMetaData> fields, List<QRecord> records)
+   public static void setDisplayValuesInRecords(Collection<QFieldMetaData> fields, List<QRecord> records)
    {
       if(records == null)
       {
          return;
       }
 
-      if(table != null)
+      for(QRecord record : records)
       {
-         ValueBehaviorApplier.applyFieldBehaviors(ValueBehaviorApplier.Action.FORMATTING, QContext.getQInstance(), table, records, null);
+         setDisplayValuesInRecord(fields, record);
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** For a list of records, set their recordLabels and display values
+    *******************************************************************************/
+   public static void setDisplayValuesInRecords(Map<String, QFieldMetaData> fields, List<QRecord> records)
+   {
+      if(records == null)
+      {
+         return;
       }
 
       for(QRecord record : records)
       {
-         setDisplayValuesInRecord(table, fields, record, true);
+         setDisplayValuesInRecord(fields, record);
       }
    }
 
 
 
    /*******************************************************************************
-    ** For a single record, set its display values - public version of this.
+    ** For a list of records, set their display values
     *******************************************************************************/
-   public static void setDisplayValuesInRecord(QTableMetaData table, Map<String, QFieldMetaData> fields, QRecord record)
+   public static void setDisplayValuesInRecord(Collection<QFieldMetaData> fields, QRecord record)
    {
-      setDisplayValuesInRecord(table, fields, record, false);
-   }
-
-
-   /*******************************************************************************
-    ** For a single record, set its display values - where caller (meant to stay private)
-    ** can specify if they've already done fieldBehaviors (to avoid re-doing).
-    *******************************************************************************/
-   private static void setDisplayValuesInRecord(QTableMetaData table, Map<String, QFieldMetaData> fields, QRecord record, boolean alreadyAppliedFieldDisplayBehaviors)
-   {
-      if(!alreadyAppliedFieldDisplayBehaviors)
+      for(QFieldMetaData field : fields)
       {
-         if(table != null)
+         if(record.getDisplayValue(field.getName()) == null)
          {
-            ValueBehaviorApplier.applyFieldBehaviors(ValueBehaviorApplier.Action.FORMATTING, QContext.getQInstance(), table, List.of(record), null);
+            String formattedValue = formatValue(field, record.getValue(field.getName()));
+            record.setDisplayValue(field.getName(), formattedValue);
          }
       }
+   }
 
+
+
+   /*******************************************************************************
+    ** For a list of records, set their display values
+    *******************************************************************************/
+   public static void setDisplayValuesInRecord(Map<String, QFieldMetaData> fields, QRecord record)
+   {
       for(Map.Entry<String, QFieldMetaData> entry : fields.entrySet())
       {
          String         fieldName = entry.getKey();
@@ -479,13 +490,6 @@ public class QValueFormatter
             {
                adornmentValues = fileDownloadAdornment.get().getValues();
             }
-            else
-            {
-               ///////////////////////////////////////////////////////
-               // don't change blobs unless they are file-downloads //
-               ///////////////////////////////////////////////////////
-               continue;
-            }
 
             String fileNameField    = ValueUtils.getValueAsString(adornmentValues.get(AdornmentType.FileDownloadValues.FILE_NAME_FIELD));
             String fileNameFormat   = ValueUtils.getValueAsString(adornmentValues.get(AdornmentType.FileDownloadValues.FILE_NAME_FORMAT));

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/SearchPossibleValueSourceAction.java

@@ -275,19 +275,8 @@ public class SearchPossibleValueSourceAction
 
       queryInput.setFilter(queryFilter);
 
-      QueryOutput queryOutput = new QueryAction().execute(queryInput);
-
-      String fieldName;
-      if(StringUtils.hasContent(possibleValueSource.getOverrideIdField()))
-      {
-         fieldName = possibleValueSource.getOverrideIdField();
-      }
-      else
-      {
-         fieldName = table.getPrimaryKeyField();
-      }
-
-      List<Serializable> ids = queryOutput.getRecords().stream().map(r -> r.getValue(fieldName)).toList();
+      QueryOutput             queryOutput     = new QueryAction().execute(queryInput);
+      List<Serializable>      ids             = queryOutput.getRecords().stream().map(r -> r.getValue(table.getPrimaryKeyField())).toList();
       List<QPossibleValue<?>> qPossibleValues = possibleValueTranslator.buildTranslatedPossibleValueList(possibleValueSource, ids);
       output.setResults(qPossibleValues);
 

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/ValueBehaviorApplier.java

@@ -23,18 +23,16 @@ package com.kingsrook.qqq.backend.core.actions.values;
 
 
 import java.util.List;
-import java.util.Set;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
 import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.FieldBehavior;
-import com.kingsrook.qqq.backend.core.model.metadata.fields.FieldDisplayBehavior;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
 
 
 /*******************************************************************************
- ** Utility class to apply value behaviors to records.
+ ** Utility class to apply value behaviors to records.  
  *******************************************************************************/
 public class ValueBehaviorApplier
 {
@@ -45,8 +43,7 @@ public class ValueBehaviorApplier
    public enum Action
    {
       INSERT,
-      UPDATE,
-      FORMATTING
+      UPDATE
    }
 
 
@@ -54,7 +51,7 @@ public class ValueBehaviorApplier
    /*******************************************************************************
     **
     *******************************************************************************/
-   public static void applyFieldBehaviors(Action action, QInstance instance, QTableMetaData table, List<QRecord> recordList, Set<FieldBehavior<?>> behaviorsToOmit)
+   public static void applyFieldBehaviors(Action action, QInstance instance, QTableMetaData table, List<QRecord> recordList)
    {
       if(CollectionUtils.nullSafeIsEmpty(recordList))
       {
@@ -65,34 +62,7 @@ public class ValueBehaviorApplier
       {
          for(FieldBehavior<?> fieldBehavior : CollectionUtils.nonNullCollection(field.getBehaviors()))
          {
-            boolean applyBehavior = true;
-            if(behaviorsToOmit != null && behaviorsToOmit.contains(fieldBehavior))
-            {
-               /////////////////////////////////////////////////////////////////////////////////////////
-               // if we're given a set of behaviors to omit, and this behavior is in there, then skip //
-               /////////////////////////////////////////////////////////////////////////////////////////
-               applyBehavior = false;
-            }
-
-            if(Action.FORMATTING == action && !(fieldBehavior instanceof FieldDisplayBehavior<?>))
-            {
-               ////////////////////////////////////////////////////////////////////////////////////////////////
-               // for the formatting action, do not apply the behavior unless it is a field-display-behavior //
-               ////////////////////////////////////////////////////////////////////////////////////////////////
-               applyBehavior = false;
-            }
-            else if(Action.FORMATTING != action && fieldBehavior instanceof FieldDisplayBehavior<?>)
-            {
-               /////////////////////////////////////////////////////////////////////////////////////////////
-               // for non-formatting actions, do not apply the behavior IF it is a field-display-behavior //
-               /////////////////////////////////////////////////////////////////////////////////////////////
-               applyBehavior = false;
-            }
-
-            if(applyBehavior)
-            {
-               fieldBehavior.apply(action, recordList, instance, table, field);
-            }
+            fieldBehavior.apply(action, recordList, instance, table, field);
          }
       }
    }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/context/CapturedContext.java

@@ -34,11 +34,5 @@ import com.kingsrook.qqq.backend.core.model.session.QSession;
  *******************************************************************************/
 public record CapturedContext(QInstance qInstance, QSession qSession, QBackendTransaction qBackendTransaction, Stack<AbstractActionInput> actionStack)
 {
-   /*******************************************************************************
-    ** Simpler constructor
-    *******************************************************************************/
-   public CapturedContext(QInstance qInstance, QSession qSession)
-   {
-      this(qInstance, qSession, null, null);
-   }
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/context/QContext.java

@@ -22,9 +22,6 @@
 package com.kingsrook.qqq.backend.core.context;
 
 
-import java.io.Serializable;
-import java.util.HashMap;
-import java.util.Map;
 import java.util.Optional;
 import java.util.Stack;
 import com.kingsrook.qqq.backend.core.actions.QBackendTransaction;
@@ -34,8 +31,6 @@ import com.kingsrook.qqq.backend.core.logging.QLogger;
 import com.kingsrook.qqq.backend.core.model.actions.AbstractActionInput;
 import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
 import com.kingsrook.qqq.backend.core.model.session.QSession;
-import com.kingsrook.qqq.backend.core.utils.lambdas.UnsafeVoidVoidMethod;
-import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
 
 
 /*******************************************************************************
@@ -52,8 +47,6 @@ public class QContext
    private static ThreadLocal<QBackendTransaction>        qBackendTransactionThreadLocal = new ThreadLocal<>();
    private static ThreadLocal<Stack<AbstractActionInput>> actionStackThreadLocal         = new ThreadLocal<>();
 
-   private static ThreadLocal<Map<String, Serializable>> objectsThreadLocal = new ThreadLocal<>();
-
 
 
    /*******************************************************************************
@@ -107,25 +100,6 @@ public class QContext
 
 
 
-   /*******************************************************************************
-    **
-    *******************************************************************************/
-   public static <T extends Throwable> void withTemporaryContext(CapturedContext context, UnsafeVoidVoidMethod<T> method) throws T
-   {
-      CapturedContext originalContext = QContext.capture();
-      try
-      {
-         QContext.init(context);
-         method.run();
-      }
-      finally
-      {
-         QContext.init(originalContext);
-      }
-   }
-
-
-
    /*******************************************************************************
     ** Init a new thread with the context captured from a different thread.  e.g.,
     ** when starting some async task.
@@ -158,7 +132,6 @@ public class QContext
       qSessionThreadLocal.remove();
       qBackendTransactionThreadLocal.remove();
       actionStackThreadLocal.remove();
-      objectsThreadLocal.remove();
    }
 
 
@@ -286,94 +259,4 @@ public class QContext
 
       return (Optional.of(actionStackThreadLocal.get().get(0)));
    }
-
-
-
-   /*******************************************************************************
-    ** get one named object from the Context for the current thread.  may return null.
-    *******************************************************************************/
-   public static Serializable getObject(String key)
-   {
-      if(objectsThreadLocal.get() == null)
-      {
-         return null;
-      }
-      return objectsThreadLocal.get().get(key);
-   }
-
-
-
-   /*******************************************************************************
-    ** get one named object from the Context for the current thread, cast to the
-    ** specified type if possible.  if not found, or wrong type, empty is returned.
-    *******************************************************************************/
-   public static <T extends Serializable> Optional<T> getObject(String key, Class<T> type)
-   {
-      Serializable object = getObject(key);
-
-      if(type.isInstance(object))
-      {
-         return Optional.of(type.cast(object));
-      }
-      else if(object == null)
-      {
-         return Optional.empty();
-      }
-      else
-      {
-         LOG.warn("Unexpected type of object found in session under key [" + key + "]",
-            logPair("expectedType", type.getName()),
-            logPair("actualType", object.getClass().getName())
-         );
-         return Optional.empty();
-      }
-   }
-
-
-
-   /*******************************************************************************
-    ** put a named object into the Context for the current thread.
-    *******************************************************************************/
-   public static void setObject(String key, Serializable object)
-   {
-      if(objectsThreadLocal.get() == null)
-      {
-         objectsThreadLocal.set(new HashMap<>());
-      }
-      objectsThreadLocal.get().put(key, object);
-   }
-
-
-
-   /*******************************************************************************
-    ** remove a named object from the Context of the current thread.
-    *******************************************************************************/
-   public static void removeObject(String key)
-   {
-      if(objectsThreadLocal.get() != null)
-      {
-         objectsThreadLocal.get().remove(key);
-      }
-   }
-
-
-
-   /*******************************************************************************
-    ** get the full map of named objects for the current thread (possibly null).
-    *******************************************************************************/
-   public static Map<String, Serializable> getObjects()
-   {
-      return objectsThreadLocal.get();
-   }
-
-
-
-   /*******************************************************************************
-    ** fully replace the map of named objets for the current thread.
-    *******************************************************************************/
-   public static void setObjects(Map<String, Serializable> objects)
-   {
-      objectsThreadLocal.set(objects);
-   }
-
 }