ES-52: updated UpdateAction to act same as insert or gets for 'retryable server errors'

Add isEnabled method to meta-data producers; Put interface on top of …

.circleci/config.yml

@@ -102,14 +102,14 @@ jobs:
   mvn_test:
     executor: localstack/default
     steps:
-      - localstack/startup
+      ## - localstack/startup
       - install_java17
       - mvn_verify
 
   mvn_deploy:
     executor: localstack/default
     steps:
-      - localstack/startup
+      ## - localstack/startup
       - install_java17
       - mvn_verify
       - mvn_jar_deploy

docs/Introduction.adoc

@@ -0,0 +1,8 @@
+= Introduction
+
+QQQ is ...
+
+- Framework
+- Declarative
+- Easy thing easy; Hard thing possible
+- Customizable

docs/actions/QueryAction.adoc

@@ -0,0 +1,179 @@
+== QueryAction
+include::../variables.adoc[]
+
+The `*QueryAction*` is the basic action that is used to get records from a {link-table}.
+In SQL/RDBMS terms, it is analogous to a `SELECT` statement, where 0 or more records may be found and returned.
+
+=== Examples
+==== Basic Form
+[source,java]
+----
+QueryInput input = new QueryInput(qInstance);
+input.setSession(session);
+input.setTableName("orders");
+input.setFilter(new QQueryFilter()
+   .withCriteria(new QFilterCriteria("total", GREATER_THAN, new BigDecimal("3.50")))
+   .withOrderBy(new QFilterOrderBy("orderDate", false))
+);
+QueryOutput output = new QueryAction.execute(input);
+List<QRecord> records = output.getRecords();
+----
+
+=== QueryInput
+* `table` - *String, Required* - Name of the table being queried against.
+* `filter` - *QQueryFilter object* - Specification for what records should be returned, based on *QFilterCriteria* objects, and how they should be sorted, based on *QFilterOrderBy* objects.
+* `skip` - *Integer* - Optional number of records to be skipped at the beginning of the result set.
+e.g., for implementing pagination.
+* `limit` - *Integer* - Optional maximum number of records to be returned by the query.
+* `transaction` - *QBackendTransaction object* - Optional transaction object.
+** Behavior for this object is backend-dependant.
+In an RDBMS backend, this object is generally needed if you want your query to see data that may have been modified within the same transaction.
+* `recordPipe` - *RecordPipe object* - Optional object that records are placed into, for asynchronous processing.
+** If a *recordPipe* is used, then records cannot be retrieved from the *QueryOutput*.
+Rather, such records must be read from the pipe's `consumeAvailableRecords()` method.
+** A *recordPipe* should only be used when a *QueryAction* is running in a separate Thread from the record's consumer.
+* `shouldTranslatePossibleValues` - *boolean, default: false* - Controls whether any fields in the table with a *possibleValueSource* assigned to them should have those possible values looked up
+(e.g., to provide text translations in the generated records' `displayValues` map).
+** For example, if running a query to present results to a user, this would generally need to be *true*.
+But if running a query to provide data as part of a process, then this can generally be left as *false*.
+* `shouldGenerateDisplayValues` - *boolean, default: false* - Controls whether if field level *displayFormats* should be used to populate the generated records' `displayValues` map.
+** For example, if running a query to present results to a user, this would generally need to be *true*.
+But if running a query to provide data as part of a process, then this can generally be left as *false*.
+* `queryJoins` - *List of QueryJoin objects* - Optional list of tables to be joined with the main *table* specified in the *QueryInput*.
+See QueryJoin below for further details.
+
+==== QQueryFilter
+A key component of *QueryInput*, a *QQueryFilter* defines both what records should be included in a query's results (e.g., an SQL `WHERE`), as well as how those results should be sorted (SQL `ORDER BY`).
+
+* `criteria` - *List of QFilterCriteria* - Individual conditions or clauses to filter records.
+They are combined using the *booleanOperator* specified in the *QQueryFilter*.  See below for further details.
+* `orderBys` - *List of QFilterOrderBy* - List of fields (and directions) to control the sorting of query results.
+In general, multiple *orderBys* can be given (depending on backend implementations).
+* `booleanOperator` - *Enum of AND, OR, default: AND* - Specifies the logical joining operator used among individual criteria.
+* `subFilters` - *List of QQueryFilter* - To build arbitrarily complex queries, with nested boolean logic, 0 or more *subFilters* may be provided.
+** Each *subFilter* can include its own additional *subFilters*.
+** Each *subFilter* can specify a different *booleanOperator*.
+** For example, consider the following *QQueryFilter*, that uses two *subFilters*, and a mix of *booleanOperators*
+
+[source,java]
+----
+ queryInput.setFilter(new QQueryFilter()
+    .withBooleanOperator(OR)
+    .withSubFilters(List.of(
+       new QQueryFilter().withBooleanOperator(AND)
+          .withCriteria(new QFilterCriteria("firstName", EQUALS, "James"))
+          .withCriteria(new QFilterCriteria("lastName", EQUALS, "Maes")),
+       new QQueryFilter().withBooleanOperator(AND)
+          .withCriteria(new QFilterCriteria("firstName", EQUALS, "Darin"))
+          .withCriteria(new QFilterCriteria("lastName", EQUALS, "Kelkhoff"))
+    )));
+
+// which would generate the following WHERE clause in an RDBMS backend:
+   WHERE (first_name='James' AND last_name='Maes') OR (first_name='Darin' AND last_name='Kelkhoff')
+----
+
+===== QFilterCriteria
+* `fieldName` - *String, required* - Reference to a field on the table being queried.
+** Or, in the case of a query with *queryJoins*, a qualified name of a field from a join-table (where the qualifier would be the joined table's name or alias, followed by a dot)
+*** For example:  `orderLine.sku` or `orderBillToCustomer.firstName`
+* `operator` - *Enum of QCriteriaOperator, required* - Comparison operation to be applied to the field specified as *fieldName* and the *values* or *otherFieldName*.
+** e.g., `EQUALS`, `NOT_IN`, `GREATER_THAN`, `BETWEEN`, `IS_BLANK`, etc.
+* `values` - *List of values, conditional* - Provides the value(s) that the field is compared against.
+The number of values (0, 1, 2, or more) be driven based on the *operator* being used.
+If an *otherFieldName* is given, and the *operator* expects 1 value, then *values* is ignored, and *otherFieldName* is used.
+* `otherFieldName` - *String, conditional* - Specifies that the *fieldName* should be compared against another field in the records, rather than the values in the *values* property.
+Only used for *operators* that expect 1 value (e.g., `EQUALS` or `LESS_THAN_OR_EQUALS` - not `IS_NOT_BLANK` or `IN`).
+
+QFilterCriteria definition examples:
+[source,java]
+----
+// one-liners, via constructors that take (List<Serializable> values) or (Serializable... values) in 3rd position
+new QFilterCriteria("id", IN, List.of(1, 2, 3))
+new QFilterCriteria("name", IS_BLANK)
+new QFilterCriteria("orderNo", IN, orderNoList)
+new QFilterCriteria("state", EQUALS, "MO");
+
+// long-form, with fluent setters
+new QFilterCriteria()
+   .withFieldName("quantity")
+   .withOpeartor(QCriteriaOperator.GREATER_THAN)
+   .withValues(List.of(47));
+
+// to use otherFieldName, long-form must be used
+new QFilterCriteria()
+   .withFieldName("firstName")
+   .withOpeartor(QCriteriaOperator.EQUALS)
+   .withOtherFieldName("lastName");
+
+// using otherFieldName to build a criterion that looks at two fields from join tables
+new QFilterCriteria()
+   .withFieldName("billToCustomer.lastName")
+   .withOpeartor(QCriteriaOperator.NOT_EQUALS)
+   .withOtherFieldName("shipToCustomer.lastName");
+
+----
+
+===== QFilterOrderBy
+* `fieldName` - *String, required* - Reference to a field on the table being queried.
+** Or, in the case of a query with *queryJoins*, a qualified name of a field from a join-table (where the qualifier would be the joined table's name or alias, followed by a dot)
+* `isAscending` - *boolean, default: true* - Specify if the sort is ascending or descending.
+
+QFilterCriteria definition examples:
+
+[source,java]
+----
+// short-form, via constructors
+new QFilterOrderBy("id") // isAscending defaults to true.
+new QFilterOrderBy("name", false)
+
+// long-form, with fluent setters
+new QFilterOrderBy()
+   .withFieldName("birthDate")
+   .withIsAscending(true);
+----
+
+==== QueryJoin
+* `joinTable` - *String, required* - Name of the table that is being joined in to the existing query.
+** Will be inferred from *joinMetaData*, if *joinTable* is not set when *joinMetaData* gets set.
+* `baseTableOrAlias` - *String, required* - Name of a table (or an alias) already defined in the query, to which the *joinTable* will be joined.
+** Will be inferred from *joinMetaData*, if *baseTableOrAlias* is not set when *joinMetaData* gets set (which will only use the leftTableName from the joinMetaData - never an alias).
+* `joinMetaData` - *QJoinMetaData object* - Optional specification of a {link-join} in the current QInstance.
+If not set, will be looked up at runtime based on *baseTableOrAlias* and *joinTable*.
+** If set before *baseTableOrAlias* and *joinTable*, then they will be set based on the *leftTable* and *rightTable* in this object.
+* `alias` - *String* - Optional (unless multiple instances of the same table are being joined together, when it becomes required).
+Behavior based on SQL `FROM` clause aliases.
+If given, must be used as the part before the dot in field name specifications throughout the rest of the query input.
+* `select` - *boolean, default: false* - Specify whether fields from the *rightTable* should be selected by the query.
+If *true*, then the `QRecord` objects returned by this query will have values with corresponding to the (table-or-alias `.` field-name) form.
+* `type` - *Enum of INNER, LEFT, RIGHT, FULL, default: INNER* - specifies the SQL-style type of join being performed.
+
+QueryJoin definition examples:
+
+[source,java]
+----
+// selecting from an "orderLine" table - then join to its corresponding "order" table
+queryInput.withTableName("orderLine");
+queryInput.withQueryJoin(new QueryJoin("order").withSelect(true));
+...
+queryOutput.getRecords().get(0).getValueBigDecimal("order.grandTotal");
+
+// given an "order" table with 2 foreign keys to a customer table (billToCustomerId and shipToCustomerId)
+// Note, we must supply the JoinMetaData to the QueryJoin, to drive what fields to join on in each case.
+queryInput.withTableName("order");
+queryInput.withQueryJoins(List.of(
+   new QueryJoin(instance.getJoin("orderJoinShipToCustomer")
+       .withAlias("shipToCustomer")
+       .withSelect(true)),
+   new QueryJoin(instance.getJoin("orderJoinBillToCustomer")
+       .withAlias("billToCustomer")
+       .withSelect(true))));
+...
+record.getValueString("billToCustomer.firstName")
+   + " placed an order for "
+   + record.getValueString("shipToCustomer.firstName")
+
+----
+
+=== QueryOutput
+* `records` - *List of QRecord* - List of 0 or more records that match the query filter.
+** _Note:  If a *recordPipe* was supplied to the QueryInput, then calling `queryOutput.getRecords()` will result in an `IllegalStateException` being thrown - as the records were placed into the pipe as they were fetched, and cannot all be accessed as a single list._

docs/actions/RenderTemplateAction.adoc

@@ -0,0 +1,33 @@
+== RenderTemplateAction
+include::../variables.adoc[]
+
+The `*RenderTemplateAction*` performs the job of taking a template - that is, a string of code, in a templating language, such as https://velocity.apache.org/engine/1.7/user-guide.html[Velocity], and merging it with a set of data (known as a context), to produce some using-facing output, such as a String of HTML.
+
+=== Examples
+==== Canonical Form
+[source,java]
+----
+RenderTemplateInput input = new RenderTemplateInput(qInstance);
+input.setSession(session);
+input.setCode("Hello, ${name}");
+input.setTemplateType(TemplateType.VELOCITY);
+input.setContext(Map.of("name", "Darin"));
+RenderTemplateOutput output = new RenderTemplateAction.execute(input);
+String result = output.getResult();
+assertEquals("Hello, Darin", result);
+----
+
+==== Convenient Form
+[source,java]
+----
+String result = RenderTemplateAction.renderVelocity(input, Map.of("name", "Darin"), "Hello, ${name}");
+assertEquals("Hello, Darin", result);
+----
+
+=== RenderTemplateInput
+* `code` - *String, Required* - String of template code to be rendered, in the templating language specified by the `type` parameter.
+* `type` - *Enum of VELOCITY, Required* - Specifies the language of the template code.
+* `context` - *Map of String → Object* - Data to be made available to the template during rendering.
+
+=== RenderTemplateOutput
+* `result` - *String* - Result of rendering the input template and context.

docs/docinfo.html

@@ -0,0 +1,27 @@
+<!--
+  ~ QQQ - Low-code Application Framework for Engineers.
+  ~ Copyright (C) 2021-2022.  Kingsrook, LLC
+  ~ 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+  ~ contact@kingsrook.com
+  ~ https://github.com/Kingsrook/
+  ~
+  ~ This program is free software: you can redistribute it and/or modify
+  ~ it under the terms of the GNU Affero General Public License as
+  ~ published by the Free Software Foundation, either version 3 of the
+  ~ License, or (at your option) any later version.
+  ~
+  ~ This program is distributed in the hope that it will be useful,
+  ~ but WITHOUT ANY WARRANTY; without even the implied warranty of
+  ~ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+  ~ GNU Affero General Public License for more details.
+  ~
+  ~ You should have received a copy of the GNU Affero General Public License
+  ~ along with this program.  If not, see <https://www.gnu.org/licenses/>.
+  -->
+
+<style>
+   pre
+   {
+      white-space: pre;
+   }
+</style>

docs/index.adoc

@@ -0,0 +1,17 @@
+= QQQ
+:doctype: book
+:toc: left
+:source-highlighter: coderay
+
+include::Introduction.adoc[leveloffset=+1]
+
+== Meta Data
+include::metaData/Tables.adoc[leveloffset=+1]
+''''
+include::metaData/Reports.adoc[leveloffset=+1]
+
+== Actions
+include::actions/QueryAction.adoc[leveloffset=+1]
+''''
+include::actions/RenderTemplateAction.adoc[leveloffset=+1]
+''''

docs/metaData/Fields.adoc

@@ -0,0 +1,24 @@
+== QQQ Fields
+include::../variables.adoc[]
+
+QQQ Fields define
+
+=== QFieldMetaData
+*QFieldMetaData Properties:*
+
+* `name` - *String, Required* - Unique name for the field within its container (table, process, etc).
+* `label` - *String* - User-facing label for the field, presented in User Interfaces.
+* `type` - *enum of QFieldType, Required* - Data type for values in the field.
+* `backendName` - *String* - Name of the field within its backend.
+** For example, in an RDBMS-backed table, a field's `name` may be written in camel case, but its `backendName` written with underscores.
+* `isRequired` - *boolean, default false* - Indicator that a value is required in this field.
+* `isEditable` - *boolean, default true* - Indicator that users may edit values in this field.
+* `displayFormat` - *String, default `%s`* - Java Format Specifier string, used to format values in the field for display in user interfaces.
+Used to set values in the `displayValues` map within a `QRecord`.
+** Recommended values for `displayFormat` come from the `DisplayFormat` interface, such as `DisplayFormat.CURRENCY`, `DisplayFormat.COMMAS`, or `DisplayFormat.DECIMAL2_COMMAS`.
+* `defaultValue` - Value to use for the field if no other value is given.  Type is based on the field's `type`.
+* `possibleValueSourceName` - *String* - Reference to a {link-pvs} to be used for this field.
+Values in this field should correspond to ids from the referenced Possible Value Source.
+* `maxLength` - *Integer* - Maximum length (number of characters) allowed for values in this field.
+Only applicable for fields with `type=STRING`.
+* `

docs/metaData/Reports.adoc

@@ -0,0 +1,173 @@
+== QQQ Reports
+include::../variables.adoc[]
+
+QQQ can generate reports based on {link-tables} defined within a QQQ Instance.
+Users can run reports, providing input values.
+Alternatively, application code can run reports as needed, supplying input values.
+
+=== QReportMetaData
+Reports are defined in a QQQ Instance with a `*QReportMetaData*` object.
+Reports are defined in terms of their sources of data (`QReportDataSource`), and their view(s) of that data (`QReportView`).
+
+*QReportMetaData Properties:*
+
+* `name` - *String, Required* - Unique name for the report within the QQQ Instance.
+* `label` - *String* - User-facing label for the report, presented in User Interfaces.
+Inferred from `name` if not set.
+* `processName` - *String* - Name of a {link-process} used to run the report in a User Interface.
+* `inputFields` - *List of {link-field}* - Optional list of fields used as input to the report.
+** The values in these fields can be used via the syntax `${input.NAME}`, where `NAME` is the `name` attribute of the `inputField`.
+** For example:
+
+[source,java]
+----
+// given this inputField:
+new QFieldMetaData("storeId", QFieldType.INTEGER)
+
+// its run-time value can be accessed, e.g., in a query filter under a data source:
+new QFilterCriteria("storeId", QCriteriaOperator.EQUALS, List.of("${input.storeId}"))
+
+// or in a report view's title or field formulas:
+.withTitleFields(List.of("${input.storeId}"))
+new QReportField().withName("storeId").withFormula("${input.storeId}")
+----
+
+* `dataSources` - *List of QReportDataSource, Required* - Definitions of the sources of data for the report.
+At least one is required.
+
+==== QReportDataSource
+Data sources for QQQ Reports can either reference {link-tables} within the QQQ Instance, or they can provide custom code in the form of a `CodeReference` to a  `Supplier`, for use cases such as a static data tab in an Excel report.
+
+*QReportDataSource Properties:*
+
+* `name` - *String, Required* - Unique name for the data source within its containing Report.
+* `sourceTable` - *String, Conditional* - Reference to a {link-table} in the QQQ Instance, which the data source queries data from.
+* `queryFilter` - *QQueryFilter* - If a `sourceTable` is defined, then the filter specified here is used to filter and sort the records queried from that table when generating the report.
+* `staticDataSupplier` - *QCodeReference, Conditional* - Reference to custom code which can be used to supply the data for the data source, as an alternative to querying a `sourceTable`.
+** Must be a `JAVA` code type
+** Must be a `REPORT_STATIC_DATA_SUPPLIER` code usage.
+** The referenced class must implement the interface: `Supplier<List<List<Serializable>>>`.
+
+==== QReportView
+Report Views control how the source data for a report is organized and presented to the user in the output report file.
+If a DataSource describes the rows for a report (e.g., what table provides what records), then a View may be thought of as describing the columns in the report.
+A single report can have multiple views, specifically, for the use-case where an Excel file is being generated, in which case each View creates a tab or sheet within the `xlsx` file.
+
+*QReportView Properties:*
+
+* `name` - *String, Required* - Unique name for the view within its containing Report.
+* `label` - *String* - Used as a sheet (tab) label in Excel formatted reports.
+* `type` - *enum of TABLE, SUMMARY, PIVOT. Required* - Defines the type of view being defined.
+** *TABLE* views are a simple listing of the records from the data source.
+** *SUMMARY* views are essentially pre-computed Pivot Tables.
+That is to say, the aggregation done by a Pivot Table in a spreadsheet file is done by QQQ while generating the report.
+In this way, a non-spreadsheet report (e.g., PDF or CSV) can have summarized data, as though it were a Pivot Table in a live spreadsheet.
+** *PIVOT* views produce actual Pivot Tables, and are only supported in Excel files _(and are not supported at the time of this writing)_.
+* `dataSourceName` - *String, Required* - Reference to a DataSource within the report, that is used to provide the rows for the view.
+* `varianceDataSourceName` - *String* - Optional reference to a second DataSource within the report, that is used in `*SUMMARY*` type views for computing variances.
+** For example, given a Data Source with a filter that selects all sales records for a given year, a Variance Data Source may have a filter that selects the previous year, for doing comparissons.
+* `pivotFields` - *List of String, Conditional* - For *SUMMARY* or *PIVOT* type views, specify the field(s) used as pivot rows.
+** For example, in a summary view of orders, you may "pivot" on the *customerId* field, to produce one row per-customer, with aggregate data for that customer.
+* `titleFormat` - *String* - Java Format String, used with `titleFields` (if given), to produce a title row, e.g., first row in the view (before any rows from the data source).
+* `titleFields` - *List of String, Conditional* - Used with `titleFormat`, to provide values for any format specifiers in the format string.
+Syntax to reference a field (e.g., from a report input field) is: `${input.NAME}`, where `NAME` is the `name` attribute of the inputField.
+** Example of using `titleFormat` and `titleFields`:
+
+[source,java]
+----
+// given these inputFields:
+new QFieldMetaData("startDate", QFieldType.DATE)
+new QFieldMetaData("endDate", QFieldType.DATE)
+
+// a view can have a title row like this:
+.withTitleFormat("Weekly Sales Report - %s - %s")
+.withTitleFields(List.of("${input.startDate}", "${input.endDate}"))
+----
+
+* `includeHeaderRow` - *boolean, default true* - Indication that first row of the view should be the column labels.
+** If true, then header row is put in the view.
+** If false, then no header row is put in the view.
+* `includeTotalRow` - *boolean, default false* - Indication that a totals row should be added to the view.
+All numeric columns are summed to produce values in the totals row.
+** If true, then totals row is put in the view.
+** If false, then no totals row is put in the view.
+* `includePivotSubTotals` - *boolean, default false* - For a *SUMMARY* or *PIVOT* type view, if there are more than 1 *pivotFields* being used, this field is an indication that each higher-level pivot should include sub-totals.
+** #TODO - provide example#
+* `columns` - *List of QReportField, required* - Definition of the columns to appear in the view.  See section on QReportField for details.
+* `orderByFields` - *List of QFilterOrderBy, optional* - For a *SUMMARY* or *PIVOT* type view, how to sort the rows.
+* `recordTransformStep` - *QCodeReference, subclass of `AbstractTransformStep`* - Custom code reference that can be used to transform records after they are queried from the data source, and before they are placed into the view.
+Can be used to transform or customize values, or to look up additional values to add to the report.
+** #TODO - provide example#
+* `viewCustomizer` - *QCodeReference, implementation of interface `Function<QReportView, QReportView>`* - Custom code reference that can be used to customize the report view, at runtime.
+Can be used, for example, to dynamically define the report's *columns*.
+** #TODO - provide example#
+
+===== QReportField
+Report Fields define the fields (AKA columns) of data that appear in a report view.
+These fields can either be direct references to fields from the report's data sources, or values computed using formula defined in the QReportField.
+
+*QReportField Properties:*
+
+* `name` - *String, required* - Unique identifier for the field within its ReportView.
+In general, will be a reference to a field from the ReportView's DataSource *unless a *formula* is given (for *SUMMARY* type views), the field is marked as *isVirtual*, or the field is marked as *showPossibleValueLabel*).
+* `label` - *String* - Optional text label to identify the field, for example, in a header row.
+If not given, may be derived from field, where possible.
+* `type` - *QFieldType*
+* `formula` - *String, conditional* - Required for *SUMMARY* type views.
+Defines the formula to be used for computing the value in this field.
+** For example:
+
+[source,java]
+----
+.withName("reportEndDate").withFormula("${input.endDate}")
+
+.withName("count").withFormula("${pivot.count.id}")
+
+.withName("percentOfTotal").withFormula("=DIVIDE(${pivot.count.id},${total.count.id})")
+
+.withName("sumCost").withFormula("${pivot.sum.cost}")
+
+.withName("sumCharge").withFormula("${pivot.sum.charge}")
+
+.withName("profit").withFormula("=MINUS(${pivot.sum.charge},${pivot.sum.cost})")
+
+.withName("totalCost").withFormula("=DIVIDE_SCALE(${pivot.sum.cost},${pivot.count.id},2)")
+
+.withName("revenuePer").withFormula("=DIVIDE_SCALE(${pivot.sum.charge},${pivot.count.id},2)")
+
+.withName("marginPer").withFormula("=MINUS(DIVIDE_SCALE(${pivot.sum.charge},${pivot.count.id},2),DIVIDE_SCALE(${pivot.sum.cost},${pivot.count.id},2))")
+
+.withName("thisWeekMargin").withFormula("=SCALE(DIVIDE(${thisRow.profit},${pivot.sum.charge}),2)")
+
+.withName("previousWeekProfit").withFormula("=MINUS(${variancePivot.sum.charge},${variancePivot.sum.cost})")
+
+.withName("previousWeekMargin").withFormula("=SCALE(DIVIDE(${thisRow.previousWeekProfit},${variancePivot.sum.charge}),2)")
+
+.withName("marginThisVsPrevious").withFormula("=SCALE(MINUS(${thisRow.margin},${thisRow.marginPrevious}),3)")
+
+.withName("exception").withFormula("""
+   =IF(LT(${thisRow.margin},0),Negative Margin,IF(LT(${thisRow.marginThisVsPrevious},0),Margin Decreased,""))""")
+----
+
+* `displayFormat` *String*
+* `isVirtual` *Boolean, default false* - (needs reviewed - may only be required for report views using a data source with a *staticDataSupplier*)
+* `showPossibleValueLabel` *Boolean, default false* - To show a translated value for a Possible Value field (e.g., a name or other value meaningful to a user, instead of a foreign key).
+* `sourceFieldName` *String* - Used for the scenario where a possibleValue field is included in a report both as the foreign key (raw, id value), and the translated "label" value.
+In that case, the field marked with *showPossibleValueLabel* = true should be given a different name, and should use *sourceFieldName* to indicate the field that has the id value.
+** For example:
+
+[source,java]
+----
+// this field would have the "raw" warehouseId values
+// e.g., integers - foreign keys to a warehouse table.  Generally useful for machines to know.
+new QReportField("warehouseId")
+   .withLabel("Warehouse Id"),
+
+// this field would have the translated values from the warehouse PossibleValueSource
+// for example, maybe the name field from the warehouse table.  A string, useful for humans to read.
+new QReportField("warehouseName")
+   .withSourceFieldName("warehouseId")
+   .withShowPossibleValueLabel(true)
+   .withLabel("Warehouse Name"),
+----
+

docs/metaData/Tables.adoc

@@ -0,0 +1,49 @@
+== QQQ Tables
+include::../variables.adoc[]
+
+The core type of object in a QQQ Instance is the Table.
+In the most common use-case, a QQQ Table may be the in-app representation of a Database table.
+That is, it is a collection of records (or rows) of data, each of which has a set of fields (or columns).
+
+QQQ also allows other types of data sources ({link-backends}) to be used as tables, such as File systems, API's, Java enums or objects, etc.
+All of these backend types present the same interfaces (both user-interfaces, and application programming interfaces), regardless of their backend type.
+
+=== QTableMetaData
+Tables are defined in a QQQ Instance in a `*QTableMetaData*` object.
+All tables must reference a {link-backend}, a list of fields that define the shape of records in the table, and additional data to describe how to work with the table within its backend.
+
+*QTableMetaData Properties:*
+
+* `name` - *String, Required* - Unique name for the table within the QQQ Instance.
+* `label` - *String* - User-facing label for the table, presented in User Interfaces.
+Inferred from `name` if not set.
+* `backendName` - *String, Required* - Name of a {link-backend} in which this table's data is managed.
+* `fields` - *Map of String → {link-field}, Required* - The columns of data that make up all records in this table.
+* `primaryKeyField` - *String, Conditional* - Name of a {link-field} that serves as the primary key (e.g., unique identifier) for records in this table.
+* `uniqueKeys` - *List of UniqueKey* - Definition of additional unique constraints (from an RDBMS point of view) from the table.
+e.g., sets of columns which must have unique values for each record in the table.
+* `backendDetails` - *QTableBackendDetails or subclass* - Additional data to configure the table within its {link-backend}.
+* `automationDetails` - *QTableAutomationDetails* - Configuration of automated jobs that run against records in the table, e.g., upon insert or update.
+* `customizers` - *Map of String → QCodeReference* - References to custom code that are injected into standard table actions, that allow applications to customize certain parts of how the table works.
+* `parentAppName` - *String* - Name of a {link-app} that this table exists within.
+* `icon` - *QIcon* - Icon associated with this table in certain user interfaces.
+* `recordLabelFormat` - *String* - Java Format String, used with `recordLabelFields` to produce a label shown for records from the table.
+* `recordLabelFields` - *List of String, Conditional* - Used with `recordLabelFormat` to provide values for any format specifiers in the format string.
+These strings must be field names within the table.
+** Example of using `recordLabelFormat` and `recordLabelFields`:
+
+[source,java]
+----
+// given these fields in the table:
+new QFieldMetaData("name", QFieldType.STRING)
+new QFieldMetaData("birthDate", QFieldType.DATE)
+
+// We can produce a record label such as "Darin Kelkhoff (1980-05-31)" via:
+.withRecordLabelFormat("%s (%s)")
+.withRecordLabelFields(List.of("name", "birthDate"))
+----
+* `sections` - *List of QFieldSection* - Mechanism to organize fields within user interfaces, into logical sections.
+If any sections are present in the table meta data, then all fields in the table must be listed in exactly 1 section.
+If no sections are defined, then instance enrichment will define default sections.
+* `associatedScripts` - *List of AssociatedScript* - Definition of user-defined scripts that can be associated with records within the table.
+* `enabledCapabilities` and `disabledCapabilities` - *Set of Capability enum values* - Overrides from the backend level, for capabilities that this table does or does not possess.

docs/metaData/Tables.html

@@ -0,0 +1,553 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+<meta charset="UTF-8">
+<meta http-equiv="X-UA-Compatible" content="IE=edge">
+<meta name="viewport" content="width=device-width, initial-scale=1.0">
+<meta name="generator" content="Asciidoctor 2.0.18">
+<title>QQQ Tables</title>
+<link rel="stylesheet" href="https://fonts.googleapis.com/css?family=Open+Sans:300,300italic,400,400italic,600,600italic%7CNoto+Serif:400,400italic,700,700italic%7CDroid+Sans+Mono:400,700">
+<style>
+/*! Asciidoctor default stylesheet | MIT License | https://asciidoctor.org */
+/* Uncomment the following line when using as a custom stylesheet */
+/* @import "https://fonts.googleapis.com/css?family=Open+Sans:300,300italic,400,400italic,600,600italic%7CNoto+Serif:400,400italic,700,700italic%7CDroid+Sans+Mono:400,700"; */
+html{font-family:sans-serif;-webkit-text-size-adjust:100%}
+a{background:none}
+a:focus{outline:thin dotted}
+a:active,a:hover{outline:0}
+h1{font-size:2em;margin:.67em 0}
+b,strong{font-weight:bold}
+abbr{font-size:.9em}
+abbr[title]{cursor:help;border-bottom:1px dotted #dddddf;text-decoration:none}
+dfn{font-style:italic}
+hr{height:0}
+mark{background:#ff0;color:#000}
+code,kbd,pre,samp{font-family:monospace;font-size:1em}
+pre{white-space:pre-wrap}
+q{quotes:"\201C" "\201D" "\2018" "\2019"}
+small{font-size:80%}
+sub,sup{font-size:75%;line-height:0;position:relative;vertical-align:baseline}
+sup{top:-.5em}
+sub{bottom:-.25em}
+img{border:0}
+svg:not(:root){overflow:hidden}
+figure{margin:0}
+audio,video{display:inline-block}
+audio:not([controls]){display:none;height:0}
+fieldset{border:1px solid silver;margin:0 2px;padding:.35em .625em .75em}
+legend{border:0;padding:0}
+button,input,select,textarea{font-family:inherit;font-size:100%;margin:0}
+button,input{line-height:normal}
+button,select{text-transform:none}
+button,html input[type=button],input[type=reset],input[type=submit]{-webkit-appearance:button;cursor:pointer}
+button[disabled],html input[disabled]{cursor:default}
+input[type=checkbox],input[type=radio]{padding:0}
+button::-moz-focus-inner,input::-moz-focus-inner{border:0;padding:0}
+textarea{overflow:auto;vertical-align:top}
+table{border-collapse:collapse;border-spacing:0}
+*,::before,::after{box-sizing:border-box}
+html,body{font-size:100%}
+body{background:#fff;color:rgba(0,0,0,.8);padding:0;margin:0;font-family:"Noto Serif","DejaVu Serif",serif;line-height:1;position:relative;cursor:auto;-moz-tab-size:4;-o-tab-size:4;tab-size:4;word-wrap:anywhere;-moz-osx-font-smoothing:grayscale;-webkit-font-smoothing:antialiased}
+a:hover{cursor:pointer}
+img,object,embed{max-width:100%;height:auto}
+object,embed{height:100%}
+img{-ms-interpolation-mode:bicubic}
+.left{float:left!important}
+.right{float:right!important}
+.text-left{text-align:left!important}
+.text-right{text-align:right!important}
+.text-center{text-align:center!important}
+.text-justify{text-align:justify!important}
+.hide{display:none}
+img,object,svg{display:inline-block;vertical-align:middle}
+textarea{height:auto;min-height:50px}
+select{width:100%}
+.subheader,.admonitionblock td.content>.title,.audioblock>.title,.exampleblock>.title,.imageblock>.title,.listingblock>.title,.literalblock>.title,.stemblock>.title,.openblock>.title,.paragraph>.title,.quoteblock>.title,table.tableblock>.title,.verseblock>.title,.videoblock>.title,.dlist>.title,.olist>.title,.ulist>.title,.qlist>.title,.hdlist>.title{line-height:1.45;color:#7a2518;font-weight:400;margin-top:0;margin-bottom:.25em}
+div,dl,dt,dd,ul,ol,li,h1,h2,h3,#toctitle,.sidebarblock>.content>.title,h4,h5,h6,pre,form,p,blockquote,th,td{margin:0;padding:0}
+a{color:#2156a5;text-decoration:underline;line-height:inherit}
+a:hover,a:focus{color:#1d4b8f}
+a img{border:0}
+p{line-height:1.6;margin-bottom:1.25em;text-rendering:optimizeLegibility}
+p aside{font-size:.875em;line-height:1.35;font-style:italic}
+h1,h2,h3,#toctitle,.sidebarblock>.content>.title,h4,h5,h6{font-family:"Open Sans","DejaVu Sans",sans-serif;font-weight:300;font-style:normal;color:#ba3925;text-rendering:optimizeLegibility;margin-top:1em;margin-bottom:.5em;line-height:1.0125em}
+h1 small,h2 small,h3 small,#toctitle small,.sidebarblock>.content>.title small,h4 small,h5 small,h6 small{font-size:60%;color:#e99b8f;line-height:0}
+h1{font-size:2.125em}
+h2{font-size:1.6875em}
+h3,#toctitle,.sidebarblock>.content>.title{font-size:1.375em}
+h4,h5{font-size:1.125em}
+h6{font-size:1em}
+hr{border:solid #dddddf;border-width:1px 0 0;clear:both;margin:1.25em 0 1.1875em}
+em,i{font-style:italic;line-height:inherit}
+strong,b{font-weight:bold;line-height:inherit}
+small{font-size:60%;line-height:inherit}
+code{font-family:"Droid Sans Mono","DejaVu Sans Mono",monospace;font-weight:400;color:rgba(0,0,0,.9)}
+ul,ol,dl{line-height:1.6;margin-bottom:1.25em;list-style-position:outside;font-family:inherit}
+ul,ol{margin-left:1.5em}
+ul li ul,ul li ol{margin-left:1.25em;margin-bottom:0}
+ul.circle{list-style-type:circle}
+ul.disc{list-style-type:disc}
+ul.square{list-style-type:square}
+ul.circle ul:not([class]),ul.disc ul:not([class]),ul.square ul:not([class]){list-style:inherit}
+ol li ul,ol li ol{margin-left:1.25em;margin-bottom:0}
+dl dt{margin-bottom:.3125em;font-weight:bold}
+dl dd{margin-bottom:1.25em}
+blockquote{margin:0 0 1.25em;padding:.5625em 1.25em 0 1.1875em;border-left:1px solid #ddd}
+blockquote,blockquote p{line-height:1.6;color:rgba(0,0,0,.85)}
+@media screen and (min-width:768px){h1,h2,h3,#toctitle,.sidebarblock>.content>.title,h4,h5,h6{line-height:1.2}
+h1{font-size:2.75em}
+h2{font-size:2.3125em}
+h3,#toctitle,.sidebarblock>.content>.title{font-size:1.6875em}
+h4{font-size:1.4375em}}
+table{background:#fff;margin-bottom:1.25em;border:1px solid #dedede;word-wrap:normal}
+table thead,table tfoot{background:#f7f8f7}
+table thead tr th,table thead tr td,table tfoot tr th,table tfoot tr td{padding:.5em .625em .625em;font-size:inherit;color:rgba(0,0,0,.8);text-align:left}
+table tr th,table tr td{padding:.5625em .625em;font-size:inherit;color:rgba(0,0,0,.8)}
+table tr.even,table tr.alt{background:#f8f8f7}
+table thead tr th,table tfoot tr th,table tbody tr td,table tr td,table tfoot tr td{line-height:1.6}
+h1,h2,h3,#toctitle,.sidebarblock>.content>.title,h4,h5,h6{line-height:1.2;word-spacing:-.05em}
+h1 strong,h2 strong,h3 strong,#toctitle strong,.sidebarblock>.content>.title strong,h4 strong,h5 strong,h6 strong{font-weight:400}
+.center{margin-left:auto;margin-right:auto}
+.stretch{width:100%}
+.clearfix::before,.clearfix::after,.float-group::before,.float-group::after{content:" ";display:table}
+.clearfix::after,.float-group::after{clear:both}
+:not(pre).nobreak{word-wrap:normal}
+:not(pre).nowrap{white-space:nowrap}
+:not(pre).pre-wrap{white-space:pre-wrap}
+:not(pre):not([class^=L])>code{font-size:.9375em;font-style:normal!important;letter-spacing:0;padding:.1em .5ex;word-spacing:-.15em;background:#f7f7f8;border-radius:4px;line-height:1.45;text-rendering:optimizeSpeed}
+pre{color:rgba(0,0,0,.9);font-family:"Droid Sans Mono","DejaVu Sans Mono",monospace;line-height:1.45;text-rendering:optimizeSpeed}
+pre code,pre pre{color:inherit;font-size:inherit;line-height:inherit}
+pre>code{display:block}
+pre.nowrap,pre.nowrap pre{white-space:pre;word-wrap:normal}
+em em{font-style:normal}
+strong strong{font-weight:400}
+.keyseq{color:rgba(51,51,51,.8)}
+kbd{font-family:"Droid Sans Mono","DejaVu Sans Mono",monospace;display:inline-block;color:rgba(0,0,0,.8);font-size:.65em;line-height:1.45;background:#f7f7f7;border:1px solid #ccc;border-radius:3px;box-shadow:0 1px 0 rgba(0,0,0,.2),inset 0 0 0 .1em #fff;margin:0 .15em;padding:.2em .5em;vertical-align:middle;position:relative;top:-.1em;white-space:nowrap}
+.keyseq kbd:first-child{margin-left:0}
+.keyseq kbd:last-child{margin-right:0}
+.menuseq,.menuref{color:#000}
+.menuseq b:not(.caret),.menuref{font-weight:inherit}
+.menuseq{word-spacing:-.02em}
+.menuseq b.caret{font-size:1.25em;line-height:.8}
+.menuseq i.caret{font-weight:bold;text-align:center;width:.45em}
+b.button::before,b.button::after{position:relative;top:-1px;font-weight:400}
+b.button::before{content:"[";padding:0 3px 0 2px}
+b.button::after{content:"]";padding:0 2px 0 3px}
+p a>code:hover{color:rgba(0,0,0,.9)}
+#header,#content,#footnotes,#footer{width:100%;margin:0 auto;max-width:62.5em;*zoom:1;position:relative;padding-left:.9375em;padding-right:.9375em}
+#header::before,#header::after,#content::before,#content::after,#footnotes::before,#footnotes::after,#footer::before,#footer::after{content:" ";display:table}
+#header::after,#content::after,#footnotes::after,#footer::after{clear:both}
+#content{margin-top:1.25em}
+#content::before{content:none}
+#header>h1:first-child{color:rgba(0,0,0,.85);margin-top:2.25rem;margin-bottom:0}
+#header>h1:first-child+#toc{margin-top:8px;border-top:1px solid #dddddf}
+#header>h1:only-child,body.toc2 #header>h1:nth-last-child(2){border-bottom:1px solid #dddddf;padding-bottom:8px}
+#header .details{border-bottom:1px solid #dddddf;line-height:1.45;padding-top:.25em;padding-bottom:.25em;padding-left:.25em;color:rgba(0,0,0,.6);display:flex;flex-flow:row wrap}
+#header .details span:first-child{margin-left:-.125em}
+#header .details span.email a{color:rgba(0,0,0,.85)}
+#header .details br{display:none}
+#header .details br+span::before{content:"\00a0\2013\00a0"}
+#header .details br+span.author::before{content:"\00a0\22c5\00a0";color:rgba(0,0,0,.85)}
+#header .details br+span#revremark::before{content:"\00a0|\00a0"}
+#header #revnumber{text-transform:capitalize}
+#header #revnumber::after{content:"\00a0"}
+#content>h1:first-child:not([class]){color:rgba(0,0,0,.85);border-bottom:1px solid #dddddf;padding-bottom:8px;margin-top:0;padding-top:1rem;margin-bottom:1.25rem}
+#toc{border-bottom:1px solid #e7e7e9;padding-bottom:.5em}
+#toc>ul{margin-left:.125em}
+#toc ul.sectlevel0>li>a{font-style:italic}
+#toc ul.sectlevel0 ul.sectlevel1{margin:.5em 0}
+#toc ul{font-family:"Open Sans","DejaVu Sans",sans-serif;list-style-type:none}
+#toc li{line-height:1.3334;margin-top:.3334em}
+#toc a{text-decoration:none}
+#toc a:active{text-decoration:underline}
+#toctitle{color:#7a2518;font-size:1.2em}
+@media screen and (min-width:768px){#toctitle{font-size:1.375em}
+body.toc2{padding-left:15em;padding-right:0}
+#toc.toc2{margin-top:0!important;background:#f8f8f7;position:fixed;width:15em;left:0;top:0;border-right:1px solid #e7e7e9;border-top-width:0!important;border-bottom-width:0!important;z-index:1000;padding:1.25em 1em;height:100%;overflow:auto}
+#toc.toc2 #toctitle{margin-top:0;margin-bottom:.8rem;font-size:1.2em}
+#toc.toc2>ul{font-size:.9em;margin-bottom:0}
+#toc.toc2 ul ul{margin-left:0;padding-left:1em}
+#toc.toc2 ul.sectlevel0 ul.sectlevel1{padding-left:0;margin-top:.5em;margin-bottom:.5em}
+body.toc2.toc-right{padding-left:0;padding-right:15em}
+body.toc2.toc-right #toc.toc2{border-right-width:0;border-left:1px solid #e7e7e9;left:auto;right:0}}
+@media screen and (min-width:1280px){body.toc2{padding-left:20em;padding-right:0}
+#toc.toc2{width:20em}
+#toc.toc2 #toctitle{font-size:1.375em}
+#toc.toc2>ul{font-size:.95em}
+#toc.toc2 ul ul{padding-left:1.25em}
+body.toc2.toc-right{padding-left:0;padding-right:20em}}
+#content #toc{border:1px solid #e0e0dc;margin-bottom:1.25em;padding:1.25em;background:#f8f8f7;border-radius:4px}
+#content #toc>:first-child{margin-top:0}
+#content #toc>:last-child{margin-bottom:0}
+#footer{max-width:none;background:rgba(0,0,0,.8);padding:1.25em}
+#footer-text{color:hsla(0,0%,100%,.8);line-height:1.44}
+#content{margin-bottom:.625em}
+.sect1{padding-bottom:.625em}
+@media screen and (min-width:768px){#content{margin-bottom:1.25em}
+.sect1{padding-bottom:1.25em}}
+.sect1:last-child{padding-bottom:0}
+.sect1+.sect1{border-top:1px solid #e7e7e9}
+#content h1>a.anchor,h2>a.anchor,h3>a.anchor,#toctitle>a.anchor,.sidebarblock>.content>.title>a.anchor,h4>a.anchor,h5>a.anchor,h6>a.anchor{position:absolute;z-index:1001;width:1.5ex;margin-left:-1.5ex;display:block;text-decoration:none!important;visibility:hidden;text-align:center;font-weight:400}
+#content h1>a.anchor::before,h2>a.anchor::before,h3>a.anchor::before,#toctitle>a.anchor::before,.sidebarblock>.content>.title>a.anchor::before,h4>a.anchor::before,h5>a.anchor::before,h6>a.anchor::before{content:"\00A7";font-size:.85em;display:block;padding-top:.1em}
+#content h1:hover>a.anchor,#content h1>a.anchor:hover,h2:hover>a.anchor,h2>a.anchor:hover,h3:hover>a.anchor,#toctitle:hover>a.anchor,.sidebarblock>.content>.title:hover>a.anchor,h3>a.anchor:hover,#toctitle>a.anchor:hover,.sidebarblock>.content>.title>a.anchor:hover,h4:hover>a.anchor,h4>a.anchor:hover,h5:hover>a.anchor,h5>a.anchor:hover,h6:hover>a.anchor,h6>a.anchor:hover{visibility:visible}
+#content h1>a.link,h2>a.link,h3>a.link,#toctitle>a.link,.sidebarblock>.content>.title>a.link,h4>a.link,h5>a.link,h6>a.link{color:#ba3925;text-decoration:none}
+#content h1>a.link:hover,h2>a.link:hover,h3>a.link:hover,#toctitle>a.link:hover,.sidebarblock>.content>.title>a.link:hover,h4>a.link:hover,h5>a.link:hover,h6>a.link:hover{color:#a53221}
+details,.audioblock,.imageblock,.literalblock,.listingblock,.stemblock,.videoblock{margin-bottom:1.25em}
+details{margin-left:1.25rem}
+details>summary{cursor:pointer;display:block;position:relative;line-height:1.6;margin-bottom:.625rem;outline:none;-webkit-tap-highlight-color:transparent}
+details>summary::-webkit-details-marker{display:none}
+details>summary::before{content:"";border:solid transparent;border-left:solid;border-width:.3em 0 .3em .5em;position:absolute;top:.5em;left:-1.25rem;transform:translateX(15%)}
+details[open]>summary::before{border:solid transparent;border-top:solid;border-width:.5em .3em 0;transform:translateY(15%)}
+details>summary::after{content:"";width:1.25rem;height:1em;position:absolute;top:.3em;left:-1.25rem}
+.admonitionblock td.content>.title,.audioblock>.title,.exampleblock>.title,.imageblock>.title,.listingblock>.title,.literalblock>.title,.stemblock>.title,.openblock>.title,.paragraph>.title,.quoteblock>.title,table.tableblock>.title,.verseblock>.title,.videoblock>.title,.dlist>.title,.olist>.title,.ulist>.title,.qlist>.title,.hdlist>.title{text-rendering:optimizeLegibility;text-align:left;font-family:"Noto Serif","DejaVu Serif",serif;font-size:1rem;font-style:italic}
+table.tableblock.fit-content>caption.title{white-space:nowrap;width:0}
+.paragraph.lead>p,#preamble>.sectionbody>[class=paragraph]:first-of-type p{font-size:1.21875em;line-height:1.6;color:rgba(0,0,0,.85)}
+.admonitionblock>table{border-collapse:separate;border:0;background:none;width:100%}
+.admonitionblock>table td.icon{text-align:center;width:80px}
+.admonitionblock>table td.icon img{max-width:none}
+.admonitionblock>table td.icon .title{font-weight:bold;font-family:"Open Sans","DejaVu Sans",sans-serif;text-transform:uppercase}
+.admonitionblock>table td.content{padding-left:1.125em;padding-right:1.25em;border-left:1px solid #dddddf;color:rgba(0,0,0,.6);word-wrap:anywhere}
+.admonitionblock>table td.content>:last-child>:last-child{margin-bottom:0}
+.exampleblock>.content{border:1px solid #e6e6e6;margin-bottom:1.25em;padding:1.25em;background:#fff;border-radius:4px}
+.exampleblock>.content>:first-child{margin-top:0}
+.exampleblock>.content>:last-child{margin-bottom:0}
+.sidebarblock{border:1px solid #dbdbd6;margin-bottom:1.25em;padding:1.25em;background:#f3f3f2;border-radius:4px}
+.sidebarblock>:first-child{margin-top:0}
+.sidebarblock>:last-child{margin-bottom:0}
+.sidebarblock>.content>.title{color:#7a2518;margin-top:0;text-align:center}
+.exampleblock>.content>:last-child>:last-child,.exampleblock>.content .olist>ol>li:last-child>:last-child,.exampleblock>.content .ulist>ul>li:last-child>:last-child,.exampleblock>.content .qlist>ol>li:last-child>:last-child,.sidebarblock>.content>:last-child>:last-child,.sidebarblock>.content .olist>ol>li:last-child>:last-child,.sidebarblock>.content .ulist>ul>li:last-child>:last-child,.sidebarblock>.content .qlist>ol>li:last-child>:last-child{margin-bottom:0}
+.literalblock pre,.listingblock>.content>pre{border-radius:4px;overflow-x:auto;padding:1em;font-size:.8125em}
+@media screen and (min-width:768px){.literalblock pre,.listingblock>.content>pre{font-size:.90625em}}
+@media screen and (min-width:1280px){.literalblock pre,.listingblock>.content>pre{font-size:1em}}
+.literalblock pre,.listingblock>.content>pre:not(.highlight),.listingblock>.content>pre[class=highlight],.listingblock>.content>pre[class^="highlight "]{background:#f7f7f8}
+.literalblock.output pre{color:#f7f7f8;background:rgba(0,0,0,.9)}
+.listingblock>.content{position:relative}
+.listingblock code[data-lang]::before{display:none;content:attr(data-lang);position:absolute;font-size:.75em;top:.425rem;right:.5rem;line-height:1;text-transform:uppercase;color:inherit;opacity:.5}
+.listingblock:hover code[data-lang]::before{display:block}
+.listingblock.terminal pre .command::before{content:attr(data-prompt);padding-right:.5em;color:inherit;opacity:.5}
+.listingblock.terminal pre .command:not([data-prompt])::before{content:"$"}
+.listingblock pre.highlightjs{padding:0}
+.listingblock pre.highlightjs>code{padding:1em;border-radius:4px}
+.listingblock pre.prettyprint{border-width:0}
+.prettyprint{background:#f7f7f8}
+pre.prettyprint .linenums{line-height:1.45;margin-left:2em}
+pre.prettyprint li{background:none;list-style-type:inherit;padding-left:0}
+pre.prettyprint li code[data-lang]::before{opacity:1}
+pre.prettyprint li:not(:first-child) code[data-lang]::before{display:none}
+table.linenotable{border-collapse:separate;border:0;margin-bottom:0;background:none}
+table.linenotable td[class]{color:inherit;vertical-align:top;padding:0;line-height:inherit;white-space:normal}
+table.linenotable td.code{padding-left:.75em}
+table.linenotable td.linenos,pre.pygments .linenos{border-right:1px solid;opacity:.35;padding-right:.5em;-webkit-user-select:none;-moz-user-select:none;-ms-user-select:none;user-select:none}
+pre.pygments span.linenos{display:inline-block;margin-right:.75em}
+.quoteblock{margin:0 1em 1.25em 1.5em;display:table}
+.quoteblock:not(.excerpt)>.title{margin-left:-1.5em;margin-bottom:.75em}
+.quoteblock blockquote,.quoteblock p{color:rgba(0,0,0,.85);font-size:1.15rem;line-height:1.75;word-spacing:.1em;letter-spacing:0;font-style:italic;text-align:justify}
+.quoteblock blockquote{margin:0;padding:0;border:0}
+.quoteblock blockquote::before{content:"\201c";float:left;font-size:2.75em;font-weight:bold;line-height:.6em;margin-left:-.6em;color:#7a2518;text-shadow:0 1px 2px rgba(0,0,0,.1)}
+.quoteblock blockquote>.paragraph:last-child p{margin-bottom:0}
+.quoteblock .attribution{margin-top:.75em;margin-right:.5ex;text-align:right}
+.verseblock{margin:0 1em 1.25em}
+.verseblock pre{font-family:"Open Sans","DejaVu Sans",sans-serif;font-size:1.15rem;color:rgba(0,0,0,.85);font-weight:300;text-rendering:optimizeLegibility}
+.verseblock pre strong{font-weight:400}
+.verseblock .attribution{margin-top:1.25rem;margin-left:.5ex}
+.quoteblock .attribution,.verseblock .attribution{font-size:.9375em;line-height:1.45;font-style:italic}
+.quoteblock .attribution br,.verseblock .attribution br{display:none}
+.quoteblock .attribution cite,.verseblock .attribution cite{display:block;letter-spacing:-.025em;color:rgba(0,0,0,.6)}
+.quoteblock.abstract blockquote::before,.quoteblock.excerpt blockquote::before,.quoteblock .quoteblock blockquote::before{display:none}
+.quoteblock.abstract blockquote,.quoteblock.abstract p,.quoteblock.excerpt blockquote,.quoteblock.excerpt p,.quoteblock .quoteblock blockquote,.quoteblock .quoteblock p{line-height:1.6;word-spacing:0}
+.quoteblock.abstract{margin:0 1em 1.25em;display:block}
+.quoteblock.abstract>.title{margin:0 0 .375em;font-size:1.15em;text-align:center}
+.quoteblock.excerpt>blockquote,.quoteblock .quoteblock{padding:0 0 .25em 1em;border-left:.25em solid #dddddf}
+.quoteblock.excerpt,.quoteblock .quoteblock{margin-left:0}
+.quoteblock.excerpt blockquote,.quoteblock.excerpt p,.quoteblock .quoteblock blockquote,.quoteblock .quoteblock p{color:inherit;font-size:1.0625rem}
+.quoteblock.excerpt .attribution,.quoteblock .quoteblock .attribution{color:inherit;font-size:.85rem;text-align:left;margin-right:0}
+p.tableblock:last-child{margin-bottom:0}
+td.tableblock>.content{margin-bottom:1.25em;word-wrap:anywhere}
+td.tableblock>.content>:last-child{margin-bottom:-1.25em}
+table.tableblock,th.tableblock,td.tableblock{border:0 solid #dedede}
+table.grid-all>*>tr>*{border-width:1px}
+table.grid-cols>*>tr>*{border-width:0 1px}
+table.grid-rows>*>tr>*{border-width:1px 0}
+table.frame-all{border-width:1px}
+table.frame-ends{border-width:1px 0}
+table.frame-sides{border-width:0 1px}
+table.frame-none>colgroup+*>:first-child>*,table.frame-sides>colgroup+*>:first-child>*{border-top-width:0}
+table.frame-none>:last-child>:last-child>*,table.frame-sides>:last-child>:last-child>*{border-bottom-width:0}
+table.frame-none>*>tr>:first-child,table.frame-ends>*>tr>:first-child{border-left-width:0}
+table.frame-none>*>tr>:last-child,table.frame-ends>*>tr>:last-child{border-right-width:0}
+table.stripes-all>*>tr,table.stripes-odd>*>tr:nth-of-type(odd),table.stripes-even>*>tr:nth-of-type(even),table.stripes-hover>*>tr:hover{background:#f8f8f7}
+th.halign-left,td.halign-left{text-align:left}
+th.halign-right,td.halign-right{text-align:right}
+th.halign-center,td.halign-center{text-align:center}
+th.valign-top,td.valign-top{vertical-align:top}
+th.valign-bottom,td.valign-bottom{vertical-align:bottom}
+th.valign-middle,td.valign-middle{vertical-align:middle}
+table thead th,table tfoot th{font-weight:bold}
+tbody tr th{background:#f7f8f7}
+tbody tr th,tbody tr th p,tfoot tr th,tfoot tr th p{color:rgba(0,0,0,.8);font-weight:bold}
+p.tableblock>code:only-child{background:none;padding:0}
+p.tableblock{font-size:1em}
+ol{margin-left:1.75em}
+ul li ol{margin-left:1.5em}
+dl dd{margin-left:1.125em}
+dl dd:last-child,dl dd:last-child>:last-child{margin-bottom:0}
+li p,ul dd,ol dd,.olist .olist,.ulist .ulist,.ulist .olist,.olist .ulist{margin-bottom:.625em}
+ul.checklist,ul.none,ol.none,ul.no-bullet,ol.no-bullet,ol.unnumbered,ul.unstyled,ol.unstyled{list-style-type:none}
+ul.no-bullet,ol.no-bullet,ol.unnumbered{margin-left:.625em}
+ul.unstyled,ol.unstyled{margin-left:0}
+li>p:empty:only-child::before{content:"";display:inline-block}
+ul.checklist>li>p:first-child{margin-left:-1em}
+ul.checklist>li>p:first-child>.fa-square-o:first-child,ul.checklist>li>p:first-child>.fa-check-square-o:first-child{width:1.25em;font-size:.8em;position:relative;bottom:.125em}
+ul.checklist>li>p:first-child>input[type=checkbox]:first-child{margin-right:.25em}
+ul.inline{display:flex;flex-flow:row wrap;list-style:none;margin:0 0 .625em -1.25em}
+ul.inline>li{margin-left:1.25em}
+.unstyled dl dt{font-weight:400;font-style:normal}
+ol.arabic{list-style-type:decimal}
+ol.decimal{list-style-type:decimal-leading-zero}
+ol.loweralpha{list-style-type:lower-alpha}
+ol.upperalpha{list-style-type:upper-alpha}
+ol.lowerroman{list-style-type:lower-roman}
+ol.upperroman{list-style-type:upper-roman}
+ol.lowergreek{list-style-type:lower-greek}
+.hdlist>table,.colist>table{border:0;background:none}
+.hdlist>table>tbody>tr,.colist>table>tbody>tr{background:none}
+td.hdlist1,td.hdlist2{vertical-align:top;padding:0 .625em}
+td.hdlist1{font-weight:bold;padding-bottom:1.25em}
+td.hdlist2{word-wrap:anywhere}
+.literalblock+.colist,.listingblock+.colist{margin-top:-.5em}
+.colist td:not([class]):first-child{padding:.4em .75em 0;line-height:1;vertical-align:top}
+.colist td:not([class]):first-child img{max-width:none}
+.colist td:not([class]):last-child{padding:.25em 0}
+.thumb,.th{line-height:0;display:inline-block;border:4px solid #fff;box-shadow:0 0 0 1px #ddd}
+.imageblock.left{margin:.25em .625em 1.25em 0}
+.imageblock.right{margin:.25em 0 1.25em .625em}
+.imageblock>.title{margin-bottom:0}
+.imageblock.thumb,.imageblock.th{border-width:6px}
+.imageblock.thumb>.title,.imageblock.th>.title{padding:0 .125em}
+.image.left,.image.right{margin-top:.25em;margin-bottom:.25em;display:inline-block;line-height:0}
+.image.left{margin-right:.625em}
+.image.right{margin-left:.625em}
+a.image{text-decoration:none;display:inline-block}
+a.image object{pointer-events:none}
+sup.footnote,sup.footnoteref{font-size:.875em;position:static;vertical-align:super}
+sup.footnote a,sup.footnoteref a{text-decoration:none}
+sup.footnote a:active,sup.footnoteref a:active{text-decoration:underline}
+#footnotes{padding-top:.75em;padding-bottom:.75em;margin-bottom:.625em}
+#footnotes hr{width:20%;min-width:6.25em;margin:-.25em 0 .75em;border-width:1px 0 0}
+#footnotes .footnote{padding:0 .375em 0 .225em;line-height:1.3334;font-size:.875em;margin-left:1.2em;margin-bottom:.2em}
+#footnotes .footnote a:first-of-type{font-weight:bold;text-decoration:none;margin-left:-1.05em}
+#footnotes .footnote:last-of-type{margin-bottom:0}
+#content #footnotes{margin-top:-.625em;margin-bottom:0;padding:.75em 0}
+div.unbreakable{page-break-inside:avoid}
+.big{font-size:larger}
+.small{font-size:smaller}
+.underline{text-decoration:underline}
+.overline{text-decoration:overline}
+.line-through{text-decoration:line-through}
+.aqua{color:#00bfbf}
+.aqua-background{background:#00fafa}
+.black{color:#000}
+.black-background{background:#000}
+.blue{color:#0000bf}
+.blue-background{background:#0000fa}
+.fuchsia{color:#bf00bf}
+.fuchsia-background{background:#fa00fa}
+.gray{color:#606060}
+.gray-background{background:#7d7d7d}
+.green{color:#006000}
+.green-background{background:#007d00}
+.lime{color:#00bf00}
+.lime-background{background:#00fa00}
+.maroon{color:#600000}
+.maroon-background{background:#7d0000}
+.navy{color:#000060}
+.navy-background{background:#00007d}
+.olive{color:#606000}
+.olive-background{background:#7d7d00}
+.purple{color:#600060}
+.purple-background{background:#7d007d}
+.red{color:#bf0000}
+.red-background{background:#fa0000}
+.silver{color:#909090}
+.silver-background{background:#bcbcbc}
+.teal{color:#006060}
+.teal-background{background:#007d7d}
+.white{color:#bfbfbf}
+.white-background{background:#fafafa}
+.yellow{color:#bfbf00}
+.yellow-background{background:#fafa00}
+span.icon>.fa{cursor:default}
+a span.icon>.fa{cursor:inherit}
+.admonitionblock td.icon [class^="fa icon-"]{font-size:2.5em;text-shadow:1px 1px 2px rgba(0,0,0,.5);cursor:default}
+.admonitionblock td.icon .icon-note::before{content:"\f05a";color:#19407c}
+.admonitionblock td.icon .icon-tip::before{content:"\f0eb";text-shadow:1px 1px 2px rgba(155,155,0,.8);color:#111}
+.admonitionblock td.icon .icon-warning::before{content:"\f071";color:#bf6900}
+.admonitionblock td.icon .icon-caution::before{content:"\f06d";color:#bf3400}
+.admonitionblock td.icon .icon-important::before{content:"\f06a";color:#bf0000}
+.conum[data-value]{display:inline-block;color:#fff!important;background:rgba(0,0,0,.8);border-radius:50%;text-align:center;font-size:.75em;width:1.67em;height:1.67em;line-height:1.67em;font-family:"Open Sans","DejaVu Sans",sans-serif;font-style:normal;font-weight:bold}
+.conum[data-value] *{color:#fff!important}
+.conum[data-value]+b{display:none}
+.conum[data-value]::after{content:attr(data-value)}
+pre .conum[data-value]{position:relative;top:-.125em}
+b.conum *{color:inherit!important}
+.conum:not([data-value]):empty{display:none}
+dt,th.tableblock,td.content,div.footnote{text-rendering:optimizeLegibility}
+h1,h2,p,td.content,span.alt,summary{letter-spacing:-.01em}
+p strong,td.content strong,div.footnote strong{letter-spacing:-.005em}
+p,blockquote,dt,td.content,span.alt,summary{font-size:1.0625rem}
+p{margin-bottom:1.25rem}
+.sidebarblock p,.sidebarblock dt,.sidebarblock td.content,p.tableblock{font-size:1em}
+.exampleblock>.content{background:#fffef7;border-color:#e0e0dc;box-shadow:0 1px 4px #e0e0dc}
+.print-only{display:none!important}
+@page{margin:1.25cm .75cm}
+@media print{*{box-shadow:none!important;text-shadow:none!important}
+html{font-size:80%}
+a{color:inherit!important;text-decoration:underline!important}
+a.bare,a[href^="#"],a[href^="mailto:"]{text-decoration:none!important}
+a[href^="http:"]:not(.bare)::after,a[href^="https:"]:not(.bare)::after{content:"(" attr(href) ")";display:inline-block;font-size:.875em;padding-left:.25em}
+abbr[title]{border-bottom:1px dotted}
+abbr[title]::after{content:" (" attr(title) ")"}
+pre,blockquote,tr,img,object,svg{page-break-inside:avoid}
+thead{display:table-header-group}
+svg{max-width:100%}
+p,blockquote,dt,td.content{font-size:1em;orphans:3;widows:3}
+h2,h3,#toctitle,.sidebarblock>.content>.title{page-break-after:avoid}
+#header,#content,#footnotes,#footer{max-width:none}
+#toc,.sidebarblock,.exampleblock>.content{background:none!important}
+#toc{border-bottom:1px solid #dddddf!important;padding-bottom:0!important}
+body.book #header{text-align:center}
+body.book #header>h1:first-child{border:0!important;margin:2.5em 0 1em}
+body.book #header .details{border:0!important;display:block;padding:0!important}
+body.book #header .details span:first-child{margin-left:0!important}
+body.book #header .details br{display:block}
+body.book #header .details br+span::before{content:none!important}
+body.book #toc{border:0!important;text-align:left!important;padding:0!important;margin:0!important}
+body.book #toc,body.book #preamble,body.book h1.sect0,body.book .sect1>h2{page-break-before:always}
+.listingblock code[data-lang]::before{display:block}
+#footer{padding:0 .9375em}
+.hide-on-print{display:none!important}
+.print-only{display:block!important}
+.hide-for-print{display:none!important}
+.show-for-print{display:inherit!important}}
+@media amzn-kf8,print{#header>h1:first-child{margin-top:1.25rem}
+.sect1{padding:0!important}
+.sect1+.sect1{border:0}
+#footer{background:none}
+#footer-text{color:rgba(0,0,0,.6);font-size:.9em}}
+@media amzn-kf8{#header,#content,#footnotes,#footer{padding:0}}
+</style>
+</head>
+<body class="article">
+<div id="header">
+</div>
+<div id="content">
+<div class="sect1">
+<h2 id="_qqq_tables">QQQ Tables</h2>
+<div class="sectionbody">
+<div class="paragraph">
+<p>The core type of object in a QQQ Instance is the Table.
+In the most common use-case, a QQQ Table may be the in-app representation of a Database table.
+That is, it is a collection of records (or rows) of data, each of which has a set of fields (or columns).</p>
+</div>
+<div class="paragraph">
+<p>QQQ also allows other types of data sources (<a href="Backends{relfilesuffix}">QQQ Backends</a>) to be used as tables, such as File systems, API&#8217;s, Java enums or objects, etc.
+All of these backend types present the same interfaces (both user-interfaces, and application programming interfaces), regardless of their backend type.</p>
+</div>
+<div class="sect2">
+<h3 id="_qtablemetadata">QTableMetaData</h3>
+<div class="paragraph">
+<p>Tables are defined in a QQQ Instance in a <code><strong>QTableMetaData</strong></code> object.
+All tables must reference a <a href="Backends{relfilesuffix}">QQQ Backend</a>, a list of fields that define the shape of records in the table, and additional data to describe how to work with the table within its backend.</p>
+</div>
+<div class="paragraph">
+<p><strong>QTableMetaData Properties:</strong></p>
+</div>
+<div class="ulist">
+<ul>
+<li>
+<p><code>name</code> - <strong>String, Required</strong> - Unique name for the table within the QQQ Instance.</p>
+</li>
+<li>
+<p><code>label</code> - <strong>String</strong> - User-facing label for the table, presented in User Interfaces.
+Inferred from <code>name</code> if not set.</p>
+</li>
+<li>
+<p><code>backendName</code> - <strong>String, Required</strong> - Name of a <a href="Backends{relfilesuffix}">QQQ Backend</a> in which this table&#8217;s data is managed.</p>
+</li>
+<li>
+<p><code>fields</code> - <strong>Map of String → <a href="Fields{relfilesuffix}">QQQ Field</a>, Required</strong> - The columns of data that make up all records in this table.</p>
+</li>
+<li>
+<p><code>primaryKeyField</code> - <strong>String, Conditional</strong> - Name of a <a href="Fields{relfilesuffix}">QQQ Field</a> that serves as the primary key (e.g., unique identifier) for records in this table.</p>
+</li>
+<li>
+<p><code>uniqueKeys</code> - <strong>List of UniqueKey</strong> - Definition of additional unique constraints (from an RDBMS point of view) from the table.
+e.g., sets of columns which must have unique values for each record in the table.</p>
+</li>
+<li>
+<p><code>backendDetails</code> - <strong>QTableBackendDetails or subclass</strong> - Additional data to configure the table within its <a href="Backends{relfilesuffix}">QQQ Backend</a>.</p>
+</li>
+<li>
+<p><code>automationDetails</code> - <strong>QTableAutomationDetails</strong> - Configuration of automated jobs that run against records in the table, e.g., upon insert or update.</p>
+</li>
+<li>
+<p><code>customizers</code> - <strong>Map of String → QCodeReference</strong> - References to custom code that are injected into standard table actions, that allow applications to customize certain parts of how the table works.</p>
+</li>
+<li>
+<p><code>parentAppName</code> - <strong>String</strong> - Name of a <a href="Apps{relfilesuffix}">QQQ App</a> that this table exists within.</p>
+</li>
+<li>
+<p><code>icon</code> - <strong>QIcon</strong> - Icon associated with this table in certain user interfaces.</p>
+</li>
+<li>
+<p><code>recordLabelFormat</code> - <strong>String</strong> - Java Format String, used with <code>recordLabelFields</code> to produce a label shown for records from the table.</p>
+</li>
+<li>
+<p><code>recordLabelFields</code> - <strong>List of String, Conditional</strong> - Used with <code>recordLabelFormat</code> to provide values for any format specifiers in the format string.
+These strings must be field names within the table.</p>
+<div class="ulist">
+<ul>
+<li>
+<p>Example of using <code>recordLabelFormat</code> and <code>recordLabelFields</code>:</p>
+</li>
+</ul>
+</div>
+</li>
+</ul>
+</div>
+<div class="listingblock">
+<div class="content">
+<pre class="CodeRay highlight"><code data-lang="java"><span style="color:#777">// given these fields in the table:</span>
+<span style="color:#080;font-weight:bold">new</span> QFieldMetaData(<span style="background-color:hsla(0,100%,50%,0.05)"><span style="color:#710">&quot;</span><span style="color:#D20">name</span><span style="color:#710">&quot;</span></span>, QFieldType.STRING)
+<span style="color:#080;font-weight:bold">new</span> QFieldMetaData(<span style="background-color:hsla(0,100%,50%,0.05)"><span style="color:#710">&quot;</span><span style="color:#D20">birthDate</span><span style="color:#710">&quot;</span></span>, QFieldType.DATE)
+
+<span style="color:#777">// We can produce a record label such as &quot;Darin Kelkhoff (1980-05-31)&quot; via:</span>
+.withRecordLabelFormat(<span style="background-color:hsla(0,100%,50%,0.05)"><span style="color:#710">&quot;</span><span style="color:#D20">%s (%s)</span><span style="color:#710">&quot;</span></span>)
+.withRecordLabelFields(<span style="color:#0a8;font-weight:bold">List</span>.of(<span style="background-color:hsla(0,100%,50%,0.05)"><span style="color:#710">&quot;</span><span style="color:#D20">name</span><span style="color:#710">&quot;</span></span>, <span style="background-color:hsla(0,100%,50%,0.05)"><span style="color:#710">&quot;</span><span style="color:#D20">birthDate</span><span style="color:#710">&quot;</span></span>))</code></pre>
+</div>
+</div>
+<div class="ulist">
+<ul>
+<li>
+<p><code>sections</code> - <strong>List of QFieldSection</strong> - Mechanism to organize fields within user interfaces, into logical sections.
+If any sections are present in the table meta data, then all fields in the table must be listed in exactly 1 section.
+If no sections are defined, then instance enrichment will define default sections.</p>
+</li>
+<li>
+<p><code>associatedScripts</code> - <strong>List of AssociatedScript</strong> - Definition of user-defined scripts that can be associated with records within the table.</p>
+</li>
+<li>
+<p><code>enabledCapabilities</code> and <code>disabledCapabilities</code> - <strong>Set of Capability enum values</strong> - Overrides from the backend level, for capabilities that this table does or does not possess.</p>
+</li>
+</ul>
+</div>
+</div>
+</div>
+</div>
+</div>
+<div id="footer">
+<div id="footer-text">
+Last updated 2022-11-21 09:02:56 -0600
+</div>
+</div>
+</body>
+</html>

docs/variables.adoc

@@ -0,0 +1,13 @@
+ifdef::env-name[:relfilesuffix: .adoc]
+:link-backend: link:Backends{relfilesuffix}[QQQ Backend]
+:link-backends: link:Backends{relfilesuffix}[QQQ Backends]
+:link-table: link:Tables{relfilesuffix}[QQQ Table]
+:link-tables: link:Tables{relfilesuffix}[QQQ Tables]
+:link-join: link:Joins{relfilesuffix}[QQQ Join]
+:link-joins: link:Joins{relfilesuffix}[QQQ Joins]
+:link-field: link:Fields{relfilesuffix}[QQQ Field]
+:link-fields: link:Fields{relfilesuffix}[QQQ Fields]
+:link-process: link:Processes{relfilesuffix}[QQQ Process]
+:link-processes: link:Processes{relfilesuffix}[QQQ Processes]
+:link-app: link:Apps{relfilesuffix}[QQQ App]
+:link-apps: link:Apps{relfilesuffix}[QQQ Apps]

pom.xml

@@ -44,7 +44,7 @@
    </modules>
 
    <properties>
-      <revision>0.18.0</revision>
+      <revision>0.20.0-SNAPSHOT</revision>
 
       <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
       <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>

qqq-backend-core/pom.xml

@@ -84,7 +84,7 @@
       <dependency>
          <groupId>org.json</groupId>
          <artifactId>json</artifactId>
-         <version>20230227</version>
+         <version>20231013</version>
       </dependency>
       <dependency>
          <groupId>org.apache.commons</groupId>

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/AbstractQActionBiConsumer.java

@@ -23,7 +23,6 @@ package com.kingsrook.qqq.backend.core.actions;
 
 
 import java.util.concurrent.CompletableFuture;
-import java.util.concurrent.Executors;
 import java.util.concurrent.Future;
 import com.kingsrook.qqq.backend.core.context.CapturedContext;
 import com.kingsrook.qqq.backend.core.context.QContext;
@@ -54,7 +53,7 @@ public abstract class AbstractQActionBiConsumer<I extends AbstractActionInput, O
    {
       CapturedContext         capturedContext   = QContext.capture();
       CompletableFuture<Void> completableFuture = new CompletableFuture<>();
-      Executors.newCachedThreadPool().submit(() ->
+      ActionHelper.getExecutorService().submit(() ->
       {
          try
          {

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/AbstractQActionFunction.java

@@ -23,7 +23,6 @@ package com.kingsrook.qqq.backend.core.actions;
 
 
 import java.util.concurrent.CompletableFuture;
-import java.util.concurrent.Executors;
 import java.util.concurrent.Future;
 import com.kingsrook.qqq.backend.core.context.CapturedContext;
 import com.kingsrook.qqq.backend.core.context.QContext;
@@ -54,7 +53,7 @@ public abstract class AbstractQActionFunction<I extends AbstractActionInput, O e
    {
       CapturedContext      capturedContext   = QContext.capture();
       CompletableFuture<O> completableFuture = new CompletableFuture<>();
-      Executors.newCachedThreadPool().submit(() ->
+      ActionHelper.getExecutorService().submit(() ->
       {
          try
          {

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/ActionHelper.java

@@ -24,6 +24,10 @@ package com.kingsrook.qqq.backend.core.actions;
 
 import java.io.Serializable;
 import java.util.List;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.SynchronousQueue;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
 import java.util.function.Function;
 import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.exceptions.QAuthenticationException;
@@ -40,6 +44,20 @@ import com.kingsrook.qqq.backend.core.modules.authentication.QAuthenticationModu
  *******************************************************************************/
 public class ActionHelper
 {
+   /////////////////////////////////////////////////////////////////////////////
+   // we would probably use Executors.newCachedThreadPool() - but - it has no //
+   // maxPoolSize...  we think some limit is good, so that at a large number  //
+   // of attempted concurrent jobs we'll have new jobs block, rather than     //
+   // exhausting all server resources and locking up "everything"             //
+   // also, it seems like keeping a handful of core-threads around is very    //
+   // little actual waste, and better than ever wasting time starting a new   //
+   // one, which we know we'll often be doing.                                //
+   /////////////////////////////////////////////////////////////////////////////
+   private static Integer         CORE_THREADS    = 8;
+   private static Integer         MAX_THREADS     = 500;
+   private static ExecutorService executorService = new ThreadPoolExecutor(CORE_THREADS, MAX_THREADS, 60L, TimeUnit.SECONDS, new SynchronousQueue<>());
+
+
 
    /*******************************************************************************
     **
@@ -69,6 +87,17 @@ public class ActionHelper
 
 
 
+   /*******************************************************************************
+    ** access an executor service for sharing among the executeAsync methods of all
+    ** actions.
+    *******************************************************************************/
+   static ExecutorService getExecutorService()
+   {
+      return (executorService);
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/async/AsyncJobManager.java

@@ -28,6 +28,9 @@ import java.util.UUID;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.CompletionException;
 import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.SynchronousQueue;
+import java.util.concurrent.ThreadPoolExecutor;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.TimeoutException;
 import com.kingsrook.qqq.backend.core.context.CapturedContext;
@@ -51,9 +54,24 @@ public class AsyncJobManager
 {
    private static final QLogger LOG = QLogger.getLogger(AsyncJobManager.class);
 
+   /////////////////////////////////////////////////////////////////////////////
+   // we would probably use Executors.newCachedThreadPool() - but - it has no //
+   // maxPoolSize...  we think some limit is good, so that at a large number  //
+   // of attempted concurrent jobs we'll have new jobs block, rather than     //
+   // exhausting all server resources and locking up "everything"             //
+   // also, it seems like keeping a handful of core-threads around is very    //
+   // little actual waste, and better than ever wasting time starting a new   //
+   // one, which we know we'll often be doing.                                //
+   /////////////////////////////////////////////////////////////////////////////
+   private static Integer         CORE_THREADS    = 8;
+   private static Integer         MAX_THREADS     = 500;
+   private static ExecutorService executorService = new ThreadPoolExecutor(CORE_THREADS, MAX_THREADS, 60L, TimeUnit.SECONDS, new SynchronousQueue<>());
+
+
    private String forcedJobUUID = null;
 
 
+
    /*******************************************************************************
     ** Start a job - if it finishes within the specified timeout, get its results,
     ** else, get back an exception with the job id.
@@ -84,7 +102,7 @@ public class AsyncJobManager
          {
             QContext.init(capturedContext);
             return (runAsyncJob(jobName, asyncJob, uuidAndTypeStateKey, asyncJobStatus));
-         });
+         }, executorService);
 
          if(timeout == 0)
          {

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/audits/AuditAction.java

@@ -29,6 +29,7 @@ import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
+import java.util.Optional;
 import com.kingsrook.qqq.backend.core.actions.AbstractQActionFunction;
 import com.kingsrook.qqq.backend.core.actions.tables.GetAction;
 import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
@@ -49,6 +50,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 import com.kingsrook.qqq.backend.core.model.session.QUser;
 import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
 import com.kingsrook.qqq.backend.core.utils.Pair;
+import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
 
 
 /*******************************************************************************
@@ -108,12 +110,26 @@ public class AuditAction extends AbstractQActionFunction<AuditInput, AuditOutput
 
 
 
+   /*******************************************************************************
+    ** Simple overload that internally figures out primary key and security key values
+    **
+    ** Be aware - if the record doesn't have its security key values set (say it's a
+    ** partial record as part of an update), then those values won't be in the
+    ** security key map...  This should probably be considered a bug.
+    *******************************************************************************/
+   public static void appendToInput(AuditInput auditInput, QTableMetaData table, QRecord record, String auditMessage)
+   {
+      appendToInput(auditInput, table.getName(), record.getValueInteger(table.getPrimaryKeyField()), getRecordSecurityKeyValues(table, record, Optional.empty()), auditMessage);
+   }
+
+
+
    /*******************************************************************************
     ** Add 1 auditSingleInput to an AuditInput object - with no details (child records).
     *******************************************************************************/
-   public static AuditInput appendToInput(AuditInput auditInput, String tableName, Integer recordId, Map<String, Serializable> securityKeyValues, String message)
+   public static void appendToInput(AuditInput auditInput, String tableName, Integer recordId, Map<String, Serializable> securityKeyValues, String message)
    {
-      return (appendToInput(auditInput, tableName, recordId, securityKeyValues, message, null));
+      appendToInput(auditInput, tableName, recordId, securityKeyValues, message, null);
    }
 
 
@@ -139,6 +155,44 @@ public class AuditAction extends AbstractQActionFunction<AuditInput, AuditOutput
 
 
 
+   /*******************************************************************************
+    ** For a given record, from a given table, build a map of the record's security
+    ** key values.
+    **
+    ** If, in case, the record has null value(s), and the oldRecord is given (e.g.,
+    ** for the case of an update, where the record may not have all fields set, and
+    ** oldRecord should be known for doing field-diffs), then try to get the value(s)
+    ** from oldRecord.
+    **
+    ** Currently, will leave values null if they aren't found after that.
+    **
+    ** An alternative could be to re-fetch the record from its source if needed...
+    *******************************************************************************/
+   public static Map<String, Serializable> getRecordSecurityKeyValues(QTableMetaData table, QRecord record, Optional<QRecord> oldRecord)
+   {
+      Map<String, Serializable> securityKeyValues = new HashMap<>();
+      for(RecordSecurityLock recordSecurityLock : RecordSecurityLockFilters.filterForReadLocks(CollectionUtils.nonNullList(table.getRecordSecurityLocks())))
+      {
+         Serializable keyValue = record == null ? null : record.getValue(recordSecurityLock.getFieldName());
+
+         if(keyValue == null && oldRecord.isPresent())
+         {
+            LOG.debug("Table with a securityLock, but value not found in field", logPair("table", table.getName()), logPair("field", recordSecurityLock.getFieldName()));
+            keyValue = oldRecord.get().getValue(recordSecurityLock.getFieldName());
+         }
+
+         if(keyValue == null)
+         {
+            LOG.debug("Table with a securityLock, but value not found in field", logPair("table", table.getName()), logPair("field", recordSecurityLock.getFieldName()), logPair("oldRecordIsPresent", oldRecord.isPresent()));
+         }
+
+         securityKeyValues.put(recordSecurityLock.getSecurityKeyType(), keyValue);
+      }
+      return securityKeyValues;
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/audits/DMLAuditAction.java

@@ -23,8 +23,10 @@ package com.kingsrook.qqq.backend.core.actions.audits;
 
 
 import java.io.Serializable;
+import java.math.BigDecimal;
 import java.time.Instant;
 import java.time.ZoneId;
+import java.time.temporal.ChronoUnit;
 import java.util.ArrayList;
 import java.util.Comparator;
 import java.util.HashMap;
@@ -52,13 +54,12 @@ import com.kingsrook.qqq.backend.core.model.metadata.audits.AuditLevel;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
 import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
-import com.kingsrook.qqq.backend.core.model.metadata.security.RecordSecurityLock;
-import com.kingsrook.qqq.backend.core.model.metadata.security.RecordSecurityLockFilters;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 import com.kingsrook.qqq.backend.core.model.session.QSession;
 import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
 import com.kingsrook.qqq.backend.core.utils.StringUtils;
 import com.kingsrook.qqq.backend.core.utils.ValueUtils;
+import static com.kingsrook.qqq.backend.core.actions.audits.AuditAction.getRecordSecurityKeyValues;
 import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
 
 
@@ -70,6 +71,8 @@ public class DMLAuditAction extends AbstractQActionFunction<DMLAuditInput, DMLAu
 {
    private static final QLogger LOG = QLogger.getLogger(DMLAuditAction.class);
 
+   public static final String AUDIT_CONTEXT_FIELD_NAME = "auditContext";
+
 
 
    /*******************************************************************************
@@ -99,34 +102,7 @@ public class DMLAuditAction extends AbstractQActionFunction<DMLAuditInput, DMLAu
             return (output);
          }
 
-         String contextSuffix = "";
-         if(StringUtils.hasContent(input.getAuditContext()))
-         {
-            contextSuffix = " " + input.getAuditContext();
-         }
-
-         Optional<AbstractActionInput> actionInput = QContext.getFirstActionInStack();
-         if(actionInput.isPresent() && actionInput.get() instanceof RunProcessInput runProcessInput)
-         {
-            String           processName = runProcessInput.getProcessName();
-            QProcessMetaData process     = QContext.getQInstance().getProcess(processName);
-            if(process != null)
-            {
-               contextSuffix = " during process: " + process.getLabel();
-            }
-         }
-
-         QSession qSession   = QContext.getQSession();
-         String   apiVersion = qSession.getValue("apiVersion");
-         if(apiVersion != null)
-         {
-            String apiLabel = qSession.getValue("apiLabel");
-            if(!StringUtils.hasContent(apiLabel))
-            {
-               apiLabel = "API";
-            }
-            contextSuffix += (" via " + apiLabel + " Version: " + apiVersion);
-         }
+         String contextSuffix = getContentSuffix(input);
 
          AuditInput auditInput = new AuditInput();
          if(auditLevel.equals(AuditLevel.RECORD) || (auditLevel.equals(AuditLevel.FIELD) && !dmlType.supportsFields))
@@ -137,7 +113,7 @@ public class DMLAuditAction extends AbstractQActionFunction<DMLAuditInput, DMLAu
             ////////////////////////////////////////////////////////////////////////////////////////////////////////////
             for(QRecord record : recordList)
             {
-               AuditAction.appendToInput(auditInput, table.getName(), record.getValueInteger(table.getPrimaryKeyField()), getRecordSecurityKeyValues(table, record), "Record was " + dmlType.pastTenseVerb + contextSuffix);
+               AuditAction.appendToInput(auditInput, table.getName(), record.getValueInteger(table.getPrimaryKeyField()), getRecordSecurityKeyValues(table, record, Optional.empty()), "Record was " + dmlType.pastTenseVerb + contextSuffix);
             }
          }
          else if(auditLevel.equals(AuditLevel.FIELD))
@@ -147,7 +123,7 @@ public class DMLAuditAction extends AbstractQActionFunction<DMLAuditInput, DMLAu
             ///////////////////////////////////////////////////////////////////
             // do many audits, all with field level details, for FIELD level //
             ///////////////////////////////////////////////////////////////////
-            QPossibleValueTranslator qPossibleValueTranslator = new QPossibleValueTranslator(QContext.getQInstance(), qSession);
+            QPossibleValueTranslator qPossibleValueTranslator = new QPossibleValueTranslator(QContext.getQInstance(), QContext.getQSession());
             qPossibleValueTranslator.translatePossibleValuesInRecords(table, CollectionUtils.mergeLists(recordList, oldRecordList));
 
             //////////////////////////////////////////
@@ -169,92 +145,8 @@ public class DMLAuditAction extends AbstractQActionFunction<DMLAuditInput, DMLAu
                List<QRecord> details = new ArrayList<>();
                for(String fieldName : sortedFieldNames)
                {
-                  if(!record.getValues().containsKey(fieldName))
-                  {
-                     ////////////////////////////////////////////////////////////////////////////////////////////////
-                     // if the stored record doesn't have this field name, then don't audit anything about it      //
-                     // this is to deal with our Patch style updates not looking like every field was cleared out. //
-                     ////////////////////////////////////////////////////////////////////////////////////////////////
-                     continue;
-                  }
-
-                  if(fieldName.equals("modifyDate") || fieldName.equals("createDate") || fieldName.equals("automationStatus"))
-                  {
-                     continue;
-                  }
-
-                  QFieldMetaData field        = table.getField(fieldName);
-                  Serializable   value        = ValueUtils.getValueAsFieldType(field.getType(), record.getValue(fieldName));
-                  Serializable   oldValue     = oldRecord == null ? null : ValueUtils.getValueAsFieldType(field.getType(), oldRecord.getValue(fieldName));
-                  QRecord        detailRecord = null;
-
-                  if(oldRecord == null)
-                  {
-                     if(DMLType.INSERT.equals(dmlType) && value == null)
-                     {
-                        continue;
-                     }
-
-                     if(field.getType().equals(QFieldType.BLOB) || field.getType().needsMasked())
-                     {
-                        detailRecord = new QRecord().withValue("message", "Set " + field.getLabel());
-                     }
-                     else
-                     {
-                        String formattedValue = getFormattedValueForAuditDetail(record, fieldName, field, value);
-                        detailRecord = new QRecord().withValue("message", "Set " + field.getLabel() + " to " + formattedValue);
-                        detailRecord.withValue("newValue", formattedValue);
-                     }
-                  }
-                  else
-                  {
-                     if(!Objects.equals(oldValue, value))
-                     {
-                        if(field.getType().equals(QFieldType.BLOB) || field.getType().needsMasked())
-                        {
-                           if(oldValue == null)
-                           {
-                              detailRecord = new QRecord().withValue("message", "Set " + field.getLabel());
-                           }
-                           else if(value == null)
-                           {
-                              detailRecord = new QRecord().withValue("message", "Removed " + field.getLabel());
-                           }
-                           else
-                           {
-                              detailRecord = new QRecord().withValue("message", "Changed " + field.getLabel());
-                           }
-                        }
-                        else
-                        {
-                           String formattedValue    = getFormattedValueForAuditDetail(record, fieldName, field, value);
-                           String formattedOldValue = getFormattedValueForAuditDetail(oldRecord, fieldName, field, oldValue);
-
-                           if(oldValue == null)
-                           {
-                              detailRecord = new QRecord().withValue("message", "Set " + field.getLabel() + " to " + formatFormattedValueForDetailMessage(field, formattedValue));
-                              detailRecord.withValue("newValue", formattedValue);
-                           }
-                           else if(value == null)
-                           {
-                              detailRecord = new QRecord().withValue("message", "Removed " + formatFormattedValueForDetailMessage(field, formattedOldValue) + " from " + field.getLabel());
-                              detailRecord.withValue("oldValue", formattedOldValue);
-                           }
-                           else
-                           {
-                              detailRecord = new QRecord().withValue("message", "Changed " + field.getLabel() + " from " + formatFormattedValueForDetailMessage(field, formattedOldValue) + " to " + formatFormattedValueForDetailMessage(field, formattedValue));
-                              detailRecord.withValue("oldValue", formattedOldValue);
-                              detailRecord.withValue("newValue", formattedValue);
-                           }
-                        }
-                     }
-                  }
-
-                  if(detailRecord != null)
-                  {
-                     detailRecord.withValue("fieldName", fieldName);
-                     details.add(detailRecord);
-                  }
+                  makeAuditDetailRecordForField(fieldName, table, dmlType, record, oldRecord)
+                     .ifPresent(details::add);
                }
 
                if(details.isEmpty() && DMLType.UPDATE.equals(dmlType))
@@ -264,7 +156,7 @@ public class DMLAuditAction extends AbstractQActionFunction<DMLAuditInput, DMLAu
                }
                else
                {
-                  AuditAction.appendToInput(auditInput, table.getName(), record.getValueInteger(table.getPrimaryKeyField()), getRecordSecurityKeyValues(table, record), "Record was " + dmlType.pastTenseVerb + contextSuffix, details);
+                  AuditAction.appendToInput(auditInput, table.getName(), record.getValueInteger(table.getPrimaryKeyField()), getRecordSecurityKeyValues(table, record, Optional.ofNullable(oldRecord)), "Record was " + dmlType.pastTenseVerb + contextSuffix, details);
                }
             }
          }
@@ -284,6 +176,256 @@ public class DMLAuditAction extends AbstractQActionFunction<DMLAuditInput, DMLAu
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   static String getContentSuffix(DMLAuditInput input)
+   {
+      StringBuilder contextSuffix = new StringBuilder();
+
+      /////////////////////////////////////////////////////////////////////////////
+      // start with context from the input wrapper                               //
+      // note, these contexts get propagated down from Input/Update/Delete Input //
+      /////////////////////////////////////////////////////////////////////////////
+      if(StringUtils.hasContent(input.getAuditContext()))
+      {
+         contextSuffix.append(" ").append(input.getAuditContext());
+      }
+
+      /////////////////////////////////////////////////////////////////////////////////////
+      // note process label (and a possible context from the process's state) if present //
+      /////////////////////////////////////////////////////////////////////////////////////
+      Optional<AbstractActionInput> actionInput = QContext.getFirstActionInStack();
+      if(actionInput.isPresent() && actionInput.get() instanceof RunProcessInput runProcessInput)
+      {
+         String processAuditContext = ValueUtils.getValueAsString(runProcessInput.getValue(AUDIT_CONTEXT_FIELD_NAME));
+         if(StringUtils.hasContent(processAuditContext))
+         {
+            contextSuffix.append(" ").append(processAuditContext);
+         }
+
+         String           processName = runProcessInput.getProcessName();
+         QProcessMetaData process     = QContext.getQInstance().getProcess(processName);
+         if(process != null)
+         {
+            contextSuffix.append(" during process: ").append(process.getLabel());
+         }
+      }
+
+      ///////////////////////////////////////////////////
+      // use api label & version if present in session //
+      ///////////////////////////////////////////////////
+      QSession qSession   = QContext.getQSession();
+      String   apiVersion = qSession.getValue("apiVersion");
+      if(apiVersion != null)
+      {
+         String apiLabel = qSession.getValue("apiLabel");
+         if(!StringUtils.hasContent(apiLabel))
+         {
+            apiLabel = "API";
+         }
+         contextSuffix.append(" via ").append(apiLabel).append(" Version: ").append(apiVersion);
+      }
+      return (contextSuffix.toString());
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   static Optional<QRecord> makeAuditDetailRecordForField(String fieldName, QTableMetaData table, DMLType dmlType, QRecord record, QRecord oldRecord)
+   {
+      if(!record.getValues().containsKey(fieldName))
+      {
+         ////////////////////////////////////////////////////////////////////////////////////////////////
+         // if the stored record doesn't have this field name, then don't audit anything about it      //
+         // this is to deal with our Patch style updates not looking like every field was cleared out. //
+         ////////////////////////////////////////////////////////////////////////////////////////////////
+         return (Optional.empty());
+      }
+
+      if(fieldName.equals("modifyDate") || fieldName.equals("createDate") || fieldName.equals("automationStatus"))
+      {
+         return (Optional.empty());
+      }
+
+      QFieldMetaData field        = table.getField(fieldName);
+      Serializable   value        = ValueUtils.getValueAsFieldType(field.getType(), record.getValue(fieldName));
+      Serializable   oldValue     = oldRecord == null ? null : ValueUtils.getValueAsFieldType(field.getType(), oldRecord.getValue(fieldName));
+      QRecord        detailRecord = null;
+
+      if(oldRecord == null)
+      {
+         if(DMLType.INSERT.equals(dmlType) && value == null)
+         {
+            return (Optional.empty());
+         }
+
+         if(field.getType().equals(QFieldType.BLOB) || field.getType().needsMasked())
+         {
+            detailRecord = new QRecord().withValue("message", "Set " + field.getLabel());
+         }
+         else
+         {
+            String formattedValue = getFormattedValueForAuditDetail(record, fieldName, field, value);
+            detailRecord = new QRecord().withValue("message", "Set " + field.getLabel() + " to " + formattedValue);
+            detailRecord.withValue("newValue", formattedValue);
+         }
+      }
+      else
+      {
+         if(areValuesDifferentForAudit(field, value, oldValue))
+         {
+            if(field.getType().equals(QFieldType.BLOB) || field.getType().needsMasked())
+            {
+               if(oldValue == null)
+               {
+                  detailRecord = new QRecord().withValue("message", "Set " + field.getLabel());
+               }
+               else if(value == null)
+               {
+                  detailRecord = new QRecord().withValue("message", "Removed " + field.getLabel());
+               }
+               else
+               {
+                  detailRecord = new QRecord().withValue("message", "Changed " + field.getLabel());
+               }
+            }
+            else
+            {
+               String formattedValue    = getFormattedValueForAuditDetail(record, fieldName, field, value);
+               String formattedOldValue = getFormattedValueForAuditDetail(oldRecord, fieldName, field, oldValue);
+
+               if(oldValue == null)
+               {
+                  detailRecord = new QRecord().withValue("message", "Set " + field.getLabel() + " to " + formatFormattedValueForDetailMessage(field, formattedValue));
+                  detailRecord.withValue("newValue", formattedValue);
+               }
+               else if(value == null)
+               {
+                  detailRecord = new QRecord().withValue("message", "Removed " + formatFormattedValueForDetailMessage(field, formattedOldValue) + " from " + field.getLabel());
+                  detailRecord.withValue("oldValue", formattedOldValue);
+               }
+               else
+               {
+                  detailRecord = new QRecord().withValue("message", "Changed " + field.getLabel() + " from " + formatFormattedValueForDetailMessage(field, formattedOldValue) + " to " + formatFormattedValueForDetailMessage(field, formattedValue));
+                  detailRecord.withValue("oldValue", formattedOldValue);
+                  detailRecord.withValue("newValue", formattedValue);
+               }
+            }
+         }
+      }
+
+      if(detailRecord != null)
+      {
+         ////////////////////////////////////////////////////////////////////
+         // useful if doing dev in here - but overkill for any other time. //
+         ////////////////////////////////////////////////////////////////////
+         // LOG.debug("Returning with message: " + detailRecord.getValueString("message"));
+         detailRecord.withValue("fieldName", fieldName);
+         return (Optional.of(detailRecord));
+      }
+
+      return (Optional.empty());
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   static boolean areValuesDifferentForAudit(QFieldMetaData field, Serializable value, Serializable oldValue)
+   {
+      try
+      {
+         ///////////////////
+         // decimal rules //
+         ///////////////////
+         if(field.getType().equals(QFieldType.DECIMAL))
+         {
+            BigDecimal newBD = ValueUtils.getValueAsBigDecimal(value);
+            BigDecimal oldBD = ValueUtils.getValueAsBigDecimal(oldValue);
+
+            if(newBD == null && oldBD == null)
+            {
+               return (false);
+            }
+
+            if(newBD == null || oldBD == null)
+            {
+               return (true);
+            }
+
+            return (newBD.compareTo(oldBD) != 0);
+         }
+
+         ////////////////////
+         // dateTime rules //
+         ////////////////////
+         if(field.getType().equals(QFieldType.DATE_TIME))
+         {
+            Instant newI = ValueUtils.getValueAsInstant(value);
+            Instant oldI = ValueUtils.getValueAsInstant(oldValue);
+
+            if(newI == null && oldI == null)
+            {
+               return (false);
+            }
+
+            if(newI == null || oldI == null)
+            {
+               return (true);
+            }
+
+            ////////////////////////////////
+            // just compare to the second //
+            ////////////////////////////////
+            return (newI.truncatedTo(ChronoUnit.SECONDS).compareTo(oldI.truncatedTo(ChronoUnit.SECONDS)) != 0);
+         }
+
+         //////////////////
+         // string rules //
+         //////////////////
+         if(field.getType().isStringLike())
+         {
+            String newString = ValueUtils.getValueAsString(value);
+            String oldString = ValueUtils.getValueAsString(oldValue);
+
+            boolean newIsNullOrEmpty = !StringUtils.hasContent(newString);
+            boolean oldIsNullOrEmpty = !StringUtils.hasContent(oldString);
+
+            if(newIsNullOrEmpty && oldIsNullOrEmpty)
+            {
+               return (false);
+            }
+
+            if(newIsNullOrEmpty || oldIsNullOrEmpty)
+            {
+               return (true);
+            }
+
+            return (newString.compareTo(oldString) != 0);
+         }
+
+         /////////////////////////////////////
+         // default just use Objects.equals //
+         /////////////////////////////////////
+         return !Objects.equals(oldValue, value);
+      }
+      catch(Exception e)
+      {
+         LOG.debug("Error checking areValuesDifferentForAudit", e, logPair("fieldName", field.getName()), logPair("value", value), logPair("oldValue", oldValue));
+      }
+
+      ////////////////////////////////////
+      // default to something simple... //
+      ////////////////////////////////////
+      return !Objects.equals(oldValue, value);
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/
@@ -373,21 +515,6 @@ public class DMLAuditAction extends AbstractQActionFunction<DMLAuditInput, DMLAu
 
 
 
-   /*******************************************************************************
-    **
-    *******************************************************************************/
-   private static Map<String, Serializable> getRecordSecurityKeyValues(QTableMetaData table, QRecord record)
-   {
-      Map<String, Serializable> securityKeyValues = new HashMap<>();
-      for(RecordSecurityLock recordSecurityLock : RecordSecurityLockFilters.filterForReadLocks(CollectionUtils.nonNullList(table.getRecordSecurityLocks())))
-      {
-         securityKeyValues.put(recordSecurityLock.getSecurityKeyType(), record == null ? null : record.getValue(recordSecurityLock.getFieldName()));
-      }
-      return securityKeyValues;
-   }
-
-
-
    /*******************************************************************************
     **
     *******************************************************************************/
@@ -407,7 +534,7 @@ public class DMLAuditAction extends AbstractQActionFunction<DMLAuditInput, DMLAu
    /*******************************************************************************
     **
     *******************************************************************************/
-   private enum DMLType
+   enum DMLType
    {
       INSERT("Inserted", true),
       UPDATE("Edited", true),

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/automation/RecordAutomationStatusUpdater.java

@@ -22,9 +22,8 @@
 package com.kingsrook.qqq.backend.core.actions.automation;
 
 
-import java.util.ArrayList;
+import java.util.Collections;
 import java.util.List;
-import java.util.Objects;
 import com.kingsrook.qqq.backend.core.actions.tables.CountAction;
 import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction;
 import com.kingsrook.qqq.backend.core.context.QContext;
@@ -90,6 +89,10 @@ public class RecordAutomationStatusUpdater
          }
       }
 
+      ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      // Avoid setting records to PENDING_INSERT or PENDING_UPDATE even if they don't have any insert or update automations or triggers //
+      // such records should go straight to OK status.                                                                                  //
+      ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
       if(canWeSkipPendingAndGoToOkay(table, automationStatus))
       {
          automationStatus = AutomationStatus.OK;
@@ -121,9 +124,13 @@ public class RecordAutomationStatusUpdater
     ** being asked to set status to PENDING_INSERT (or PENDING_UPDATE), then just
     ** move the status straight to OK.
     *******************************************************************************/
-   private static boolean canWeSkipPendingAndGoToOkay(QTableMetaData table, AutomationStatus automationStatus)
+   static boolean canWeSkipPendingAndGoToOkay(QTableMetaData table, AutomationStatus automationStatus)
    {
-      List<TableAutomationAction> tableActions = Objects.requireNonNullElse(table.getAutomationDetails().getActions(), new ArrayList<>());
+      List<TableAutomationAction> tableActions = Collections.emptyList();
+      if(table.getAutomationDetails() != null && table.getAutomationDetails().getActions() != null)
+      {
+         tableActions = table.getAutomationDetails().getActions();
+      }
 
       if(automationStatus.equals(AutomationStatus.PENDING_INSERT_AUTOMATIONS))
       {
@@ -135,6 +142,12 @@ public class RecordAutomationStatusUpdater
          {
             return (false);
          }
+
+         ////////////////////////////////////////////////////////////////////////////////////////
+         // if we're going to pending-insert, and there are no insert automations or triggers, //
+         // then we may skip pending and go to okay.                                           //
+         ////////////////////////////////////////////////////////////////////////////////////////
+         return (true);
       }
       else if(automationStatus.equals(AutomationStatus.PENDING_UPDATE_AUTOMATIONS))
       {
@@ -146,9 +159,21 @@ public class RecordAutomationStatusUpdater
          {
             return (false);
          }
-      }
 
-      return (true);
+         ////////////////////////////////////////////////////////////////////////////////////////
+         // if we're going to pending-update, and there are no insert automations or triggers, //
+         // then we may skip pending and go to okay.                                           //
+         ////////////////////////////////////////////////////////////////////////////////////////
+         return (true);
+      }
+      else
+      {
+         ///////////////////////////////////////////////////////////////////////////////////////////////////////
+         // if we're going to any other automation status - then we may never "skip pending" and go to okay - //
+         // because we weren't asked to go to pending!                                                        //
+         ///////////////////////////////////////////////////////////////////////////////////////////////////////
+         return (false);
+      }
    }
 
 

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/automation/polling/PollingAutomationPerTableRunner.java

@@ -342,22 +342,9 @@ public class PollingAutomationPerTableRunner implements Runnable
       boolean anyActionsFailed = false;
       for(TableAutomationAction action : actions)
       {
-         try
+         boolean hadError = applyActionToRecords(table, records, action);
+         if(hadError)
          {
-            //////////////////////////////////////////////////////////////////////////////////////////////////////////////
-            // note - this method - will re-query the objects, so we should have confidence that their data is fresh... //
-            //////////////////////////////////////////////////////////////////////////////////////////////////////////////
-            List<QRecord> matchingQRecords = getRecordsMatchingActionFilter(table, records, action);
-            LOG.debug("Of the {} records that were pending automations, {} of them match the filter on the action {}", records.size(), matchingQRecords.size(), action);
-            if(CollectionUtils.nullSafeHasContents(matchingQRecords))
-            {
-               LOG.debug("  Processing " + matchingQRecords.size() + " records in " + table + " for action " + action);
-               applyActionToMatchingRecords(table, matchingQRecords, action);
-            }
-         }
-         catch(Exception e)
-         {
-            LOG.warn("Caught exception processing records on " + table + " for action " + action, e);
             anyActionsFailed = true;
          }
       }
@@ -377,6 +364,37 @@ public class PollingAutomationPerTableRunner implements Runnable
 
 
 
+   /*******************************************************************************
+    ** Run one action over a list of records (if they match the action's filter).
+    **
+    ** @return hadError - true if an exception was caught; false if all OK.
+    *******************************************************************************/
+   protected boolean applyActionToRecords(QTableMetaData table, List<QRecord> records, TableAutomationAction action)
+   {
+      try
+      {
+         //////////////////////////////////////////////////////////////////////////////////////////////////////////////
+         // note - this method - will re-query the objects, so we should have confidence that their data is fresh... //
+         //////////////////////////////////////////////////////////////////////////////////////////////////////////////
+         List<QRecord> matchingQRecords = getRecordsMatchingActionFilter(table, records, action);
+         LOG.debug("Of the {} records that were pending automations, {} of them match the filter on the action {}", records.size(), matchingQRecords.size(), action);
+         if(CollectionUtils.nullSafeHasContents(matchingQRecords))
+         {
+            LOG.debug("  Processing " + matchingQRecords.size() + " records in " + table + " for action " + action);
+            applyActionToMatchingRecords(table, matchingQRecords, action);
+         }
+
+         return (false);
+      }
+      catch(Exception e)
+      {
+         LOG.warn("Caught exception processing records on " + table + " for action " + action, e);
+         return (true);
+      }
+   }
+
+
+
    /*******************************************************************************
     ** For a given action, and a list of records - return a new list, of the ones
     ** which match the action's filter (if there is one - if not, then all match).

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/AbstractPreInsertCustomizer.java

@@ -55,6 +55,21 @@ public abstract class AbstractPreInsertCustomizer
 
 
 
+   /////////////////////////////////////////////////////////////////////////////////
+   // allow the customizer to specify when it should be executed as part of the   //
+   // insert action.  default (per method in this class) is AFTER_ALL_VALIDATIONS //
+   /////////////////////////////////////////////////////////////////////////////////
+   public enum WhenToRun
+   {
+      BEFORE_ALL_VALIDATIONS,
+      BEFORE_UNIQUE_KEY_CHECKS,
+      BEFORE_REQUIRED_FIELD_CHECKS,
+      BEFORE_SECURITY_CHECKS,
+      AFTER_ALL_VALIDATIONS
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/
@@ -62,6 +77,16 @@ public abstract class AbstractPreInsertCustomizer
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public WhenToRun getWhenToRun()
+   {
+      return (WhenToRun.AFTER_ALL_VALIDATIONS);
+   }
+
+
+
    /*******************************************************************************
     ** Getter for insertInput
     **

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/dashboard/widgets/AbstractWidgetRenderer.java

@@ -46,6 +46,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleVal
 import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSource;
 import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
 import com.kingsrook.qqq.backend.core.utils.StringUtils;
+import com.kingsrook.qqq.backend.core.utils.collections.MapBuilder;
 
 
 /*******************************************************************************
@@ -123,7 +124,7 @@ public abstract class AbstractWidgetRenderer
          output.getResults().removeIf(pvs -> !exists.add(pvs.getLabel()));
          for(QPossibleValue<?> possibleValue : output.getResults())
          {
-            dropdownOptionList.add(Map.of(
+            dropdownOptionList.add(MapBuilder.of(
                "id", String.valueOf(possibleValue.getId()),
                "label", possibleValue.getLabel()
             ));

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/dashboard/widgets/DateTimeGroupBy.java

@@ -99,7 +99,17 @@ public enum DateTimeGroupBy
    public String getSqlExpression()
    {
       ZoneId sessionOrInstanceZoneId = ValueUtils.getSessionOrInstanceZoneId();
-      String targetTimezone          = sessionOrInstanceZoneId.toString();
+      return (getSqlExpression(sessionOrInstanceZoneId));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public String getSqlExpression(ZoneId targetZoneId)
+   {
+      String targetTimezone = targetZoneId.toString();
 
       if("Z".equals(targetTimezone) || !StringUtils.hasContent(targetTimezone))
       {
@@ -158,7 +168,18 @@ public enum DateTimeGroupBy
     *******************************************************************************/
    public String makeSelectedString(Instant time)
    {
-      ZonedDateTime zoned = time.atZone(ValueUtils.getSessionOrInstanceZoneId());
+      return (makeSelectedString(time, ValueUtils.getSessionOrInstanceZoneId()));
+   }
+
+
+
+   /*******************************************************************************
+    ** Make an Instant into a string that will match what came out of the database's
+    ** DATE_FORMAT() function
+    *******************************************************************************/
+   public String makeSelectedString(Instant time, ZoneId zoneId)
+   {
+      ZonedDateTime zoned = time.atZone(zoneId);
 
       if(this == WEEK)
       {
@@ -182,7 +203,17 @@ public enum DateTimeGroupBy
     *******************************************************************************/
    public String makeHumanString(Instant instant)
    {
-      ZonedDateTime zoned = instant.atZone(ValueUtils.getSessionOrInstanceZoneId());
+      return (makeHumanString(instant, ValueUtils.getSessionOrInstanceZoneId()));
+   }
+
+
+
+   /*******************************************************************************
+    ** Make a string to show to a user
+    *******************************************************************************/
+   public String makeHumanString(Instant instant, ZoneId zoneId)
+   {
+      ZonedDateTime zoned = instant.atZone(zoneId);
       if(this.equals(WEEK))
       {
          DateTimeFormatter dateTimeFormatter = DateTimeFormatter.ofPattern("M'/'d");
@@ -215,25 +246,35 @@ public enum DateTimeGroupBy
    /*******************************************************************************
     **
     *******************************************************************************/
-   @SuppressWarnings("checkstyle:indentation")
    public Instant roundDown(Instant instant)
    {
-      ZonedDateTime zoned = instant.atZone(ValueUtils.getSessionOrInstanceZoneId());
+      return roundDown(instant, ValueUtils.getSessionOrInstanceZoneId());
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @SuppressWarnings("checkstyle:indentation")
+   public Instant roundDown(Instant instant, ZoneId zoneId)
+   {
+      ZonedDateTime zoned = instant.atZone(zoneId);
       return switch(this)
+      {
+         case YEAR -> zoned.with(TemporalAdjusters.firstDayOfYear()).truncatedTo(ChronoUnit.DAYS).toInstant();
+         case MONTH -> zoned.with(TemporalAdjusters.firstDayOfMonth()).truncatedTo(ChronoUnit.DAYS).toInstant();
+         case WEEK ->
          {
-            case YEAR -> zoned.with(TemporalAdjusters.firstDayOfYear()).truncatedTo(ChronoUnit.DAYS).toInstant();
-            case MONTH -> zoned.with(TemporalAdjusters.firstDayOfMonth()).truncatedTo(ChronoUnit.DAYS).toInstant();
-            case WEEK ->
+            while(zoned.get(ChronoField.DAY_OF_WEEK) != DayOfWeek.SUNDAY.getValue())
             {
-               while(zoned.get(ChronoField.DAY_OF_WEEK) != DayOfWeek.SUNDAY.getValue())
-               {
-                  zoned = zoned.minusDays(1);
-               }
-               yield (zoned.truncatedTo(ChronoUnit.DAYS).toInstant());
+               zoned = zoned.minusDays(1);
             }
-            case DAY -> zoned.truncatedTo(ChronoUnit.DAYS).toInstant();
-            case HOUR -> zoned.truncatedTo(ChronoUnit.HOURS).toInstant();
-         };
+            yield (zoned.truncatedTo(ChronoUnit.DAYS).toInstant());
+         }
+         case DAY -> zoned.truncatedTo(ChronoUnit.DAYS).toInstant();
+         case HOUR -> zoned.truncatedTo(ChronoUnit.HOURS).toInstant();
+      };
    }
 
 
@@ -243,7 +284,17 @@ public enum DateTimeGroupBy
     *******************************************************************************/
    public Instant increment(Instant instant)
    {
-      ZonedDateTime zoned = instant.atZone(ValueUtils.getSessionOrInstanceZoneId());
+      return (increment(instant, ValueUtils.getSessionOrInstanceZoneId()));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public Instant increment(Instant instant, ZoneId zoneId)
+   {
+      ZonedDateTime zoned = instant.atZone(zoneId);
       return (zoned.plus(noOfChronoUnitsToAdd, chronoUnitToAdd).toInstant());
    }
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/dashboard/widgets/ParentWidgetRenderer.java

@@ -60,6 +60,8 @@ public class ParentWidgetRenderer extends AbstractWidgetRenderer
             widgetData.setChildWidgetNameList(metaData.getChildWidgetNameList());
          }
 
+         widgetData.setLayoutType(metaData.getLayoutType());
+
          return (new RenderWidgetOutput(widgetData));
       }
       catch(Exception e)

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/interfaces/GetInterface.java

@@ -28,6 +28,7 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetOutput;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.UniqueKey;
+import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
 
 
 /*******************************************************************************
@@ -55,7 +56,7 @@ public interface GetInterface
       {
          QTableMetaData table      = getInput.getTable();
          boolean        foundMatch = false;
-         for(UniqueKey uniqueKey : table.getUniqueKeys())
+         for(UniqueKey uniqueKey : CollectionUtils.nonNullList(table.getUniqueKeys()))
          {
             if(new HashSet<>(uniqueKey.getFieldNames()).equals(getInput.getUniqueKey().keySet()))
             {

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/ExportAction.java

@@ -197,7 +197,27 @@ public class ExportAction
                String   joinTableName = parts[0];
                if(!addedJoinNames.contains(joinTableName))
                {
-                  queryJoins.add(new QueryJoin(joinTableName).withType(QueryJoin.Type.LEFT).withSelect(true));
+                  QueryJoin queryJoin = new QueryJoin(joinTableName).withType(QueryJoin.Type.LEFT).withSelect(true);
+                  queryJoins.add(queryJoin);
+
+                  /////////////////////////////////////////////////////////////////////////////////////////////
+                  // in at least some cases, we need to let the queryJoin know what join-meta-data to use... //
+                  // This code basically mirrors what QFMD is doing right now, so it's better -              //
+                  // but shouldn't all of this just be in JoinsContext?  it does some of this...             //
+                  /////////////////////////////////////////////////////////////////////////////////////////////
+                  QTableMetaData        table               = exportInput.getTable();
+                  Optional<ExposedJoin> exposedJoinOptional = CollectionUtils.nonNullList(table.getExposedJoins()).stream().filter(ej -> ej.getJoinTable().equals(joinTableName)).findFirst();
+                  if(exposedJoinOptional.isEmpty())
+                  {
+                     throw (new QException("Could not find exposed join between base table " + table.getName() + " and requested join table " + joinTableName));
+                  }
+                  ExposedJoin exposedJoin = exposedJoinOptional.get();
+
+                  if(exposedJoin.getJoinPath().size() == 1)
+                  {
+                     queryJoin.setJoinMetaData(QContext.getQInstance().getJoin(exposedJoin.getJoinPath().get(exposedJoin.getJoinPath().size() - 1)));
+                  }
+
                   addedJoinNames.add(joinTableName);
                }
             }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/scripts/ExecuteCodeAction.java

@@ -33,6 +33,7 @@ import com.kingsrook.qqq.backend.core.actions.scripts.logging.QCodeExecutionLogg
 import com.kingsrook.qqq.backend.core.actions.scripts.logging.ScriptExecutionLoggerInterface;
 import com.kingsrook.qqq.backend.core.actions.scripts.logging.StoreScriptLogAndScriptLogLineExecutionLogger;
 import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
+import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.exceptions.QCodeException;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.logging.QLogger;
@@ -50,6 +51,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeType;
 import com.kingsrook.qqq.backend.core.model.scripts.ScriptRevision;
 import com.kingsrook.qqq.backend.core.model.scripts.ScriptRevisionFile;
 import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
+import com.kingsrook.qqq.backend.core.utils.ObjectUtils;
 import com.kingsrook.qqq.backend.core.utils.StringUtils;
 
 
@@ -112,6 +114,11 @@ public class ExecuteCodeAction
             context.putAll(input.getInput());
          }
 
+         //////////////////////////////////////////
+         // safely always set the deploymentMode //
+         //////////////////////////////////////////
+         context.put("deploymentMode", ObjectUtils.tryAndRequireNonNullElse(() -> QContext.getQInstance().getDeploymentMode(), null));
+
          /////////////////////////////////////////////////////////////////////////////////
          // set the qCodeExecutor into any context objects which are QCodeExecutorAware //
          /////////////////////////////////////////////////////////////////////////////////

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/scripts/RunAssociatedScriptAction.java

@@ -68,6 +68,7 @@ public class RunAssociatedScriptAction
       new ExecuteCodeAction().run(executeCodeInput, executeCodeOutput);
 
       output.setOutput(executeCodeOutput.getOutput());
+      output.setScriptRevisionId(scriptRevision.getId());
    }
 
 

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/InsertAction.java

@@ -193,25 +193,76 @@ public class InsertAction extends AbstractQActionFunction<InsertInput, InsertOut
    {
       QTableMetaData table = insertInput.getTable();
 
-      ValueBehaviorApplier.applyFieldBehaviors(insertInput.getInstance(), table, insertInput.getRecords());
-      setErrorsIfUniqueKeyErrors(insertInput, table);
-
-      if(insertInput.getInputSource().shouldValidateRequiredFields())
-      {
-         validateRequiredFields(insertInput);
-      }
-
-      ValidateRecordSecurityLockHelper.validateSecurityFields(insertInput.getTable(), insertInput.getRecords(), ValidateRecordSecurityLockHelper.Action.INSERT);
-
-      ///////////////////////////////////////////////////////////////////////////
-      // after all validations, run the pre-insert customizer, if there is one //
-      ///////////////////////////////////////////////////////////////////////////
+      ///////////////////////////////////////////////////////////////////
+      // load the pre-insert customizer and set it up, if there is one //
+      // then we'll run it based on its WhenToRun value                //
+      ///////////////////////////////////////////////////////////////////
       Optional<AbstractPreInsertCustomizer> preInsertCustomizer = QCodeLoader.getTableCustomizer(AbstractPreInsertCustomizer.class, table, TableCustomizers.PRE_INSERT_RECORD.getRole());
       if(preInsertCustomizer.isPresent())
       {
          preInsertCustomizer.get().setInsertInput(insertInput);
          preInsertCustomizer.get().setIsPreview(isPreview);
-         insertInput.setRecords(preInsertCustomizer.get().apply(insertInput.getRecords()));
+         runPreInsertCustomizerIfItIsTime(insertInput, preInsertCustomizer, AbstractPreInsertCustomizer.WhenToRun.BEFORE_ALL_VALIDATIONS);
+      }
+
+      setDefaultValuesInRecords(table, insertInput.getRecords());
+
+      ValueBehaviorApplier.applyFieldBehaviors(insertInput.getInstance(), table, insertInput.getRecords());
+
+      runPreInsertCustomizerIfItIsTime(insertInput, preInsertCustomizer, AbstractPreInsertCustomizer.WhenToRun.BEFORE_UNIQUE_KEY_CHECKS);
+      setErrorsIfUniqueKeyErrors(insertInput, table);
+
+      runPreInsertCustomizerIfItIsTime(insertInput, preInsertCustomizer, AbstractPreInsertCustomizer.WhenToRun.BEFORE_REQUIRED_FIELD_CHECKS);
+      if(insertInput.getInputSource().shouldValidateRequiredFields())
+      {
+         validateRequiredFields(insertInput);
+      }
+
+      runPreInsertCustomizerIfItIsTime(insertInput, preInsertCustomizer, AbstractPreInsertCustomizer.WhenToRun.BEFORE_SECURITY_CHECKS);
+      ValidateRecordSecurityLockHelper.validateSecurityFields(insertInput.getTable(), insertInput.getRecords(), ValidateRecordSecurityLockHelper.Action.INSERT);
+
+      runPreInsertCustomizerIfItIsTime(insertInput, preInsertCustomizer, AbstractPreInsertCustomizer.WhenToRun.AFTER_ALL_VALIDATIONS);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private void setDefaultValuesInRecords(QTableMetaData table, List<QRecord> records)
+   {
+      ////////////////////////////////////////////////////////////////////////////////////////////////
+      // for all fields in the table - if any have a default value, then look at all input records, //
+      // and if they have null value, then apply the default                                        //
+      ////////////////////////////////////////////////////////////////////////////////////////////////
+      for(QFieldMetaData field : table.getFields().values())
+      {
+         if(field.getDefaultValue() != null)
+         {
+            for(QRecord record : records)
+            {
+               if(record.getValue(field.getName()) == null)
+               {
+                  record.setValue(field.getName(), field.getDefaultValue());
+               }
+            }
+         }
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private void runPreInsertCustomizerIfItIsTime(InsertInput insertInput, Optional<AbstractPreInsertCustomizer> preInsertCustomizer, AbstractPreInsertCustomizer.WhenToRun whenToRun) throws QException
+   {
+      if(preInsertCustomizer.isPresent())
+      {
+         if(whenToRun.equals(preInsertCustomizer.get().getWhenToRun()))
+         {
+            insertInput.setRecords(preInsertCustomizer.get().apply(insertInput.getRecords()));
+         }
       }
    }
 

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/ReplaceAction.java

@@ -31,6 +31,7 @@ import com.kingsrook.qqq.backend.core.actions.AbstractQActionFunction;
 import com.kingsrook.qqq.backend.core.actions.QBackendTransaction;
 import com.kingsrook.qqq.backend.core.actions.tables.helpers.UniqueKeyHelper;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.logging.QLogger;
 import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteOutput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
@@ -61,6 +62,9 @@ import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
  *******************************************************************************/
 public class ReplaceAction extends AbstractQActionFunction<ReplaceInput, ReplaceOutput>
 {
+   private static final QLogger LOG = QLogger.getLogger(ReplaceAction.class);
+
+
 
    /*******************************************************************************
     **
@@ -159,6 +163,7 @@ public class ReplaceAction extends AbstractQActionFunction<ReplaceInput, Replace
       {
          if(weOwnTheTransaction)
          {
+            LOG.warn("Caught top-level ReplaceAction exception - rolling back exception", e);
             transaction.rollback();
          }
          throw (new QException("Error executing replace action", e));

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/QPossibleValueTranslator.java

@@ -32,11 +32,14 @@ import java.util.List;
 import java.util.Map;
 import java.util.Objects;
 import java.util.Set;
+import com.kingsrook.qqq.backend.core.actions.QBackendTransaction;
 import com.kingsrook.qqq.backend.core.actions.customizers.QCodeLoader;
+import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
 import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
 import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.exceptions.QValueException;
 import com.kingsrook.qqq.backend.core.logging.QLogger;
+import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
@@ -74,9 +77,47 @@ public class QPossibleValueTranslator
    ///////////////////////////////////////////////////////
    private Map<String, Map<Serializable, String>> possibleValueCache = new HashMap<>();
 
+   private int maxSizePerPvsCache = 50_000;
+
+   private Map<String, QBackendTransaction> transactionsPerTable = new HashMap<>();
+
    // todo not commit - remove instance & session - use Context
 
 
+   boolean useTransactionsAsConnectionPool = false;
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private QBackendTransaction getTransaction(String tableName)
+   {
+      /////////////////////////////////////////////////////////////
+      // mmm, this does cut down on connections used -           //
+      // especially seems helpful in big exports.                //
+      // but, let's just start using connection pools instead... //
+      /////////////////////////////////////////////////////////////
+      if(useTransactionsAsConnectionPool)
+      {
+         try
+         {
+            if(!transactionsPerTable.containsKey(tableName))
+            {
+               transactionsPerTable.put(tableName, new InsertAction().openTransaction(new InsertInput(tableName)));
+            }
+
+            return (transactionsPerTable.get(tableName));
+         }
+         catch(Exception e)
+         {
+            LOG.warn("Error opening transaction for table", logPair("tableName", tableName));
+         }
+      }
+
+      return null;
+   }
+
 
    /*******************************************************************************
     ** Constructor
@@ -425,9 +466,10 @@ public class QPossibleValueTranslator
       for(Map.Entry<String, Map<Serializable, String>> entry : possibleValueCache.entrySet())
       {
          int size = entry.getValue().size();
-         if(size > 50_000)
+         if(size > maxSizePerPvsCache)
          {
             LOG.info("Found a big PVS cache - clearing it.", logPair("name", entry.getKey()), logPair("size", size));
+            entry.getValue().clear();
          }
       }
 
@@ -521,6 +563,7 @@ public class QPossibleValueTranslator
             QueryInput queryInput = new QueryInput();
             queryInput.setTableName(tableName);
             queryInput.setFilter(new QQueryFilter().withCriteria(new QFilterCriteria(primaryKeyField, QCriteriaOperator.IN, page)));
+            queryInput.setTransaction(getTransaction(tableName));
 
             /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
             // when querying for possible values, we do want to generate their display values, which makes record labels, which are usually used as PVS labels //
@@ -613,4 +656,24 @@ public class QPossibleValueTranslator
       return (count < 5);
    }
 
+
+
+   /*******************************************************************************
+    ** Getter for maxSizePerPvsCache
+    *******************************************************************************/
+   public int getMaxSizePerPvsCache()
+   {
+      return (this.maxSizePerPvsCache);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for maxSizePerPvsCache
+    *******************************************************************************/
+   public void setMaxSizePerPvsCache(int maxSizePerPvsCache)
+   {
+      this.maxSizePerPvsCache = maxSizePerPvsCache;
+   }
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/QValueFormatter.java

@@ -112,6 +112,12 @@ public class QValueFormatter
          {
             return formatLocalTime(lt);
          }
+
+         //////////////////////////////////////////////////////////////////////////////////////////
+         // else, just return the value as a string, rather than going through String.formatted  //
+         // this saves some overhead incurred by String.formatted when called millions of times. //
+         //////////////////////////////////////////////////////////////////////////////////////////
+         return (ValueUtils.getValueAsString(value));
       }
 
       ////////////////////////////////////////////////////////
@@ -268,6 +274,14 @@ public class QValueFormatter
     *******************************************************************************/
    private static String formatRecordLabelExceptionalCases(QTableMetaData table, QRecord record)
    {
+      //////////////////////////////////////////////////////////////////////////////////////
+      // if the record already has a label (say, from a query-customizer), then return it //
+      //////////////////////////////////////////////////////////////////////////////////////
+      if(record.getRecordLabel() != null)
+      {
+         return (record.getRecordLabel());
+      }
+
       ///////////////////////////////////////////////////////////////////////////////////////
       // if there's no record label format, then just return the primary key display value //
       ///////////////////////////////////////////////////////////////////////////////////////

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/SearchPossibleValueSourceAction.java

@@ -23,6 +23,8 @@ package com.kingsrook.qqq.backend.core.actions.values;
 
 
 import java.io.Serializable;
+import java.time.Instant;
+import java.time.LocalDate;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Objects;
@@ -208,44 +210,55 @@ public class SearchPossibleValueSourceAction
       }
       else
       {
-         if(StringUtils.hasContent(input.getSearchTerm()))
+         String searchTerm = input.getSearchTerm();
+         if(StringUtils.hasContent(searchTerm))
          {
             for(String valueField : possibleValueSource.getSearchFields())
             {
-               QFieldMetaData field = table.getField(valueField);
-               if(field.getType().equals(QFieldType.STRING))
+               try
                {
-                  queryFilter.addCriteria(new QFilterCriteria(valueField, QCriteriaOperator.STARTS_WITH, List.of(input.getSearchTerm())));
-               }
-               else if(field.getType().equals(QFieldType.DATE) || field.getType().equals(QFieldType.DATE_TIME))
-               {
-                  LOG.debug("Not querying PVS [" + possibleValueSource.getName() + "] on date field [" + field.getName() + "]");
-                  // todo - what? queryFilter.addCriteria(new QFilterCriteria(valueField, QCriteriaOperator.STARTS_WITH, List.of(input.getSearchTerm())));
-               }
-               else
-               {
-                  try
+                  QFieldMetaData field = table.getField(valueField);
+                  if(field.getType().equals(QFieldType.STRING))
                   {
-                     Integer valueAsInteger = ValueUtils.getValueAsInteger(input.getSearchTerm());
+                     queryFilter.addCriteria(new QFilterCriteria(valueField, QCriteriaOperator.STARTS_WITH, List.of(searchTerm)));
+                  }
+                  else if(field.getType().equals(QFieldType.DATE))
+                  {
+                     LocalDate searchDate = ValueUtils.getValueAsLocalDate(searchTerm);
+                     if(searchDate != null)
+                     {
+                        queryFilter.addCriteria(new QFilterCriteria(valueField, QCriteriaOperator.EQUALS, searchDate));
+                     }
+                  }
+                  else if(field.getType().equals(QFieldType.DATE_TIME))
+                  {
+                     Instant searchDate = ValueUtils.getValueAsInstant(searchTerm);
+                     if(searchDate != null)
+                     {
+                        queryFilter.addCriteria(new QFilterCriteria(valueField, QCriteriaOperator.EQUALS, searchDate));
+                     }
+                  }
+                  else
+                  {
+                     Integer valueAsInteger = ValueUtils.getValueAsInteger(searchTerm);
                      if(valueAsInteger != null)
                      {
                         queryFilter.addCriteria(new QFilterCriteria(valueField, QCriteriaOperator.EQUALS, List.of(valueAsInteger)));
                      }
                   }
-                  catch(Exception e)
-                  {
-                     ////////////////////////////////////////////////////////
-                     // write a FALSE criteria if the value isn't a number //
-                     ////////////////////////////////////////////////////////
-                     queryFilter.addCriteria(new QFilterCriteria(valueField, QCriteriaOperator.IN, List.of()));
-                  }
+               }
+               catch(Exception e)
+               {
+                  //////////////////////////////////////////////////////////////////////////////////////////
+                  // write a FALSE criteria upon exceptions (e.g., type conversion fails)                 //
+                  // Why are we doing this?  so a single-field query finds nothing instead of everything. //
+                  //////////////////////////////////////////////////////////////////////////////////////////
+                  queryFilter.addCriteria(new QFilterCriteria(valueField, QCriteriaOperator.IN, List.of()));
                }
             }
          }
       }
 
-      queryFilter.setOrderBys(possibleValueSource.getOrderByFields());
-
       // todo - skip & limit as params
       queryFilter.setLimit(250);
 
@@ -257,6 +270,9 @@ public class SearchPossibleValueSourceAction
          input.getDefaultQueryFilter().addSubFilter(queryFilter);
          queryFilter = input.getDefaultQueryFilter();
       }
+
+      queryFilter.setOrderBys(possibleValueSource.getOrderByFields());
+
       queryInput.setFilter(queryFilter);
 
       QueryOutput             queryOutput     = new QueryAction().execute(queryInput);

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/adapters/CsvToQRecordAdapter.java

@@ -36,7 +36,9 @@ import com.kingsrook.qqq.backend.core.model.actions.shared.mapping.AbstractQFiel
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+import com.kingsrook.qqq.backend.core.model.statusmessages.BadInputStatusMessage;
 import com.kingsrook.qqq.backend.core.utils.StringUtils;
+import com.kingsrook.qqq.backend.core.utils.ValueUtils;
 import org.apache.commons.csv.CSVFormat;
 import org.apache.commons.csv.CSVParser;
 import org.apache.commons.csv.CSVRecord;
@@ -156,14 +158,21 @@ public class CsvToQRecordAdapter
                // now move values into the QRecord, using the mapping to get the 'header' corresponding to each QField //
                //////////////////////////////////////////////////////////////////////////////////////////////////////////
                QRecord qRecord = new QRecord();
-               for(QFieldMetaData field : table.getFields().values())
+               try
                {
-                  String fieldSource = mapping == null ? field.getName() : String.valueOf(mapping.getFieldSource(field.getName()));
-                  fieldSource = adjustHeaderCase(fieldSource, inputWrapper);
-                  qRecord.setValue(field.getName(), csvValues.get(fieldSource));
-               }
+                  for(QFieldMetaData field : table.getFields().values())
+                  {
+                     String fieldSource = mapping == null ? field.getName() : String.valueOf(mapping.getFieldSource(field.getName()));
+                     fieldSource = adjustHeaderCase(fieldSource, inputWrapper);
+                     setValue(inputWrapper, qRecord, field, csvValues.get(fieldSource));
+                  }
 
-               runRecordCustomizer(recordCustomizer, qRecord);
+                  runRecordCustomizer(recordCustomizer, qRecord);
+               }
+               catch(Exception e)
+               {
+                  qRecord.addError(new BadInputStatusMessage("Error parsing line #" + (recordCount + 1) + ": " + e.getMessage()));
+               }
                addRecord(qRecord);
 
                recordCount++;
@@ -202,13 +211,20 @@ public class CsvToQRecordAdapter
                // now move values into the QRecord, using the mapping to get the 'header' corresponding to each QField //
                //////////////////////////////////////////////////////////////////////////////////////////////////////////
                QRecord qRecord = new QRecord();
-               for(QFieldMetaData field : table.getFields().values())
+               try
                {
-                  Integer fieldIndex = (Integer) mapping.getFieldSource(field.getName());
-                  qRecord.setValue(field.getName(), csvValues.get(fieldIndex));
-               }
+                  for(QFieldMetaData field : table.getFields().values())
+                  {
+                     Integer fieldIndex = (Integer) mapping.getFieldSource(field.getName());
+                     setValue(inputWrapper, qRecord, field, csvValues.get(fieldIndex));
+                  }
 
-               runRecordCustomizer(recordCustomizer, qRecord);
+                  runRecordCustomizer(recordCustomizer, qRecord);
+               }
+               catch(Exception e)
+               {
+                  qRecord.addError(new BadInputStatusMessage("Error parsing line #" + (recordCount + 1) + ": " + e.getMessage()));
+               }
                addRecord(qRecord);
 
                recordCount++;
@@ -231,6 +247,23 @@ public class CsvToQRecordAdapter
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private void setValue(InputWrapper inputWrapper, QRecord qRecord, QFieldMetaData field, String valueString)
+   {
+      if(inputWrapper.doCorrectValueTypes)
+      {
+         qRecord.setValue(field.getName(), ValueUtils.getValueAsFieldType(field.getType(), valueString));
+      }
+      else
+      {
+         qRecord.setValue(field.getName(), valueString);
+      }
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/
@@ -341,6 +374,7 @@ public class CsvToQRecordAdapter
       private AbstractQFieldMapping<?> mapping;
       private Consumer<QRecord>        recordCustomizer;
       private Integer                  limit;
+      private boolean                  doCorrectValueTypes = false;
 
       private boolean caseSensitiveHeaders = false;
 
@@ -582,6 +616,40 @@ public class CsvToQRecordAdapter
          return (this);
       }
 
+
+
+      /*******************************************************************************
+       ** Getter for doCorrectValueTypes
+       **
+       *******************************************************************************/
+      public boolean getDoCorrectValueTypes()
+      {
+         return doCorrectValueTypes;
+      }
+
+
+
+      /*******************************************************************************
+       ** Setter for doCorrectValueTypes
+       **
+       *******************************************************************************/
+      public void setDoCorrectValueTypes(boolean doCorrectValueTypes)
+      {
+         this.doCorrectValueTypes = doCorrectValueTypes;
+      }
+
+
+
+      /*******************************************************************************
+       ** Fluent setter for doCorrectValueTypes
+       **
+       *******************************************************************************/
+      public InputWrapper withDoCorrectValueTypes(boolean doCorrectValueTypes)
+      {
+         this.doCorrectValueTypes = doCorrectValueTypes;
+         return (this);
+      }
+
    }
 
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/adapters/QRecordToCsvAdapter.java

@@ -80,8 +80,23 @@ public class QRecordToCsvAdapter
    /*******************************************************************************
     ** todo - kinda weak... can we find this in a CSV lib??
     *******************************************************************************/
-   private String sanitize(String value)
+   static String sanitize(String value)
    {
-      return (value.replaceAll("\"", "\"\"").replaceAll("\n", " "));
+      /////////////////////////////////////////////////////////////////////////////////////
+      // especially in big exports, we see a TON of memory allocated and CPU spent here, //
+      // if we just blindly replaceAll.  So, only do it if needed.                       //
+      /////////////////////////////////////////////////////////////////////////////////////
+      if(value.contains("\""))
+      {
+         value = value.replaceAll("\"", "\"\"");
+      }
+
+      if(value.contains("\n"))
+      {
+         value = value.replaceAll("\n", " ");
+      }
+
+      return (value);
    }
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/context/QContext.java

@@ -84,7 +84,7 @@ public class QContext
          actionStackThreadLocal.get().add(actionInput);
       }
 
-      if(!qInstance.getHasBeenValidated())
+      if(qInstance != null && !qInstance.getHasBeenValidated())
       {
          try
          {

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java

@@ -77,6 +77,7 @@ import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.Bulk
 import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.ExtractViaQueryStep;
 import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess;
 import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
+import com.kingsrook.qqq.backend.core.utils.ListingHash;
 import com.kingsrook.qqq.backend.core.utils.StringUtils;
 
 
@@ -531,11 +532,70 @@ public class QInstanceEnricher
          enrichAppSection(section);
       }
 
+      ensureAppSectionMembersAreAppChildren(app);
+
       enrichPermissionRules(app);
    }
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private void ensureAppSectionMembersAreAppChildren(QAppMetaData app)
+   {
+      ListingHash<Class<? extends QAppChildMetaData>, String> childrenByType = new ListingHash<>();
+      childrenByType.put(QTableMetaData.class, new ArrayList<>());
+      childrenByType.put(QProcessMetaData.class, new ArrayList<>());
+      childrenByType.put(QReportMetaData.class, new ArrayList<>());
+
+      for(QAppChildMetaData qAppChildMetaData : CollectionUtils.nonNullList(app.getChildren()))
+      {
+         childrenByType.add(qAppChildMetaData.getClass(), qAppChildMetaData.getName());
+      }
+
+      for(QAppSection section : CollectionUtils.nonNullList(app.getSections()))
+      {
+         for(String tableName : CollectionUtils.nonNullList(section.getTables()))
+         {
+            if(!childrenByType.get(QTableMetaData.class).contains(tableName))
+            {
+               QTableMetaData table = qInstance.getTable(tableName);
+               if(table != null)
+               {
+                  app.withChild(table);
+               }
+            }
+         }
+
+         for(String processName : CollectionUtils.nonNullList(section.getProcesses()))
+         {
+            if(!childrenByType.get(QProcessMetaData.class).contains(processName))
+            {
+               QProcessMetaData process = qInstance.getProcess(processName);
+               if(process != null)
+               {
+                  app.withChild(process);
+               }
+            }
+         }
+
+         for(String reportName : CollectionUtils.nonNullList(section.getReports()))
+         {
+            if(!childrenByType.get(QReportMetaData.class).contains(reportName))
+            {
+               QReportMetaData report = qInstance.getReport(reportName);
+               if(report != null)
+               {
+                  app.withChild(report);
+               }
+            }
+         }
+      }
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceHelpContentManager.java

@@ -0,0 +1,269 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2023.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.instances;
+
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.ListIterator;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
+import com.kingsrook.qqq.backend.core.logging.QLogger;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.helpcontent.HelpContent;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.help.HelpFormat;
+import com.kingsrook.qqq.backend.core.model.metadata.help.HelpRole;
+import com.kingsrook.qqq.backend.core.model.metadata.help.QHelpContent;
+import com.kingsrook.qqq.backend.core.model.metadata.help.QHelpRole;
+import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QFieldSection;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
+import com.kingsrook.qqq.backend.core.utils.StringUtils;
+import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
+
+
+/*******************************************************************************
+ ** Utility methods for working with (dynamic, from a table) HelpContent - and
+ ** putting it into meta-data in a QInstance.
+ *******************************************************************************/
+public class QInstanceHelpContentManager
+{
+   private static final QLogger LOG = QLogger.getLogger(QInstanceHelpContentManager.class);
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public static void loadHelpContent(QInstance qInstance)
+   {
+      try
+      {
+         if(qInstance.getTable(HelpContent.TABLE_NAME) == null)
+         {
+            return;
+         }
+
+         QueryInput queryInput = new QueryInput();
+         queryInput.setTableName(HelpContent.TABLE_NAME);
+         QueryOutput queryOutput = new QueryAction().execute(queryInput);
+         for(QRecord record : queryOutput.getRecords())
+         {
+            processHelpContentRecord(qInstance, record);
+         }
+      }
+      catch(Exception e)
+      {
+         LOG.error("Error loading help content", e);
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public static void processHelpContentRecord(QInstance qInstance, QRecord record)
+   {
+      try
+      {
+         /////////////////////////////////////////////////
+         // parse the key into its parts that we expect //
+         /////////////////////////////////////////////////
+         String              key            = record.getValueString("key");
+         Map<String, String> nameValuePairs = new HashMap<>();
+         for(String part : key.split(";"))
+         {
+            String[] parts = part.split(":");
+            nameValuePairs.put(parts[0], parts[1]);
+         }
+
+         String tableName   = nameValuePairs.get("table");
+         String processName = nameValuePairs.get("process");
+         String fieldName   = nameValuePairs.get("field");
+         String sectionName = nameValuePairs.get("section");
+
+         ///////////////////////////////////////////////////////////
+         // build a help content meta-data object from the record //
+         ///////////////////////////////////////////////////////////
+         QHelpContent helpContent = new QHelpContent()
+            .withContent(record.getValueString("content"))
+            .withRole(QHelpRole.valueOf(record.getValueString("role"))); // mmm, we could fall down a bit here w/ other app-defined roles...
+
+         if(StringUtils.hasContent(record.getValueString("format")))
+         {
+            helpContent.setFormat(HelpFormat.valueOf(record.getValueString("format")));
+         }
+         Set<HelpRole> roles = helpContent.getRoles();
+
+         ///////////////////////////////////////////////////////////////////////////////////////////////////
+         // check - if there are no contents, then let's remove this help content from the container      //
+         // (note pre-delete customizer will take advantage of this, passing in empty content on purpose) //
+         ///////////////////////////////////////////////////////////////////////////////////////////////////
+         if(!StringUtils.hasContent(helpContent.getContent()))
+         {
+            helpContent = null;
+         }
+
+         ///////////////////////////////////////////////////////////////////////////////////
+         // look at what parts of the key we got, and find the meta-data object to update //
+         ///////////////////////////////////////////////////////////////////////////////////
+         if(StringUtils.hasContent(tableName))
+         {
+            QTableMetaData table = qInstance.getTable(tableName);
+            if(table == null)
+            {
+               LOG.info("Unrecognized table in help content", logPair("key", key));
+               return;
+            }
+
+            if(StringUtils.hasContent(fieldName))
+            {
+               //////////////////////////
+               // handle a table field //
+               //////////////////////////
+               QFieldMetaData field = table.getFields().get(fieldName);
+               if(field == null)
+               {
+                  LOG.info("Unrecognized table field in help content", logPair("key", key));
+                  return;
+               }
+
+               if(helpContent != null)
+               {
+                  field.withHelpContent(helpContent);
+               }
+               else
+               {
+                  field.removeHelpContent(roles);
+               }
+            }
+            else if(StringUtils.hasContent(sectionName))
+            {
+               ////////////////////////////
+               // handle a table section //
+               ////////////////////////////
+               Optional<QFieldSection> optionalSection = table.getSections().stream().filter(s -> sectionName.equals(s.getName())).findFirst();
+               if(optionalSection.isEmpty())
+               {
+                  LOG.info("Unrecognized table section in help content", logPair("key", key));
+                  return;
+               }
+
+               if(helpContent != null)
+               {
+                  optionalSection.get().withHelpContent(helpContent);
+               }
+               else
+               {
+                  optionalSection.get().removeHelpContent(roles);
+               }
+            }
+         }
+         else if(StringUtils.hasContent(processName))
+         {
+            QProcessMetaData process = qInstance.getProcess(processName);
+            if(process == null)
+            {
+               LOG.info("Unrecognized process in help content", logPair("key", key));
+               return;
+            }
+
+            if(StringUtils.hasContent(fieldName))
+            {
+               ////////////////////////////
+               // handle a process field //
+               ////////////////////////////
+               Optional<QFieldMetaData> optionalField = CollectionUtils.mergeLists(process.getInputFields(), process.getOutputFields())
+                  .stream().filter(f -> fieldName.equals(f.getName()))
+                  .findFirst();
+
+               if(optionalField.isEmpty())
+               {
+                  LOG.info("Unrecognized process field in help content", logPair("key", key));
+                  return;
+               }
+
+               if(helpContent != null)
+               {
+                  optionalField.get().withHelpContent(helpContent);
+               }
+               else
+               {
+                  optionalField.get().removeHelpContent(roles);
+               }
+            }
+         }
+      }
+      catch(Exception e)
+      {
+         LOG.warn("Error processing a helpContent record", e, logPair("id", record.getValue("id")));
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** add a help content object to a list - replacing an entry in the list with the
+    ** same roles if one is found.
+    *******************************************************************************/
+   public static void putHelpContentInList(QHelpContent helpContent, List<QHelpContent> helpContents)
+   {
+      ListIterator<QHelpContent> iterator = helpContents.listIterator();
+      while(iterator.hasNext())
+      {
+         QHelpContent existingContent = iterator.next();
+         if(Objects.equals(existingContent.getRoles(), helpContent.getRoles()))
+         {
+            iterator.set(helpContent);
+            return;
+         }
+      }
+
+      helpContents.add(helpContent);
+   }
+
+
+
+   /*******************************************************************************
+    ** Remove any helpContent entries in a list if they have a set of roles that matches
+    ** the input set.
+    *******************************************************************************/
+   public static void removeHelpContentByRoleSetFromList(Set<HelpRole> roles, List<QHelpContent> helpContents)
+   {
+      if(helpContents == null)
+      {
+         return;
+      }
+
+      helpContents.removeIf(existingContent -> Objects.equals(existingContent.getRoles(), roles));
+   }
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QMetaDataVariableInterpreter.java

@@ -270,6 +270,32 @@ public class QMetaDataVariableInterpreter
 
 
 
+   /*******************************************************************************
+    ** First look for a string in the specified system property -
+    ** Next look for a string in the specified env var name -
+    ** Finally return the default.
+    *******************************************************************************/
+   public String getStringFromPropertyOrEnvironment(String systemPropertyName, String environmentVariableName, String defaultIfNotSet)
+   {
+      String propertyValue = System.getProperty(systemPropertyName);
+      if(StringUtils.hasContent(propertyValue))
+      {
+         LOG.info("Read system property [" + systemPropertyName + "] as [" + propertyValue + "].");
+         return (propertyValue);
+      }
+
+      String envValue = interpret("${env." + environmentVariableName + "}");
+      if(StringUtils.hasContent(envValue))
+      {
+         LOG.info("Read env var [" + environmentVariableName + "] as [" + envValue + "].");
+         return (envValue);
+      }
+
+      return defaultIfNotSet;
+   }
+
+
+
    /*******************************************************************************
     ** First look for a boolean ("true" or "false") in the specified system property -
     ** Next look for a boolean in the specified env var name -
@@ -347,7 +373,7 @@ public class QMetaDataVariableInterpreter
          if(canParseAsInteger(envValue))
          {
             LOG.info("Read env var [" + environmentVariableName + "] as integer " + environmentVariableName);
-            return (Integer.parseInt(propertyValue));
+            return (Integer.parseInt(envValue));
          }
          else
          {

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/logging/LogUtils.java

@@ -26,6 +26,7 @@ import java.util.Arrays;
 import java.util.List;
 import java.util.Objects;
 import java.util.stream.Collectors;
+import com.kingsrook.qqq.backend.core.instances.QMetaDataVariableInterpreter;
 import com.kingsrook.qqq.backend.core.utils.lambdas.UnsafeSupplier;
 
 
@@ -34,6 +35,17 @@ import com.kingsrook.qqq.backend.core.utils.lambdas.UnsafeSupplier;
  *******************************************************************************/
 public class LogUtils
 {
+   ///////////////////////////////////////////////////////////////////////////////////////////////
+   // This string will be used in regex, inside ()'s, so you can supply pipe-delimited packages //
+   // as in, com.kingsrook|com.yourdomain|org.some.other.package                                //
+   ///////////////////////////////////////////////////////////////////////////////////////////////
+   private static String packagesToKeep = ".";
+
+   static
+   {
+      packagesToKeep = new QMetaDataVariableInterpreter().getStringFromPropertyOrEnvironment("qqq.logger.packagesToKeep", "QQQ_LOGGER_PACKAGES_TO_KEEP", ".");
+   }
+
 
    /*******************************************************************************
     **
@@ -118,9 +130,8 @@ public class LogUtils
    {
       try
       {
-         String        packagesToKeep = "com.kingsrook|com.coldtrack"; // todo - parameterize!!
-         StringBuilder rs             = new StringBuilder();
-         String[]      lines          = stackTrace.split("\n");
+         StringBuilder rs    = new StringBuilder();
+         String[]      lines = stackTrace.split("\n");
 
          int    indexWithinSubStack = 0;
          int    skipsInThisPackage  = 0;
@@ -134,7 +145,13 @@ public class LogUtils
             {
                keepLine = false;
                indexWithinSubStack++;
-               if(line.matches("^\\s+at (" + packagesToKeep + ").*"))
+
+               /////////////////////////////////////////////////////////////////////////////
+               // avoid NPE on packages to keep (and keep all packages)                   //
+               // also, avoid the regex call if it's the default of "." (e.g., match all) //
+               // otherwise, check if the line matches "at (packagesToKeep).*"            //
+               /////////////////////////////////////////////////////////////////////////////
+               if(packagesToKeep == null || ".".equals(packagesToKeep) || line.matches("^\\s+at (" + packagesToKeep + ").*"))
                {
                   keepLine = true;
                }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/logging/QLogger.java

@@ -124,7 +124,7 @@ public class QLogger
     *******************************************************************************/
    public void log(Level level, String message)
    {
-      logger.log(level, makeJsonString(message));
+      logger.log(level, () -> makeJsonString(message));
    }
 
 
@@ -134,7 +134,7 @@ public class QLogger
     *******************************************************************************/
    public void log(Level level, String message, Throwable t)
    {
-      logger.log(level, makeJsonString(message, t));
+      logger.log(level, () -> makeJsonString(message, t));
    }
 
 
@@ -144,7 +144,7 @@ public class QLogger
     *******************************************************************************/
    public void log(Level level, String message, Throwable t, LogPair... logPairs)
    {
-      logger.log(level, makeJsonString(message, t, logPairs));
+      logger.log(level, () -> makeJsonString(message, t, logPairs));
    }
 
 
@@ -154,7 +154,7 @@ public class QLogger
     *******************************************************************************/
    public void log(Level level, Throwable t)
    {
-      logger.log(level, makeJsonString(null, t));
+      logger.log(level, () -> makeJsonString(null, t));
    }
 
 
@@ -164,7 +164,7 @@ public class QLogger
     *******************************************************************************/
    public void trace(String message)
    {
-      logger.trace(makeJsonString(message));
+      logger.trace(() -> makeJsonString(message));
    }
 
 
@@ -174,7 +174,7 @@ public class QLogger
     *******************************************************************************/
    public void trace(String message, LogPair... logPairs)
    {
-      logger.trace(makeJsonString(message, null, logPairs));
+      logger.trace(() -> makeJsonString(message, null, logPairs));
    }
 
 
@@ -194,7 +194,7 @@ public class QLogger
     *******************************************************************************/
    public void trace(String message, Throwable t)
    {
-      logger.trace(makeJsonString(message, t));
+      logger.trace(() -> makeJsonString(message, t));
    }
 
 
@@ -204,7 +204,7 @@ public class QLogger
     *******************************************************************************/
    public void trace(String message, Throwable t, LogPair... logPairs)
    {
-      logger.trace(makeJsonString(message, t, logPairs));
+      logger.trace(() -> makeJsonString(message, t, logPairs));
    }
 
 
@@ -214,7 +214,7 @@ public class QLogger
     *******************************************************************************/
    public void trace(Throwable t)
    {
-      logger.trace(makeJsonString(null, t));
+      logger.trace(() -> makeJsonString(null, t));
    }
 
 
@@ -224,7 +224,7 @@ public class QLogger
     *******************************************************************************/
    public void debug(String message)
    {
-      logger.debug(makeJsonString(message));
+      logger.debug(() -> makeJsonString(message));
    }
 
 
@@ -234,7 +234,7 @@ public class QLogger
     *******************************************************************************/
    public void debug(String message, LogPair... logPairs)
    {
-      logger.debug(makeJsonString(message, null, logPairs));
+      logger.debug(() -> makeJsonString(message, null, logPairs));
    }
 
 
@@ -254,7 +254,7 @@ public class QLogger
     *******************************************************************************/
    public void debug(String message, Throwable t)
    {
-      logger.debug(makeJsonString(message, t));
+      logger.debug(() -> makeJsonString(message, t));
    }
 
 
@@ -264,7 +264,7 @@ public class QLogger
     *******************************************************************************/
    public void debug(String message, Throwable t, LogPair... logPairs)
    {
-      logger.debug(makeJsonString(message, t, logPairs));
+      logger.debug(() -> makeJsonString(message, t, logPairs));
    }
 
 
@@ -274,7 +274,7 @@ public class QLogger
     *******************************************************************************/
    public void debug(Throwable t)
    {
-      logger.debug(makeJsonString(null, t));
+      logger.debug(() -> makeJsonString(null, t));
    }
 
 
@@ -284,7 +284,7 @@ public class QLogger
     *******************************************************************************/
    public void info(String message)
    {
-      logger.info(makeJsonString(message));
+      logger.info(() -> makeJsonString(message));
    }
 
 
@@ -294,7 +294,7 @@ public class QLogger
     *******************************************************************************/
    public void info(LogPair... logPairs)
    {
-      logger.info(makeJsonString(null, null, logPairs));
+      logger.info(() -> makeJsonString(null, null, logPairs));
    }
 
 
@@ -304,7 +304,7 @@ public class QLogger
     *******************************************************************************/
    public void info(List<LogPair> logPairList)
    {
-      logger.info(makeJsonString(null, null, logPairList));
+      logger.info(() -> makeJsonString(null, null, logPairList));
    }
 
 
@@ -314,7 +314,7 @@ public class QLogger
     *******************************************************************************/
    public void info(String message, LogPair... logPairs)
    {
-      logger.info(makeJsonString(message, null, logPairs));
+      logger.info(() -> makeJsonString(message, null, logPairs));
    }
 
 
@@ -334,7 +334,7 @@ public class QLogger
     *******************************************************************************/
    public void info(String message, Throwable t)
    {
-      logger.info(makeJsonString(message, t));
+      logger.info(() -> makeJsonString(message, t));
    }
 
 
@@ -344,7 +344,7 @@ public class QLogger
     *******************************************************************************/
    public void info(String message, Throwable t, LogPair... logPairs)
    {
-      logger.info(makeJsonString(message, t, logPairs));
+      logger.info(() -> makeJsonString(message, t, logPairs));
    }
 
 
@@ -354,7 +354,7 @@ public class QLogger
     *******************************************************************************/
    public void info(Throwable t)
    {
-      logger.info(makeJsonString(null, t));
+      logger.info(() -> makeJsonString(null, t));
    }
 
 
@@ -364,7 +364,7 @@ public class QLogger
     *******************************************************************************/
    public void warn(String message)
    {
-      logger.warn(makeJsonString(message));
+      logger.warn(() -> makeJsonString(message));
    }
 
 
@@ -374,7 +374,7 @@ public class QLogger
     *******************************************************************************/
    public void warn(String message, LogPair... logPairs)
    {
-      logger.warn(makeJsonString(message, null, logPairs));
+      logger.warn(() -> makeJsonString(message, null, logPairs));
    }
 
 
@@ -394,7 +394,7 @@ public class QLogger
     *******************************************************************************/
    public void warn(String message, Throwable t)
    {
-      logger.log(determineIfShouldDowngrade(t, Level.WARN), makeJsonString(message, t));
+      logger.log(determineIfShouldDowngrade(t, Level.WARN), () -> makeJsonString(message, t));
    }
 
 
@@ -404,7 +404,7 @@ public class QLogger
     *******************************************************************************/
    public void warn(String message, Throwable t, LogPair... logPairs)
    {
-      logger.log(determineIfShouldDowngrade(t, Level.WARN), makeJsonString(message, t, logPairs));
+      logger.log(determineIfShouldDowngrade(t, Level.WARN), () -> makeJsonString(message, t, logPairs));
    }
 
 
@@ -414,7 +414,7 @@ public class QLogger
     *******************************************************************************/
    public void warn(Throwable t)
    {
-      logger.log(determineIfShouldDowngrade(t, Level.WARN), makeJsonString(null, t));
+      logger.log(determineIfShouldDowngrade(t, Level.WARN), () -> makeJsonString(null, t));
    }
 
 
@@ -424,7 +424,7 @@ public class QLogger
     *******************************************************************************/
    public void error(String message)
    {
-      logger.error(makeJsonString(message));
+      logger.error(() -> makeJsonString(message));
    }
 
 
@@ -434,7 +434,7 @@ public class QLogger
     *******************************************************************************/
    public void error(String message, LogPair... logPairs)
    {
-      logger.error(makeJsonString(message, null, logPairs));
+      logger.error(() -> makeJsonString(message, null, logPairs));
    }
 
 
@@ -454,7 +454,7 @@ public class QLogger
     *******************************************************************************/
    public void error(String message, Throwable t)
    {
-      logger.log(determineIfShouldDowngrade(t, Level.ERROR), makeJsonString(message, t));
+      logger.log(determineIfShouldDowngrade(t, Level.ERROR), () -> makeJsonString(message, t));
    }
 
 
@@ -464,7 +464,7 @@ public class QLogger
     *******************************************************************************/
    public void error(String message, Throwable t, LogPair... logPairs)
    {
-      logger.log(determineIfShouldDowngrade(t, Level.ERROR), makeJsonString(message, t, logPairs));
+      logger.log(determineIfShouldDowngrade(t, Level.ERROR), () -> makeJsonString(message, t, logPairs));
    }
 
 
@@ -474,7 +474,7 @@ public class QLogger
     *******************************************************************************/
    public void error(Throwable t)
    {
-      logger.log(determineIfShouldDowngrade(t, Level.ERROR), makeJsonString(null, t));
+      logger.log(determineIfShouldDowngrade(t, Level.ERROR), () -> makeJsonString(null, t));
    }
 
 

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/MetaDataProducerInterface.java

@@ -0,0 +1,78 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2023.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.model;
+
+
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.TopLevelMetaDataInterface;
+
+
+/*******************************************************************************
+ ** Interface for classes that know how to produce meta data objects.  Useful with
+ ** MetaDataProducerHelper, to put point at a package full of these, and populate
+ ** your whole QInstance.
+ **
+ ** See also MetaDataProducer - an implementer of this interface, which actually
+ ** came first, and is fine to extend if producing a meta-data class is all your
+ ** clas means to do (nice and "Single-responsibility principle").
+ **
+ ** But, in some applications you may want to, for example, have one class that
+ ** defines a process step, and also produces the meta-data for that process, so
+ ** your whole process can just be one class - so then just have your step class
+ ** implement this interface.  or, same idea for a QRecordEntity that provides
+ ** its own TableMetaData.
+ *******************************************************************************/
+public interface MetaDataProducerInterface<T extends TopLevelMetaDataInterface>
+{
+   int DEFAULT_SORT_ORDER = 500;
+
+
+   /*******************************************************************************
+    ** Produce the metaData object.  Generally, you don't want to add it to the instance
+    ** yourself - but the instance is there in case you need it to get other metaData.
+    *******************************************************************************/
+   T produce(QInstance qInstance) throws QException;
+
+
+   /*******************************************************************************
+    ** In case this producer needs to run before (or after) others, this method
+    ** can control influence that (e.g., if used by MetaDataProducerHelper).
+    **
+    ** Smaller values run first.
+    *******************************************************************************/
+   default int getSortOrder()
+   {
+      return (DEFAULT_SORT_ORDER);
+   }
+
+
+   /*******************************************************************************
+    ** turn this producer on or off - e.g., maybe based on an env value.
+    **
+    *******************************************************************************/
+   default boolean isEnabled()
+   {
+      return (true);
+   }
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/audits/AuditSingleInput.java

@@ -28,6 +28,9 @@ import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Optional;
+import com.kingsrook.qqq.backend.core.actions.audits.AuditAction;
+import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
 import com.kingsrook.qqq.backend.core.model.metadata.security.RecordSecurityLock;
 import com.kingsrook.qqq.backend.core.model.metadata.security.RecordSecurityLockFilters;
@@ -52,6 +55,41 @@ public class AuditSingleInput
 
 
 
+   /*******************************************************************************
+    ** Constructor
+    **
+    *******************************************************************************/
+   public AuditSingleInput()
+   {
+   }
+
+
+
+   /*******************************************************************************
+    ** Constructor
+    **
+    *******************************************************************************/
+   public AuditSingleInput(QTableMetaData table, QRecord record, String auditMessage)
+   {
+      setAuditTableName(table.getName());
+      setRecordId(record.getValueInteger(table.getPrimaryKeyField()));
+      setSecurityKeyValues(AuditAction.getRecordSecurityKeyValues(table, record, Optional.empty()));
+      setMessage(auditMessage);
+   }
+
+
+
+   /*******************************************************************************
+    ** Constructor
+    **
+    *******************************************************************************/
+   public AuditSingleInput(String tableName, QRecord record, String auditMessage)
+   {
+      this(QContext.getQInstance().getTable(tableName), record, auditMessage);
+   }
+
+
+
    /*******************************************************************************
     ** Getter for auditTableName
     *******************************************************************************/

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/ProcessSummaryFilterLink.java

@@ -131,6 +131,17 @@ public class ProcessSummaryFilterLink implements ProcessSummaryLineInterface
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public String getMessage()
+   {
+      return getFullText();
+   }
+
+
+
    /*******************************************************************************
     ** Setter for status
     **

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/ProcessSummaryLine.java

@@ -182,6 +182,7 @@ public class ProcessSummaryLine implements ProcessSummaryLineInterface
     ** Getter for message
     **
     *******************************************************************************/
+   @Override
    public String getMessage()
    {
       return message;

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/ProcessSummaryLineInterface.java

@@ -38,6 +38,10 @@ public interface ProcessSummaryLineInterface extends Serializable
     *******************************************************************************/
    Status getStatus();
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   String getMessage();
 
    /*******************************************************************************
     ** meant to be called by framework, after process is complete, give the

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/ProcessSummaryRecordLink.java

@@ -95,6 +95,17 @@ public class ProcessSummaryRecordLink implements ProcessSummaryLineInterface
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public String getMessage()
+   {
+      return getFullText();
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/scripts/RunAssociatedScriptOutput.java

@@ -32,6 +32,7 @@ import com.kingsrook.qqq.backend.core.model.actions.AbstractActionOutput;
 public class RunAssociatedScriptOutput extends AbstractActionOutput
 {
    private Serializable output;
+   private Integer      scriptRevisionId;
 
 
 
@@ -67,4 +68,36 @@ public class RunAssociatedScriptOutput extends AbstractActionOutput
       return (this);
    }
 
+
+
+   /*******************************************************************************
+    ** Getter for scriptRevisionId
+    *******************************************************************************/
+   public Integer getScriptRevisionId()
+   {
+      return (this.scriptRevisionId);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for scriptRevisionId
+    *******************************************************************************/
+   public void setScriptRevisionId(Integer scriptRevisionId)
+   {
+      this.scriptRevisionId = scriptRevisionId;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for scriptRevisionId
+    *******************************************************************************/
+   public RunAssociatedScriptOutput withScriptRevisionId(Integer scriptRevisionId)
+   {
+      this.scriptRevisionId = scriptRevisionId;
+      return (this);
+   }
+
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QFilterCriteria.java

@@ -120,7 +120,7 @@ public class QFilterCriteria implements Serializable, Cloneable
       }
       else
       {
-         this.values = Arrays.stream(values).toList();
+         this.values = new ArrayList<>(Arrays.stream(values).toList());
       }
    }
 

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/audits/AuditsMetaDataProvider.java

@@ -116,16 +116,19 @@ public class AuditsMetaDataProvider
       instance.addPossibleValueSource(new QPossibleValueSource()
          .withName(TABLE_NAME_AUDIT_TABLE)
          .withTableName(TABLE_NAME_AUDIT_TABLE)
+         .withOrderByField("name")
       );
 
       instance.addPossibleValueSource(new QPossibleValueSource()
          .withName(TABLE_NAME_AUDIT_USER)
          .withTableName(TABLE_NAME_AUDIT_USER)
+         .withOrderByField("name")
       );
 
       instance.addPossibleValueSource(new QPossibleValueSource()
          .withName(TABLE_NAME_AUDIT)
          .withTableName(TABLE_NAME_AUDIT)
+         .withOrderByField("id", false)
       );
    }
 

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/dashboard/widgets/ChartData.java

@@ -48,6 +48,8 @@ public class ChartData extends QWidgetData
    private boolean isCurrency = false;
    private int     height;
 
+   private ChartSubheaderData chartSubheaderData;
+
 
 
    /*******************************************************************************
@@ -387,6 +389,7 @@ public class ChartData extends QWidgetData
          private String       color;
          private String       backgroundColor;
          private List<String> urls;
+         private List<String> backgroundColors;
 
 
 
@@ -423,6 +426,17 @@ public class ChartData extends QWidgetData
 
 
 
+         /*******************************************************************************
+          ** Getter for backgroundColors
+          **
+          *******************************************************************************/
+         public List<String> getBackgroundColors()
+         {
+            return backgroundColors;
+         }
+
+
+
          /*******************************************************************************
           ** Setter for backgroundColor
           **
@@ -434,6 +448,17 @@ public class ChartData extends QWidgetData
 
 
 
+         /*******************************************************************************
+          ** Setter for backgroundColor
+          **
+          *******************************************************************************/
+         public void setBackgroundColors(List<String> backgroundColors)
+         {
+            this.backgroundColors = backgroundColors;
+         }
+
+
+
          /*******************************************************************************
           ** Fluent setter for backgroundColor
           **
@@ -446,6 +471,18 @@ public class ChartData extends QWidgetData
 
 
 
+         /*******************************************************************************
+          ** Fluent setter for backgroundColor
+          **
+          *******************************************************************************/
+         public Dataset withBackgroundColors(List<String> backgroundColors)
+         {
+            this.backgroundColors = backgroundColors;
+            return (this);
+         }
+
+
+
          /*******************************************************************************
           ** Getter for color
           **
@@ -559,4 +596,36 @@ public class ChartData extends QWidgetData
          }
       }
    }
+
+
+
+   /*******************************************************************************
+    ** Getter for chartSubheaderData
+    *******************************************************************************/
+   public ChartSubheaderData getChartSubheaderData()
+   {
+      return (this.chartSubheaderData);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for chartSubheaderData
+    *******************************************************************************/
+   public void setChartSubheaderData(ChartSubheaderData chartSubheaderData)
+   {
+      this.chartSubheaderData = chartSubheaderData;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for chartSubheaderData
+    *******************************************************************************/
+   public ChartData withChartSubheaderData(ChartSubheaderData chartSubheaderData)
+   {
+      this.chartSubheaderData = chartSubheaderData;
+      return (this);
+   }
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/dashboard/widgets/ChartSubheaderData.java

@@ -0,0 +1,331 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2023.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.model.dashboard.widgets;
+
+
+import java.math.BigDecimal;
+import java.math.MathContext;
+import java.math.RoundingMode;
+
+
+/*******************************************************************************
+ **
+ *******************************************************************************/
+public class ChartSubheaderData
+{
+   private Number  mainNumber;
+   private Number  vsPreviousPercent;
+   private Number  vsPreviousNumber;
+   private Boolean isUpVsPrevious;
+   private Boolean isGoodVsPrevious;
+   private String  vsDescription = "vs prev period";
+
+   private String mainNumberUrl;
+   private String previousNumberUrl;
+
+
+
+   /*******************************************************************************
+    ** Getter for mainNumber
+    *******************************************************************************/
+   public Number getMainNumber()
+   {
+      return (this.mainNumber);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for mainNumber
+    *******************************************************************************/
+   public void setMainNumber(Number mainNumber)
+   {
+      this.mainNumber = mainNumber;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for mainNumber
+    *******************************************************************************/
+   public ChartSubheaderData withMainNumber(Number mainNumber)
+   {
+      this.mainNumber = mainNumber;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for vsPreviousNumber
+    *******************************************************************************/
+   public Number getVsPreviousNumber()
+   {
+      return (this.vsPreviousNumber);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for vsPreviousNumber
+    *******************************************************************************/
+   public void setVsPreviousNumber(Number vsPreviousNumber)
+   {
+      this.vsPreviousNumber = vsPreviousNumber;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for vsPreviousNumber
+    *******************************************************************************/
+   public ChartSubheaderData withVsPreviousNumber(Number vsPreviousNumber)
+   {
+      this.vsPreviousNumber = vsPreviousNumber;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for vsDescription
+    *******************************************************************************/
+   public String getVsDescription()
+   {
+      return (this.vsDescription);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for vsDescription
+    *******************************************************************************/
+   public void setVsDescription(String vsDescription)
+   {
+      this.vsDescription = vsDescription;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for vsDescription
+    *******************************************************************************/
+   public ChartSubheaderData withVsDescription(String vsDescription)
+   {
+      this.vsDescription = vsDescription;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for vsPreviousPercent
+    *******************************************************************************/
+   public Number getVsPreviousPercent()
+   {
+      return (this.vsPreviousPercent);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for vsPreviousPercent
+    *******************************************************************************/
+   public void setVsPreviousPercent(Number vsPreviousPercent)
+   {
+      this.vsPreviousPercent = vsPreviousPercent;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for vsPreviousPercent
+    *******************************************************************************/
+   public ChartSubheaderData withVsPreviousPercent(Number vsPreviousPercent)
+   {
+      this.vsPreviousPercent = vsPreviousPercent;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for isUpVsPrevious
+    *******************************************************************************/
+   public Boolean getIsUpVsPrevious()
+   {
+      return (this.isUpVsPrevious);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for isUpVsPrevious
+    *******************************************************************************/
+   public void setIsUpVsPrevious(Boolean isUpVsPrevious)
+   {
+      this.isUpVsPrevious = isUpVsPrevious;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for isUpVsPrevious
+    *******************************************************************************/
+   public ChartSubheaderData withIsUpVsPrevious(Boolean isUpVsPrevious)
+   {
+      this.isUpVsPrevious = isUpVsPrevious;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for isGoodVsPrevious
+    *******************************************************************************/
+   public Boolean getIsGoodVsPrevious()
+   {
+      return (this.isGoodVsPrevious);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for isGoodVsPrevious
+    *******************************************************************************/
+   public void setIsGoodVsPrevious(Boolean isGoodVsPrevious)
+   {
+      this.isGoodVsPrevious = isGoodVsPrevious;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for isGoodVsPrevious
+    *******************************************************************************/
+   public ChartSubheaderData withIsGoodVsPrevious(Boolean isGoodVsPrevious)
+   {
+      this.isGoodVsPrevious = isGoodVsPrevious;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for mainNumberUrl
+    *******************************************************************************/
+   public String getMainNumberUrl()
+   {
+      return (this.mainNumberUrl);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for mainNumberUrl
+    *******************************************************************************/
+   public void setMainNumberUrl(String mainNumberUrl)
+   {
+      this.mainNumberUrl = mainNumberUrl;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for mainNumberUrl
+    *******************************************************************************/
+   public ChartSubheaderData withMainNumberUrl(String mainNumberUrl)
+   {
+      this.mainNumberUrl = mainNumberUrl;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for previousNumberUrl
+    *******************************************************************************/
+   public String getPreviousNumberUrl()
+   {
+      return (this.previousNumberUrl);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for previousNumberUrl
+    *******************************************************************************/
+   public void setPreviousNumberUrl(String previousNumberUrl)
+   {
+      this.previousNumberUrl = previousNumberUrl;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for previousNumberUrl
+    *******************************************************************************/
+   public ChartSubheaderData withPreviousNumberUrl(String previousNumberUrl)
+   {
+      this.previousNumberUrl = previousNumberUrl;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public void calculatePercentsEtc(boolean isUpGood)
+   {
+      if(mainNumber != null && vsPreviousNumber != null && vsPreviousNumber.doubleValue() > 0)
+      {
+         /////////////////////////////////////////////////////////////////
+         // these are the results we're going for:                      //
+         // current: 10, previous: 20 = -50%                            //
+         // current: 15, previous: 20 = -25%                            //
+         // current: 20, previous: 10 = +100%                           //
+         // current: 15, previous: 10 = +50%                            //
+         // this formula gets us that:  (current - previous) / previous //
+         // (with a *100 in there to make it a percent-looking value)   //
+         /////////////////////////////////////////////////////////////////
+         BigDecimal current    = new BigDecimal(String.valueOf(mainNumber));
+         BigDecimal previous   = new BigDecimal(String.valueOf(vsPreviousNumber));
+         BigDecimal difference = current.subtract(previous);
+         BigDecimal ratio      = difference.divide(previous, new MathContext(3, RoundingMode.HALF_UP));
+         BigDecimal percentBD  = ratio.multiply(new BigDecimal(100));
+         BigDecimal percent    = percentBD.compareTo(BigDecimal.ZERO) < 0 ? BigDecimal.ZERO.subtract(percentBD) : percentBD;
+         if(mainNumber.doubleValue() < vsPreviousNumber.doubleValue())
+         {
+            setIsUpVsPrevious(false);
+            setIsGoodVsPrevious(isUpGood ? false : true);
+            setVsPreviousPercent(percent);
+         }
+         else // note - equal is being considered here in the good.
+         {
+            setIsUpVsPrevious(true);
+            setIsGoodVsPrevious(isUpGood ? true : false);
+            setVsPreviousPercent(percent);
+         }
+      }
+   }
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/dashboard/widgets/ParentWidgetData.java

@@ -23,6 +23,7 @@ package com.kingsrook.qqq.backend.core.model.dashboard.widgets;
 
 
 import java.util.List;
+import com.kingsrook.qqq.backend.core.model.metadata.dashboard.ParentWidgetMetaData;
 
 
 /*******************************************************************************
@@ -32,6 +33,7 @@ import java.util.List;
 public class ParentWidgetData extends QWidgetData
 {
    private List<String> childWidgetNameList;
+   private ParentWidgetMetaData.LayoutType layoutType = ParentWidgetMetaData.LayoutType.GRID;
 
 
 
@@ -87,4 +89,36 @@ public class ParentWidgetData extends QWidgetData
       return (this);
    }
 
+
+
+   /*******************************************************************************
+    ** Getter for layoutType
+    *******************************************************************************/
+   public ParentWidgetMetaData.LayoutType getLayoutType()
+   {
+      return (this.layoutType);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for layoutType
+    *******************************************************************************/
+   public void setLayoutType(ParentWidgetMetaData.LayoutType layoutType)
+   {
+      this.layoutType = layoutType;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for layoutType
+    *******************************************************************************/
+   public ParentWidgetData withLayoutType(ParentWidgetMetaData.LayoutType layoutType)
+   {
+      this.layoutType = layoutType;
+      return (this);
+   }
+
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/dashboard/widgets/QWidgetData.java

@@ -36,6 +36,7 @@ public abstract class QWidgetData
    private String       footerHTML;
    private List<String> dropdownNameList;
    private List<String> dropdownLabelList;
+   private List<String> dropdownDefaultValueList;
    private Boolean      hasPermission;
 
    /////////////////////////////////////////////////////////////////////////////////////////
@@ -291,4 +292,36 @@ public abstract class QWidgetData
       return (this);
    }
 
+
+
+   /*******************************************************************************
+    ** Getter for dropdownDefaultValueList
+    *******************************************************************************/
+   public List<String> getDropdownDefaultValueList()
+   {
+      return (this.dropdownDefaultValueList);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for dropdownDefaultValueList
+    *******************************************************************************/
+   public void setDropdownDefaultValueList(List<String> dropdownDefaultValueList)
+   {
+      this.dropdownDefaultValueList = dropdownDefaultValueList;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for dropdownDefaultValueList
+    *******************************************************************************/
+   public QWidgetData withDropdownDefaultValueList(List<String> dropdownDefaultValueList)
+   {
+      this.dropdownDefaultValueList = dropdownDefaultValueList;
+      return (this);
+   }
+
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/dashboard/widgets/TableData.java

@@ -39,6 +39,8 @@ public class TableData extends QWidgetData
    private List<Map<String, Object>> rows;
    private Integer                   rowsPerPage;
    private Boolean                   hidePaginationDropdown;
+   private Boolean                   fixedStickyLastRow = false;
+   private Integer                   fixedHeight;
 
 
 
@@ -543,4 +545,67 @@ public class TableData extends QWidgetData
       }
 
    }
+
+
+
+   /*******************************************************************************
+    ** Getter for fixedStickyLastRow
+    *******************************************************************************/
+   public Boolean getFixedStickyLastRow()
+   {
+      return (this.fixedStickyLastRow);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for fixedStickyLastRow
+    *******************************************************************************/
+   public void setFixedStickyLastRow(Boolean fixedStickyLastRow)
+   {
+      this.fixedStickyLastRow = fixedStickyLastRow;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for fixedStickyLastRow
+    *******************************************************************************/
+   public TableData withFixedStickyLastRow(Boolean fixedStickyLastRow)
+   {
+      this.fixedStickyLastRow = fixedStickyLastRow;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for fixedHeight
+    *******************************************************************************/
+   public Integer getFixedHeight()
+   {
+      return (this.fixedHeight);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for fixedHeight
+    *******************************************************************************/
+   public void setFixedHeight(Integer fixedHeight)
+   {
+      this.fixedHeight = fixedHeight;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for fixedHeight
+    *******************************************************************************/
+   public TableData withFixedHeight(Integer fixedHeight)
+   {
+      this.fixedHeight = fixedHeight;
+      return (this);
+   }
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/data/QRecord.java

@@ -27,18 +27,22 @@ import java.math.BigDecimal;
 import java.time.Instant;
 import java.time.LocalDate;
 import java.time.LocalTime;
+import java.time.temporal.Temporal;
 import java.util.ArrayList;
+import java.util.Date;
 import java.util.HashMap;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.logging.QLogger;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 import com.kingsrook.qqq.backend.core.model.statusmessages.QErrorMessage;
 import com.kingsrook.qqq.backend.core.model.statusmessages.QWarningMessage;
 import com.kingsrook.qqq.backend.core.utils.ValueUtils;
-import org.apache.commons.lang.SerializationUtils;
+import org.apache.commons.lang3.SerializationUtils;
+import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
 
 
 /*******************************************************************************
@@ -61,6 +65,8 @@ import org.apache.commons.lang.SerializationUtils;
  *******************************************************************************/
 public class QRecord implements Serializable
 {
+   private static final QLogger LOG = QLogger.getLogger(QRecord.class);
+
    private String tableName;
    private String recordLabel;
 
@@ -110,12 +116,14 @@ public class QRecord implements Serializable
       this.tableName = record.tableName;
       this.recordLabel = record.recordLabel;
 
-      this.values = doDeepCopy(record.values);
-      this.displayValues = doDeepCopy(record.displayValues);
-      this.backendDetails = doDeepCopy(record.backendDetails);
-      this.errors = doDeepCopy(record.errors);
-      this.warnings = doDeepCopy(record.warnings);
-      this.associatedRecords = doDeepCopy(record.associatedRecords);
+      this.values = deepCopySimpleMap(record.values);
+      this.displayValues = deepCopySimpleMap(record.displayValues);
+      this.backendDetails = deepCopySimpleMap(record.backendDetails);
+
+      this.associatedRecords = deepCopyAssociatedRecords(record.associatedRecords);
+
+      this.errors = record.errors == null ? null : new ArrayList<>(record.errors);
+      this.warnings = record.warnings == null ? null : new ArrayList<>(record.warnings);
    }
 
 
@@ -135,40 +143,62 @@ public class QRecord implements Serializable
     ** todo - move to a cloning utils maybe?
     *******************************************************************************/
    @SuppressWarnings({ "unchecked" })
-   private <K, V> Map<K, V> doDeepCopy(Map<K, V> map)
+   private <K, V> Map<K, V> deepCopySimpleMap(Map<K, V> map)
    {
       if(map == null)
       {
          return (null);
       }
 
-      if(map instanceof Serializable serializableMap)
+      Map<K, V> clone = new LinkedHashMap<>();
+      for(Map.Entry<K, V> entry : map.entrySet())
       {
-         return (Map<K, V>) SerializationUtils.clone(serializableMap);
-      }
+         V value = entry.getValue();
 
-      return (new LinkedHashMap<>(map));
+         //////////////////////////////////////////////////////////////////////////
+         // not sure from where/how java.sql.Date objects are getting in here... //
+         //////////////////////////////////////////////////////////////////////////
+         if(value == null || value instanceof String || value instanceof Number || value instanceof Boolean || value instanceof Temporal || value instanceof Date || value instanceof byte[])
+         {
+            clone.put(entry.getKey(), entry.getValue());
+         }
+         else if(entry.getValue() instanceof ArrayList<?> arrayList)
+         {
+            ArrayList<?> cloneList = new ArrayList<>(arrayList);
+            clone.put(entry.getKey(), (V) cloneList);
+         }
+         else if(entry.getValue() instanceof Serializable serializableValue)
+         {
+            LOG.info("Non-primitive serializable value in QRecord - calling SerializationUtils.clone...", logPair("key", entry.getKey()), logPair("type", value.getClass()));
+            clone.put(entry.getKey(), (V) SerializationUtils.clone(serializableValue));
+         }
+         else
+         {
+            LOG.warn("Non-serializable value in QRecord...", logPair("key", entry.getKey()), logPair("type", value.getClass()));
+            clone.put(entry.getKey(), entry.getValue());
+         }
+      }
+      return (clone);
    }
 
 
 
    /*******************************************************************************
-    ** todo - move to a cloning utils maybe?
+    **
     *******************************************************************************/
-   @SuppressWarnings({ "unchecked" })
-   private <T> List<T> doDeepCopy(List<T> list)
+   private Map<String, List<QRecord>> deepCopyAssociatedRecords(Map<String, List<QRecord>> input)
    {
-      if(list == null)
+      if(input == null)
       {
          return (null);
       }
 
-      if(list instanceof Serializable serializableList)
+      Map<String, List<QRecord>> clone = new HashMap<>();
+      for(Map.Entry<String, List<QRecord>> entry : input.entrySet())
       {
-         return (List<T>) SerializationUtils.clone(serializableList);
+         clone.put(entry.getKey(), new ArrayList<>(entry.getValue()));
       }
-
-      return (new ArrayList<>(list));
+      return (clone);
    }
 
 

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/helpcontent/HelpContent.java

@@ -0,0 +1,296 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2023.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.model.helpcontent;
+
+
+import java.time.Instant;
+import com.kingsrook.qqq.backend.core.model.data.QField;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.data.QRecordEntity;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.ValueTooLongBehavior;
+
+
+/*******************************************************************************
+ ** QRecord Entity for HelpContent table
+ *******************************************************************************/
+public class HelpContent extends QRecordEntity
+{
+   public static final String TABLE_NAME = "helpContent";
+
+   @QField(isEditable = false)
+   private Integer id;
+
+   @QField(isEditable = false)
+   private Instant createDate;
+
+   @QField(isEditable = false)
+   private Instant modifyDate;
+
+   @QField(isRequired = true, maxLength = 250, valueTooLongBehavior = ValueTooLongBehavior.ERROR)
+   private String key;
+
+   @QField()
+   private String content;
+
+   @QField(possibleValueSourceName = HelpContentFormat.NAME)
+   private String format;
+
+   @QField(possibleValueSourceName = HelpContentRole.NAME, isRequired = true)
+   private String role;
+
+
+
+   /*******************************************************************************
+    ** Default constructor
+    *******************************************************************************/
+   public HelpContent()
+   {
+   }
+
+
+
+   /*******************************************************************************
+    ** Constructor that takes a QRecord
+    *******************************************************************************/
+   public HelpContent(QRecord record)
+   {
+      populateFromQRecord(record);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for id
+    *******************************************************************************/
+   public Integer getId()
+   {
+      return (this.id);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for id
+    *******************************************************************************/
+   public void setId(Integer id)
+   {
+      this.id = id;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for id
+    *******************************************************************************/
+   public HelpContent withId(Integer id)
+   {
+      this.id = id;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for createDate
+    *******************************************************************************/
+   public Instant getCreateDate()
+   {
+      return (this.createDate);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for createDate
+    *******************************************************************************/
+   public void setCreateDate(Instant createDate)
+   {
+      this.createDate = createDate;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for createDate
+    *******************************************************************************/
+   public HelpContent withCreateDate(Instant createDate)
+   {
+      this.createDate = createDate;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for modifyDate
+    *******************************************************************************/
+   public Instant getModifyDate()
+   {
+      return (this.modifyDate);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for modifyDate
+    *******************************************************************************/
+   public void setModifyDate(Instant modifyDate)
+   {
+      this.modifyDate = modifyDate;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for modifyDate
+    *******************************************************************************/
+   public HelpContent withModifyDate(Instant modifyDate)
+   {
+      this.modifyDate = modifyDate;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for key
+    *******************************************************************************/
+   public String getKey()
+   {
+      return (this.key);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for key
+    *******************************************************************************/
+   public void setKey(String key)
+   {
+      this.key = key;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for key
+    *******************************************************************************/
+   public HelpContent withKey(String key)
+   {
+      this.key = key;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for content
+    *******************************************************************************/
+   public String getContent()
+   {
+      return (this.content);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for content
+    *******************************************************************************/
+   public void setContent(String content)
+   {
+      this.content = content;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for content
+    *******************************************************************************/
+   public HelpContent withContent(String content)
+   {
+      this.content = content;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for format
+    *******************************************************************************/
+   public String getFormat()
+   {
+      return (this.format);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for format
+    *******************************************************************************/
+   public void setFormat(String format)
+   {
+      this.format = format;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for format
+    *******************************************************************************/
+   public HelpContent withFormat(String format)
+   {
+      this.format = format;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for role
+    *******************************************************************************/
+   public String getRole()
+   {
+      return (this.role);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for role
+    *******************************************************************************/
+   public void setRole(String role)
+   {
+      this.role = role;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for role
+    *******************************************************************************/
+   public HelpContent withRole(String role)
+   {
+      this.role = role;
+      return (this);
+   }
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/helpcontent/HelpContentFormat.java

@@ -0,0 +1,121 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2023.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.model.helpcontent;
+
+
+import java.util.Objects;
+import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.PossibleValueEnum;
+
+
+/*******************************************************************************
+ ** HelpContentFormat - possible value enum
+ *******************************************************************************/
+public enum HelpContentFormat implements PossibleValueEnum<String>
+{
+   TEXT("TEXT", "Plain Text"),
+   HTML("HTML", "HTML"),
+   MARKDOWN("MARKDOWN", "Markdown");
+
+   private final String id;
+   private final String label;
+
+   public static final String NAME = "helpContentFormat";
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   HelpContentFormat(String id, String label)
+   {
+      this.id = id;
+      this.label = label;
+   }
+
+
+
+   /*******************************************************************************
+    ** Get instance by id
+    **
+    *******************************************************************************/
+   public static HelpContentFormat getById(String id)
+   {
+      if(id == null)
+      {
+         return (null);
+      }
+
+      for(HelpContentFormat value : HelpContentFormat.values())
+      {
+         if(Objects.equals(value.id, id))
+         {
+            return (value);
+         }
+      }
+
+      return (null);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for id
+    **
+    *******************************************************************************/
+   public String getId()
+   {
+      return id;
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for label
+    **
+    *******************************************************************************/
+   public String getLabel()
+   {
+      return label;
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public String getPossibleValueId()
+   {
+      return (getId());
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public String getPossibleValueLabel()
+   {
+      return (getLabel());
+   }
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/helpcontent/HelpContentMetaDataProvider.java

@@ -0,0 +1,94 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2023.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.model.helpcontent;
+
+
+import java.util.List;
+import java.util.function.Consumer;
+import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizers;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.AdornmentType;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.FieldAdornment;
+import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon;
+import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSource;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QFieldSection;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.Tier;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.UniqueKey;
+
+
+/*******************************************************************************
+ ** Meta-data provider for table & PVS's for defining help-content for other
+ ** meta-data objects within a QQQ app
+ *******************************************************************************/
+public class HelpContentMetaDataProvider
+{
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public void defineAll(QInstance instance, String backendName, Consumer<QTableMetaData> backendDetailEnricher) throws QException
+   {
+      defineHelpContentTable(instance, backendName, backendDetailEnricher);
+      instance.addPossibleValueSource(QPossibleValueSource.newForEnum(HelpContentFormat.NAME, HelpContentFormat.values()));
+      instance.addPossibleValueSource(QPossibleValueSource.newForEnum(HelpContentRole.NAME, HelpContentRole.values()));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private void defineHelpContentTable(QInstance instance, String backendName, Consumer<QTableMetaData> backendDetailEnricher) throws QException
+   {
+      QTableMetaData table = new QTableMetaData()
+         .withName(HelpContent.TABLE_NAME)
+         .withBackendName(backendName)
+         .withRecordLabelFormat("%s %s")
+         .withRecordLabelFields("key", "role")
+         .withPrimaryKeyField("id")
+         .withUniqueKey(new UniqueKey("key", "role"))
+         .withFieldsFromEntity(HelpContent.class)
+         .withSection(new QFieldSection("identity", new QIcon("badge"), Tier.T1, List.of("id", "key", "role")))
+         .withSection(new QFieldSection("content", new QIcon("dataset"), Tier.T2, List.of("format", "content")))
+         .withSection(new QFieldSection("dates", new QIcon("calendar_month"), Tier.T3, List.of("createDate", "modifyDate")))
+         .withCustomizer(TableCustomizers.POST_INSERT_RECORD, new QCodeReference(HelpContentPostInsertCustomizer.class))
+         .withCustomizer(TableCustomizers.POST_UPDATE_RECORD, new QCodeReference(HelpContentPostUpdateCustomizer.class))
+         .withCustomizer(TableCustomizers.PRE_UPDATE_RECORD, new QCodeReference(HelpContentPreUpdateCustomizer.class))
+         .withCustomizer(TableCustomizers.PRE_DELETE_RECORD, new QCodeReference(HelpContentPreDeleteCustomizer.class));
+
+      table.getField("format").withFieldAdornment(AdornmentType.Size.SMALL.toAdornment());
+      table.getField("key").withFieldAdornment(AdornmentType.Size.LARGE.toAdornment());
+      table.getField("content").withFieldAdornment(AdornmentType.Size.LARGE.toAdornment());
+      table.getField("content").withFieldAdornment(new FieldAdornment(AdornmentType.CODE_EDITOR).withValue(AdornmentType.CodeEditorValues.languageMode("html")));
+
+      if(backendDetailEnricher != null)
+      {
+         backendDetailEnricher.accept(table);
+      }
+
+      instance.addTable(table);
+   }
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/helpcontent/HelpContentPostInsertCustomizer.java

@@ -0,0 +1,66 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2023.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.model.helpcontent;
+
+
+import java.util.List;
+import com.kingsrook.qqq.backend.core.actions.customizers.AbstractPostInsertCustomizer;
+import com.kingsrook.qqq.backend.core.context.QContext;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.instances.QInstanceHelpContentManager;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+
+
+/*******************************************************************************
+ ** after records are inserted, put their help content in meta-data
+ *******************************************************************************/
+public class HelpContentPostInsertCustomizer extends AbstractPostInsertCustomizer
+{
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public List<QRecord> apply(List<QRecord> records) throws QException
+   {
+      return insertRecordsIntoMetaData(records);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   static List<QRecord> insertRecordsIntoMetaData(List<QRecord> records)
+   {
+      if(records != null)
+      {
+         for(QRecord record : records)
+         {
+            QInstanceHelpContentManager.processHelpContentRecord(QContext.getQInstance(), record);
+         }
+      }
+
+      return records;
+   }
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/helpcontent/HelpContentPostUpdateCustomizer.java

@@ -0,0 +1,46 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2023.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.model.helpcontent;
+
+
+import java.util.List;
+import com.kingsrook.qqq.backend.core.actions.customizers.AbstractPostUpdateCustomizer;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+
+
+/*******************************************************************************
+ ** after records are updated, put their help content in meta-data
+ *******************************************************************************/
+public class HelpContentPostUpdateCustomizer extends AbstractPostUpdateCustomizer
+{
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public List<QRecord> apply(List<QRecord> records) throws QException
+   {
+      return HelpContentPostInsertCustomizer.insertRecordsIntoMetaData(records);
+   }
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/helpcontent/HelpContentPreDeleteCustomizer.java

@@ -0,0 +1,73 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2023.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.model.helpcontent;
+
+
+import java.util.List;
+import com.kingsrook.qqq.backend.core.actions.customizers.AbstractPreDeleteCustomizer;
+import com.kingsrook.qqq.backend.core.context.QContext;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.instances.QInstanceHelpContentManager;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+
+
+/*******************************************************************************
+ ** remove existing helpContent from meta-data when a record is deleted
+ *******************************************************************************/
+public class HelpContentPreDeleteCustomizer extends AbstractPreDeleteCustomizer
+{
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public List<QRecord> apply(List<QRecord> records) throws QException
+   {
+      if(records != null)
+      {
+         for(QRecord record : records)
+         {
+            removeOldRecordFromMetaData(record);
+         }
+      }
+
+      return (records);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   static void removeOldRecordFromMetaData(QRecord oldRecord)
+   {
+      ////////////////////////////////////////////////////////////////////////////
+      // this (clearing the content) will remove the helpContent under this key //
+      ////////////////////////////////////////////////////////////////////////////
+      if(oldRecord != null)
+      {
+         QRecord recordWithoutContent = new QRecord(oldRecord);
+         recordWithoutContent.setValue("content", null);
+         QInstanceHelpContentManager.processHelpContentRecord(QContext.getQInstance(), recordWithoutContent);
+      }
+   }
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/helpcontent/HelpContentPreUpdateCustomizer.java

@@ -0,0 +1,55 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2023.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.model.helpcontent;
+
+
+import java.util.List;
+import com.kingsrook.qqq.backend.core.actions.customizers.AbstractPreUpdateCustomizer;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+
+
+/*******************************************************************************
+ ** in case a row's Key or Role was changed, remove existing helpContent from that key.
+ *******************************************************************************/
+public class HelpContentPreUpdateCustomizer extends AbstractPreUpdateCustomizer
+{
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public List<QRecord> apply(List<QRecord> records) throws QException
+   {
+      if(records != null)
+      {
+         for(QRecord record : records)
+         {
+            QRecord oldRecord = getOldRecordMap().get(record.getValueInteger("id"));
+            HelpContentPreDeleteCustomizer.removeOldRecordFromMetaData(oldRecord);
+         }
+      }
+
+      return (records);
+   }
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/helpcontent/HelpContentRole.java

@@ -0,0 +1,128 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2023.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.model.helpcontent;
+
+
+import java.util.Objects;
+import com.kingsrook.qqq.backend.core.model.metadata.help.QHelpRole;
+import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.PossibleValueEnum;
+
+
+/*******************************************************************************
+ ** HelpContentRole - possible value enum
+ *******************************************************************************/
+public enum HelpContentRole implements PossibleValueEnum<String>
+{
+   ALL_SCREENS(QHelpRole.ALL_SCREENS.name(), "All Screens"),
+   READ_SCREENS(QHelpRole.READ_SCREENS.name(), "Query & View Screens"),
+   WRITE_SCREENS(QHelpRole.WRITE_SCREENS.name(), "Insert & Edit Screens"),
+   QUERY_SCREEN(QHelpRole.QUERY_SCREEN.name(), "Query Screen Only"),
+   VIEW_SCREEN(QHelpRole.VIEW_SCREEN.name(), "View Screen Only"),
+   EDIT_SCREEN(QHelpRole.EDIT_SCREEN.name(), "Edit Screen Only"),
+   INSERT_SCREEN(QHelpRole.INSERT_SCREEN.name(), "Insert Screen Only"),
+   PROCESS_SCREEN(QHelpRole.PROCESS_SCREEN.name(), "Process Screens");
+
+
+   private final String id;
+   private final String label;
+
+   public static final String NAME = "HelpContentRole";
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   HelpContentRole(String id, String label)
+   {
+      this.id = id;
+      this.label = label;
+   }
+
+
+
+   /*******************************************************************************
+    ** Get instance by id
+    **
+    *******************************************************************************/
+   public static HelpContentRole getById(String id)
+   {
+      if(id == null)
+      {
+         return (null);
+      }
+
+      for(HelpContentRole value : HelpContentRole.values())
+      {
+         if(Objects.equals(value.id, id))
+         {
+            return (value);
+         }
+      }
+
+      return (null);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for id
+    **
+    *******************************************************************************/
+   public String getId()
+   {
+      return id;
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for label
+    **
+    *******************************************************************************/
+   public String getLabel()
+   {
+      return label;
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public String getPossibleValueId()
+   {
+      return (getId());
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public String getPossibleValueLabel()
+   {
+      return (getLabel());
+   }
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/MetaDataProducer.java

@@ -22,7 +22,7 @@
 package com.kingsrook.qqq.backend.core.model.metadata;
 
 
-import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.MetaDataProducerInterface;
 
 
 /*******************************************************************************
@@ -30,29 +30,7 @@ import com.kingsrook.qqq.backend.core.exceptions.QException;
  ** MetaDataProducerHelper, to put point at a package full of these, and populate
  ** your whole QInstance.
  *******************************************************************************/
-public abstract class MetaDataProducer<T extends TopLevelMetaDataInterface>
+public abstract class MetaDataProducer<T extends TopLevelMetaDataInterface> implements MetaDataProducerInterface<T>
 {
-   public static final int DEFAULT_SORT_ORDER = 500;
-
-
-
-   /*******************************************************************************
-    ** Produce the metaData object.  Generally, you don't want to add it to the instance
-    ** yourself - but the instance is there in case you need it to get other metaData.
-    *******************************************************************************/
-   public abstract T produce(QInstance qInstance) throws QException;
-
-
-
-   /*******************************************************************************
-    ** In case this producer needs to run before (or after) others, this method
-    ** can control influence that (e.g., if used by MetaDataProducerHelper).
-    **
-    ** Smaller values run first.
-    *******************************************************************************/
-   public int getSortOrder()
-   {
-      return (DEFAULT_SORT_ORDER);
-   }
 
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/MetaDataProducerHelper.java

@@ -30,6 +30,8 @@ import java.util.Comparator;
 import java.util.List;
 import com.google.common.reflect.ClassPath;
 import com.kingsrook.qqq.backend.core.logging.QLogger;
+import com.kingsrook.qqq.backend.core.model.MetaDataProducerInterface;
+import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppMetaData;
 import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
 
 
@@ -43,18 +45,18 @@ public class MetaDataProducerHelper
 
 
    /*******************************************************************************
-    ** Recursively find all classes in the given package, that extend MetaDataProducer,
+    ** Recursively find all classes in the given package, that implement MetaDataProducerInterface
     ** run them, and add their output to the given qInstance.
     **
     ** Note - they'll be sorted by the sortOrder they provide.
     *******************************************************************************/
    public static void processAllMetaDataProducersInPackage(QInstance instance, String packageName) throws IOException
    {
-      ////////////////////////////////////////////////////////////
-      // find all the meta data producer classes in the package //
-      ////////////////////////////////////////////////////////////
-      List<Class<?>>            classesInPackage = getClassesInPackage(packageName);
-      List<MetaDataProducer<?>> producers        = new ArrayList<>();
+      ////////////////////////////////////////////////////////////////////////
+      // find all the meta data producer classes in (and under) the package //
+      ////////////////////////////////////////////////////////////////////////
+      List<Class<?>>                     classesInPackage = getClassesInPackage(packageName);
+      List<MetaDataProducerInterface<?>> producers        = new ArrayList<>();
       for(Class<?> aClass : classesInPackage)
       {
          try
@@ -64,46 +66,81 @@ public class MetaDataProducerHelper
                continue;
             }
 
-            for(Constructor<?> constructor : aClass.getConstructors())
+            if(MetaDataProducerInterface.class.isAssignableFrom(aClass))
             {
-               if(constructor.getParameterCount() == 0)
+               boolean foundValidConstructor = false;
+               for(Constructor<?> constructor : aClass.getConstructors())
                {
-                  Object o = constructor.newInstance();
-                  if(o instanceof MetaDataProducer<?> metaDataProducer)
+                  if(constructor.getParameterCount() == 0)
                   {
-                     producers.add(metaDataProducer);
+                     Object o = constructor.newInstance();
+                     producers.add((MetaDataProducerInterface<?>) o);
+                     foundValidConstructor = true;
+                     break;
                   }
-                  break;
+               }
+
+               if(!foundValidConstructor)
+               {
+                  LOG.warn("Found a class which implements MetaDataProducerInterface, but it does not have a no-arg constructor, so it cannot be used.", logPair("class", aClass.getSimpleName()));
                }
             }
          }
          catch(Exception e)
          {
-            LOG.info("Error adding metaData from producer", logPair("producer", aClass.getSimpleName()), e);
+            LOG.warn("Error evaluating a possible meta-data producer class", e, logPair("class", aClass.getSimpleName()));
          }
       }
 
-      /////////////////////////////
-      // sort them by sort order //
-      /////////////////////////////
-      producers.sort(Comparator.comparing(p -> p.getSortOrder()));
-
-      //////////////////////////////////////////////////////////////
-      // execute each one, adding their meta data to the instance //
-      //////////////////////////////////////////////////////////////
-      for(MetaDataProducer<?> producer : producers)
-      {
-         try
+      ////////////////////////////////////////////////////////////////////////////////////////////
+      // sort them by sort order, then by the type that they return - specifically - doing apps //
+      // after all other types (as apps often try to get other types from the instance)         //
+      ////////////////////////////////////////////////////////////////////////////////////////////
+      producers.sort(Comparator
+         .comparing((MetaDataProducerInterface<?> p) -> p.getSortOrder())
+         .thenComparing((MetaDataProducerInterface<?> p) ->
          {
-            TopLevelMetaDataInterface metaData = producer.produce(instance);
-            if(metaData != null)
+            try
             {
-               metaData.addSelfToInstance(instance);
+               Class<?> outputType = p.getClass().getMethod("produce", QInstance.class).getReturnType();
+               if(outputType.equals(QAppMetaData.class))
+               {
+                  return (1);
+               }
+               else
+               {
+                  return (0);
+               }
+            }
+            catch(Exception e)
+            {
+               return (0);
+            }
+         }));
+
+      ///////////////////////////////////////////////////////////////////////////
+      // execute each one (if enabled), adding their meta data to the instance //
+      ///////////////////////////////////////////////////////////////////////////
+      for(MetaDataProducerInterface<?> producer : producers)
+      {
+         if(producer.isEnabled())
+         {
+            try
+            {
+               TopLevelMetaDataInterface metaData = producer.produce(instance);
+               if(metaData != null)
+               {
+                  metaData.addSelfToInstance(instance);
+               }
+            }
+            catch(Exception e)
+            {
+               LOG.warn("error executing metaDataProducer", logPair("producer", producer.getClass().getSimpleName()), e);
             }
          }
-         catch(Exception e)
+         else
          {
-            LOG.warn("error executing metaDataProducer", logPair("producer", producer.getClass().getSimpleName()), e);
+            LOG.debug("Not using producer which is not enabled", logPair("producer", producer.getClass().getSimpleName()));
          }
       }
 

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/QInstance.java

@@ -93,6 +93,7 @@ public class QInstance
 
    private Map<String, QSupplementalInstanceMetaData> supplementalMetaData = new LinkedHashMap<>();
 
+   private String              deploymentMode;
    private Map<String, String> environmentValues = new LinkedHashMap<>();
    private String              defaultTimeZoneId = "UTC";
 
@@ -1165,4 +1166,36 @@ public class QInstance
       }
       this.joinGraph = joinGraph;
    }
+
+
+
+   /*******************************************************************************
+    ** Getter for deploymentMode
+    *******************************************************************************/
+   public String getDeploymentMode()
+   {
+      return (this.deploymentMode);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for deploymentMode
+    *******************************************************************************/
+   public void setDeploymentMode(String deploymentMode)
+   {
+      this.deploymentMode = deploymentMode;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for deploymentMode
+    *******************************************************************************/
+   public QInstance withDeploymentMode(String deploymentMode)
+   {
+      this.deploymentMode = deploymentMode;
+      return (this);
+   }
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/QSupplementalInstanceMetaData.java

@@ -30,7 +30,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
  ** Base-class for instance-level meta-data defined by some supplemental module, etc,
  ** outside of qqq core
  *******************************************************************************/
-public abstract class QSupplementalInstanceMetaData
+public abstract class QSupplementalInstanceMetaData implements TopLevelMetaDataInterface
 {
 
    /*******************************************************************************
@@ -61,4 +61,16 @@ public abstract class QSupplementalInstanceMetaData
       // noop in base class //
       ////////////////////////
    }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public void addSelfToInstance(QInstance qInstance)
+   {
+      qInstance.withSupplementalMetaData(this);
+   }
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/authentication/Auth0AuthenticationMetaData.java

@@ -60,7 +60,6 @@ public class Auth0AuthenticationMetaData extends QAuthenticationMetaData
    private String       auth0ClientSecretField;
    private Serializable qqqRecordIdField;
 
-
    /////////////////////////////////////
    // fields on the accessToken table //
    /////////////////////////////////////

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/authentication/QAuthenticationMetaData.java

@@ -26,6 +26,8 @@ import java.util.HashMap;
 import java.util.Map;
 import com.fasterxml.jackson.annotation.JsonFilter;
 import com.kingsrook.qqq.backend.core.model.metadata.QAuthenticationType;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.TopLevelMetaDataInterface;
 
 
 /*******************************************************************************
@@ -33,7 +35,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.QAuthenticationType;
  ** etc) within a qqq instance
  **
  *******************************************************************************/
-public class QAuthenticationMetaData
+public class QAuthenticationMetaData implements TopLevelMetaDataInterface
 {
    private String name;
    private QAuthenticationType type;
@@ -179,4 +181,15 @@ public class QAuthenticationMetaData
       return (this);
    }
 
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public void addSelfToInstance(QInstance qInstance)
+   {
+      qInstance.setAuthentication(this);
+   }
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/automation/QAutomationProviderMetaData.java

@@ -22,13 +22,15 @@
 package com.kingsrook.qqq.backend.core.model.metadata.automation;
 
 
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.TopLevelMetaDataInterface;
 import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.QScheduleMetaData;
 
 
 /*******************************************************************************
  ** Meta-data definition of a qqq service to drive record automations.
  *******************************************************************************/
-public class QAutomationProviderMetaData
+public class QAutomationProviderMetaData implements TopLevelMetaDataInterface
 {
    private String                  name;
    private QAutomationProviderType type;
@@ -137,4 +139,15 @@ public class QAutomationProviderMetaData
       return (this);
    }
 
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public void addSelfToInstance(QInstance qInstance)
+   {
+      qInstance.addAutomationProvider(this);
+   }
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/branding/QBrandingMetaData.java

@@ -22,11 +22,15 @@
 package com.kingsrook.qqq.backend.core.model.metadata.branding;
 
 
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.TopLevelMetaDataInterface;
+
+
 /*******************************************************************************
  ** Meta-Data to define branding in a QQQ instance.
  **
  *******************************************************************************/
-public class QBrandingMetaData
+public class QBrandingMetaData implements TopLevelMetaDataInterface
 {
    private String companyName;
    private String companyUrl;
@@ -309,4 +313,14 @@ public class QBrandingMetaData
       return (this);
    }
 
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public void addSelfToInstance(QInstance qInstance)
+   {
+      qInstance.setBranding(this);
+   }
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/dashboard/ParentWidgetMetaData.java

@@ -35,6 +35,16 @@ public class ParentWidgetMetaData extends QWidgetMetaData
    private List<String> childWidgetNameList;
    private List<String> childProcessNameList;
 
+   private LayoutType layoutType = LayoutType.GRID;
+
+
+
+   public enum LayoutType
+   {
+      GRID,
+      TABS
+   }
+
 
 
    /*******************************************************************************
@@ -137,4 +147,36 @@ public class ParentWidgetMetaData extends QWidgetMetaData
       return (this);
    }
 
+
+
+   /*******************************************************************************
+    ** Getter for layoutType
+    *******************************************************************************/
+   public LayoutType getLayoutType()
+   {
+      return (this.layoutType);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for layoutType
+    *******************************************************************************/
+   public void setLayoutType(LayoutType layoutType)
+   {
+      this.layoutType = layoutType;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for layoutType
+    *******************************************************************************/
+   public ParentWidgetMetaData withLayoutType(LayoutType layoutType)
+   {
+      this.layoutType = layoutType;
+      return (this);
+   }
+
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/dashboard/QWidgetMetaData.java

@@ -28,6 +28,7 @@ import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
 import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
+import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon;
 import com.kingsrook.qqq.backend.core.model.metadata.permissions.QPermissionRules;
 
 
@@ -40,6 +41,7 @@ public class QWidgetMetaData implements QWidgetMetaDataInterface
    protected String         name;
    protected String         icon;
    protected String         label;
+   protected String         tooltip;
    protected String         type;
    protected String         minHeight;
    protected String         footerHTML;
@@ -55,6 +57,8 @@ public class QWidgetMetaData implements QWidgetMetaDataInterface
    private boolean showReloadButton = true;
    private boolean showExportButton = true;
 
+   protected Map<String, QIcon> icons;
+
    protected Map<String, Serializable> defaultValues = new LinkedHashMap<>();
 
 
@@ -594,4 +598,81 @@ public class QWidgetMetaData implements QWidgetMetaDataInterface
       return (this);
    }
 
+
+
+   /*******************************************************************************
+    ** Getter for icons
+    *******************************************************************************/
+   public Map<String, QIcon> getIcons()
+   {
+      return (this.icons);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for icons
+    *******************************************************************************/
+   public void setIcons(Map<String, QIcon> icons)
+   {
+      this.icons = icons;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for icons
+    *******************************************************************************/
+   public QWidgetMetaData withIcon(String role, QIcon icon)
+   {
+      if(this.icons == null)
+      {
+         this.icons = new LinkedHashMap<>();
+      }
+      this.icons.put(role, icon);
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for icons
+    *******************************************************************************/
+   public QWidgetMetaData withIcons(Map<String, QIcon> icons)
+   {
+      this.icons = icons;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for tooltip
+    *******************************************************************************/
+   public String getTooltip()
+   {
+      return (this.tooltip);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for tooltip
+    *******************************************************************************/
+   public void setTooltip(String tooltip)
+   {
+      this.tooltip = tooltip;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for tooltip
+    *******************************************************************************/
+   public QWidgetMetaData withTooltip(String tooltip)
+   {
+      this.tooltip = tooltip;
+      return (this);
+   }
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/dashboard/QWidgetMetaDataInterface.java

@@ -25,6 +25,8 @@ package com.kingsrook.qqq.backend.core.model.metadata.dashboard;
 import java.io.Serializable;
 import java.util.List;
 import java.util.Map;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.TopLevelMetaDataInterface;
 import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
 import com.kingsrook.qqq.backend.core.model.metadata.permissions.MetaDataWithPermissionRules;
 import com.kingsrook.qqq.backend.core.model.metadata.permissions.QPermissionRules;
@@ -34,7 +36,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.permissions.QPermissionRule
  ** Interface for qqq widget meta data
  **
  *******************************************************************************/
-public interface QWidgetMetaDataInterface extends MetaDataWithPermissionRules
+public interface QWidgetMetaDataInterface extends MetaDataWithPermissionRules, TopLevelMetaDataInterface
 {
    /*******************************************************************************
     ** Getter for name
@@ -216,5 +218,23 @@ public interface QWidgetMetaDataInterface extends MetaDataWithPermissionRules
     ** Fluent setter for dropdowns
     *******************************************************************************/
    QWidgetMetaData withDropdown(WidgetDropdownData dropdown);
+
+
+   /*******************************************************************************
+    ** Getter for tooltip
+    *******************************************************************************/
+   default String getTooltip()
+   {
+      return (null);
+   }
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   default void addSelfToInstance(QInstance qInstance)
+   {
+      qInstance.addWidget(this);
+   }
+
 }
 

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/dashboard/QuickSightChartMetaData.java

@@ -32,7 +32,7 @@ import java.util.Map;
  ** AWS Quicksite specific meta data for frontend dashboard widget
  **
  *******************************************************************************/
-public class QuickSightChartMetaData extends QWidgetMetaData implements QWidgetMetaDataInterface
+public class QuickSightChartMetaData extends QWidgetMetaData
 {
    private String             accessKey;
    private String             secretKey;

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/dashboard/WidgetDropdownData.java

@@ -33,6 +33,16 @@ public class WidgetDropdownData
    private String  label;
    private boolean isRequired;
 
+   private Integer width;
+   private String  startIconName;
+   private Boolean allowBackAndForth;
+   private Boolean backAndForthInverted;
+   private Boolean disableClearable;
+
+   ////////////////////////////////////////////////////////////////////////////////////////////////
+   // an option to put at the top of the dropdown, that represents a value of "null" (e.g., All) //
+   ////////////////////////////////////////////////////////////////////////////////////////////////
+   private String labelForNullValue;
 
 
    /*******************************************************************************
@@ -169,4 +179,191 @@ public class WidgetDropdownData
       return (this);
    }
 
+
+
+   /*******************************************************************************
+    ** Getter for width
+    *******************************************************************************/
+   public Integer getWidth()
+   {
+      return (this.width);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for width
+    *******************************************************************************/
+   public void setWidth(Integer width)
+   {
+      this.width = width;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for width
+    *******************************************************************************/
+   public WidgetDropdownData withWidth(Integer width)
+   {
+      this.width = width;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for startIconName
+    *******************************************************************************/
+   public String getStartIconName()
+   {
+      return (this.startIconName);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for startIconName
+    *******************************************************************************/
+   public void setStartIconName(String startIconName)
+   {
+      this.startIconName = startIconName;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for startIconName
+    *******************************************************************************/
+   public WidgetDropdownData withStartIconName(String startIconName)
+   {
+      this.startIconName = startIconName;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for allowBackAndForth
+    *******************************************************************************/
+   public Boolean getAllowBackAndForth()
+   {
+      return (this.allowBackAndForth);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for allowBackAndForth
+    *******************************************************************************/
+   public void setAllowBackAndForth(Boolean allowBackAndForth)
+   {
+      this.allowBackAndForth = allowBackAndForth;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for allowBackAndForth
+    *******************************************************************************/
+   public WidgetDropdownData withAllowBackAndForth(Boolean allowBackAndForth)
+   {
+      this.allowBackAndForth = allowBackAndForth;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for disableClearable
+    *******************************************************************************/
+   public Boolean getDisableClearable()
+   {
+      return (this.disableClearable);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for disableClearable
+    *******************************************************************************/
+   public void setDisableClearable(Boolean disableClearable)
+   {
+      this.disableClearable = disableClearable;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for disableClearable
+    *******************************************************************************/
+   public WidgetDropdownData withDisableClearable(Boolean disableClearable)
+   {
+      this.disableClearable = disableClearable;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for labelForNullValue
+    *******************************************************************************/
+   public String getLabelForNullValue()
+   {
+      return (this.labelForNullValue);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for labelForNullValue
+    *******************************************************************************/
+   public void setLabelForNullValue(String labelForNullValue)
+   {
+      this.labelForNullValue = labelForNullValue;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for labelForNullValue
+    *******************************************************************************/
+   public WidgetDropdownData withLabelForNullValue(String labelForNullValue)
+   {
+      this.labelForNullValue = labelForNullValue;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for backAndForthInverted
+    *******************************************************************************/
+   public Boolean getBackAndForthInverted()
+   {
+      return (this.backAndForthInverted);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for backAndForthInverted
+    *******************************************************************************/
+   public void setBackAndForthInverted(Boolean backAndForthInverted)
+   {
+      this.backAndForthInverted = backAndForthInverted;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for backAndForthInverted
+    *******************************************************************************/
+   public WidgetDropdownData withBackAndForthInverted(Boolean backAndForthInverted)
+   {
+      this.backAndForthInverted = backAndForthInverted;
+      return (this);
+   }
+
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/QFieldMetaData.java

@@ -34,10 +34,13 @@ import java.util.Set;
 import com.fasterxml.jackson.annotation.JsonIgnore;
 import com.github.hervian.reflection.Fun;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.instances.QInstanceHelpContentManager;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
 import com.kingsrook.qqq.backend.core.model.data.QField;
 import com.kingsrook.qqq.backend.core.model.data.QRecordEntity;
 import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.help.HelpRole;
+import com.kingsrook.qqq.backend.core.model.metadata.help.QHelpContent;
 import com.kingsrook.qqq.backend.core.model.metadata.security.FieldSecurityLock;
 import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
 import com.kingsrook.qqq.backend.core.utils.StringUtils;
@@ -65,7 +68,7 @@ public class QFieldMetaData implements Cloneable
    // propose doing that in a secondary field, e.g., "onlyEditableOn=insert|update" //
    ///////////////////////////////////////////////////////////////////////////////////
 
-   private String       displayFormat = "%s";
+   private String displayFormat = "%s";
    private Serializable defaultValue;
    private String       possibleValueSourceName;
    private QQueryFilter possibleValueSourceFilter;
@@ -84,6 +87,7 @@ public class QFieldMetaData implements Cloneable
    ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
 
    private List<FieldAdornment> adornments;
+   private List<QHelpContent>   helpContents;
 
    private Map<String, QSupplementalFieldMetaData> supplementalMetaData;
 
@@ -928,4 +932,61 @@ public class QFieldMetaData implements Cloneable
       return (this);
    }
 
+
+
+   /*******************************************************************************
+    ** Getter for helpContents
+    *******************************************************************************/
+   public List<QHelpContent> getHelpContents()
+   {
+      return (this.helpContents);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for helpContents
+    *******************************************************************************/
+   public void setHelpContents(List<QHelpContent> helpContents)
+   {
+      this.helpContents = helpContents;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for helpContents
+    *******************************************************************************/
+   public QFieldMetaData withHelpContents(List<QHelpContent> helpContents)
+   {
+      this.helpContents = helpContents;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for adding 1 helpContent
+    *******************************************************************************/
+   public QFieldMetaData withHelpContent(QHelpContent helpContent)
+   {
+      if(this.helpContents == null)
+      {
+         this.helpContents = new ArrayList<>();
+      }
+
+      QInstanceHelpContentManager.putHelpContentInList(helpContent, this.helpContents);
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** remove a single helpContent based on its set of roles
+    *******************************************************************************/
+   public void removeHelpContent(Set<HelpRole> roles)
+   {
+      QInstanceHelpContentManager.removeHelpContentByRoleSetFromList(roles, this.helpContents);
+   }
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/frontend/QFrontendFieldMetaData.java

@@ -29,6 +29,7 @@ import com.fasterxml.jackson.annotation.JsonInclude.Include;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.FieldAdornment;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
+import com.kingsrook.qqq.backend.core.model.metadata.help.QHelpContent;
 
 
 /*******************************************************************************
@@ -50,6 +51,7 @@ public class QFrontendFieldMetaData
    private Serializable defaultValue;
 
    private List<FieldAdornment> adornments;
+   private List<QHelpContent>   helpContents;
 
    //////////////////////////////////////////////////////////////////////////////////
    // do not add setters.  take values from the source-object in the constructor!! //
@@ -72,6 +74,7 @@ public class QFrontendFieldMetaData
       this.displayFormat = fieldMetaData.getDisplayFormat();
       this.adornments = fieldMetaData.getAdornments();
       this.defaultValue = fieldMetaData.getDefaultValue();
+      this.helpContents = fieldMetaData.getHelpContents();
    }
 
 
@@ -183,4 +186,16 @@ public class QFrontendFieldMetaData
    {
       return defaultValue;
    }
+
+
+
+   /*******************************************************************************
+    ** Getter for helpContents
+    **
+    *******************************************************************************/
+   public List<QHelpContent> getHelpContents()
+   {
+      return helpContents;
+   }
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/frontend/QFrontendWidgetMetaData.java

@@ -23,6 +23,7 @@ package com.kingsrook.qqq.backend.core.model.metadata.frontend;
 
 
 import java.util.List;
+import java.util.Map;
 import com.fasterxml.jackson.annotation.JsonInclude;
 import com.fasterxml.jackson.annotation.JsonInclude.Include;
 import com.kingsrook.qqq.backend.core.actions.permissions.PermissionsHelper;
@@ -30,6 +31,7 @@ import com.kingsrook.qqq.backend.core.model.actions.AbstractActionInput;
 import com.kingsrook.qqq.backend.core.model.metadata.dashboard.QWidgetMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.dashboard.QWidgetMetaDataInterface;
 import com.kingsrook.qqq.backend.core.model.metadata.dashboard.WidgetDropdownData;
+import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon;
 
 
 /*******************************************************************************
@@ -42,6 +44,7 @@ public class QFrontendWidgetMetaData
 {
    private final String                   name;
    private final String                   label;
+   private final String                   tooltip;
    private final String                   type;
    private final String                   icon;
    private final Integer                  gridColumns;
@@ -54,10 +57,13 @@ public class QFrontendWidgetMetaData
    private boolean showReloadButton = false;
    private boolean showExportButton = false;
 
+   protected Map<String, QIcon> icons;
+
    private final boolean hasPermission;
 
    //////////////////////////////////////////////////////////////////////////////////
-   // do not add setters.  take values from the source-object in the constructor!! //
+   // DO add getters for all fields - this tells Jackson to include them in JSON.  //
+   // do NOT add setters.  take values from the source-object in the constructor!! //
    //////////////////////////////////////////////////////////////////////////////////
 
 
@@ -69,6 +75,7 @@ public class QFrontendWidgetMetaData
    {
       this.name = widgetMetaData.getName();
       this.label = widgetMetaData.getLabel();
+      this.tooltip = widgetMetaData.getTooltip();
       this.type = widgetMetaData.getType();
       this.icon = widgetMetaData.getIcon();
       this.gridColumns = widgetMetaData.getGridColumns();
@@ -82,6 +89,7 @@ public class QFrontendWidgetMetaData
       {
          this.showExportButton = qWidgetMetaData.getShowExportButton();
          this.showReloadButton = qWidgetMetaData.getShowReloadButton();
+         this.icons = qWidgetMetaData.getIcons();
       }
 
       hasPermission = PermissionsHelper.hasWidgetPermission(actionInput, name);
@@ -229,4 +237,26 @@ public class QFrontendWidgetMetaData
    {
       return showExportButton;
    }
+
+
+
+   /*******************************************************************************
+    ** Getter for icons
+    **
+    *******************************************************************************/
+   public Map<String, QIcon> getIcons()
+   {
+      return icons;
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for tooltip
+    **
+    *******************************************************************************/
+   public String getTooltip()
+   {
+      return tooltip;
+   }
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/help/HelpFormat.java

@@ -0,0 +1,33 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2023.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.model.metadata.help;
+
+
+/*******************************************************************************
+ ** How a piece of help content is formatted.
+ *******************************************************************************/
+public enum HelpFormat
+{
+   TEXT,
+   HTML,
+   MARKDOWN
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/help/HelpRole.java

@@ -0,0 +1,35 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2023.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.model.metadata.help;
+
+
+/*******************************************************************************
+ ** Interface to be associated with a HelpContent, to identify where the content
+ ** is meant to be used (e.g., only on "write" screens, vs. on app home pages, etc).
+ **
+ ** Defined in HelpContext to be this interface, so alternate frontends can
+ ** specify their own particular values - but a standard set of values is provided
+ ** by QQQ in QHelpRole.
+ *******************************************************************************/
+public interface HelpRole
+{
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/help/QHelpContent.java

@@ -0,0 +1,232 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2023.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.model.metadata.help;
+
+
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Set;
+
+
+/*******************************************************************************
+ ** meta-data defintion of "Help Content" to show to a user - for use in
+ ** a specific "role" (e.g., insert screens but not view screens), and in a
+ ** particular "format" (e.g., plain text, html, markdown).
+ **
+ ** Meant to be assigned to several different pieces of QQQ meta data (fields,
+ ** tables, processes, etc), and used as-needed by various frontends.
+ **
+ ** May evolve something like a "Presentation" attribute in the future - e.g.,
+ ** to say "present this one as a tooltip" vs. "present this one as inline text"
+ **
+ ** May be dynamically added to meta-data via (non-meta-) data - see
+ ** HelpContentMetaDataProvider and QInstanceHelpContentManager
+ *******************************************************************************/
+public class QHelpContent
+{
+   private String        content;
+   private HelpFormat    format;
+   private Set<HelpRole> roles;
+
+
+
+   /*******************************************************************************
+    ** Constructor
+    **
+    *******************************************************************************/
+   public QHelpContent()
+   {
+   }
+
+
+
+   /*******************************************************************************
+    ** Constructor
+    **
+    *******************************************************************************/
+   public QHelpContent(String content)
+   {
+      setContent(content);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for content
+    *******************************************************************************/
+   public String getContent()
+   {
+      return (this.content);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for content
+    *******************************************************************************/
+   public void setContent(String content)
+   {
+      this.content = content;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for content
+    *******************************************************************************/
+   public QHelpContent withContent(String content)
+   {
+      this.content = content;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for content that also sets format as HTML
+    *******************************************************************************/
+   public QHelpContent withContentAsHTML(String content)
+   {
+      this.content = content;
+      this.format = HelpFormat.HTML;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for content that also sets format as TEXT
+    *******************************************************************************/
+   public QHelpContent withContentAsText(String content)
+   {
+      this.content = content;
+      this.format = HelpFormat.TEXT;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for content that also sets format as Markdown
+    *******************************************************************************/
+   public QHelpContent withContentAsMarkdown(String content)
+   {
+      this.content = content;
+      this.format = HelpFormat.MARKDOWN;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for format
+    *******************************************************************************/
+   public HelpFormat getFormat()
+   {
+      return (this.format);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for format
+    *******************************************************************************/
+   public void setFormat(HelpFormat format)
+   {
+      this.format = format;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for format
+    *******************************************************************************/
+   public QHelpContent withFormat(HelpFormat format)
+   {
+      this.format = format;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for roles
+    *******************************************************************************/
+   public Set<HelpRole> getRoles()
+   {
+      return (this.roles);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for roles
+    *******************************************************************************/
+   public void setRoles(Set<HelpRole> roles)
+   {
+      this.roles = roles;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for roles
+    *******************************************************************************/
+   public QHelpContent withRoles(Set<HelpRole> roles)
+   {
+      this.roles = roles;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent method to add a role
+    *******************************************************************************/
+   public QHelpContent withRole(HelpRole role)
+   {
+      return (withRoles(role));
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent method to add a role
+    *******************************************************************************/
+   public QHelpContent withRoles(HelpRole... roles)
+   {
+      if(roles == null || roles.length == 0)
+      {
+         return (this);
+      }
+
+      if(this.roles == null)
+      {
+         this.roles = new HashSet<>();
+      }
+
+      Collections.addAll(this.roles, roles);
+
+      return (this);
+   }
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/help/QHelpRole.java

@@ -0,0 +1,40 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2023.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.model.metadata.help;
+
+
+/*******************************************************************************
+ ** QQQ default or standard HelpRoles.
+ *******************************************************************************/
+public enum QHelpRole implements HelpRole
+{
+   ALL_SCREENS,
+   READ_SCREENS,
+   WRITE_SCREENS,
+   QUERY_SCREEN,
+   VIEW_SCREEN,
+   EDIT_SCREEN,
+   INSERT_SCREEN,
+   PROCESS_SCREEN,
+   APP_SCREEN,
+   TABLE_ACTION_MENU
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/layout/QIcon.java

@@ -35,6 +35,7 @@ public class QIcon
 {
    private String name;
    private String path;
+   private String color;
 
 
 
@@ -123,4 +124,36 @@ public class QIcon
       return (this);
    }
 
+
+
+   /*******************************************************************************
+    ** Getter for color
+    *******************************************************************************/
+   public String getColor()
+   {
+      return (this.color);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for color
+    *******************************************************************************/
+   public void setColor(String color)
+   {
+      this.color = color;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for color
+    *******************************************************************************/
+   public QIcon withColor(String color)
+   {
+      this.color = color;
+      return (this);
+   }
+
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/possiblevalues/QPossibleValueSource.java

@@ -92,6 +92,21 @@ public class QPossibleValueSource implements TopLevelMetaDataInterface
 
 
 
+   /*******************************************************************************
+    ** Create a new possible value source, for an enum, with default settings.
+    ** e.g., type=ENUM; name from param values from the param; LABEL_ONLY format
+    *******************************************************************************/
+   public static <T extends PossibleValueEnum<?>> QPossibleValueSource newForEnum(String name, T[] values)
+   {
+      return new QPossibleValueSource()
+         .withName(name)
+         .withType(QPossibleValueSourceType.ENUM)
+         .withValuesFromEnum(values)
+         .withValueFormatAndFields(PVSValueFormatAndFields.LABEL_ONLY);
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/queues/QQueueProviderMetaData.java

@@ -22,10 +22,14 @@
 package com.kingsrook.qqq.backend.core.model.metadata.queues;
 
 
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.TopLevelMetaDataInterface;
+
+
 /*******************************************************************************
  ** Define a provider of queues (e.g., an MQ system, or SQS)
  *******************************************************************************/
-public class QQueueProviderMetaData
+public class QQueueProviderMetaData implements TopLevelMetaDataInterface
 {
    private String    name;
    private QueueType type;
@@ -98,4 +102,15 @@ public class QQueueProviderMetaData
       return (this);
    }
 
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public void addSelfToInstance(QInstance qInstance)
+   {
+      qInstance.addQueueProvider(this);
+   }
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/reporting/QReportMetaData.java

@@ -24,6 +24,8 @@ package com.kingsrook.qqq.backend.core.model.metadata.reporting;
 
 import java.util.ArrayList;
 import java.util.List;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.TopLevelMetaDataInterface;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppChildMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon;
@@ -35,7 +37,7 @@ import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
 /*******************************************************************************
  ** Meta-data definition of a report generated by QQQ
  *******************************************************************************/
-public class QReportMetaData implements QAppChildMetaData, MetaDataWithPermissionRules
+public class QReportMetaData implements QAppChildMetaData, MetaDataWithPermissionRules, TopLevelMetaDataInterface
 {
    private String name;
    private String label;
@@ -384,4 +386,15 @@ public class QReportMetaData implements QAppChildMetaData, MetaDataWithPermissio
       return (this);
    }
 
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public void addSelfToInstance(QInstance qInstance)
+   {
+      qInstance.addReport(this);
+   }
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/security/QSecurityKeyType.java

@@ -22,11 +22,15 @@
 package com.kingsrook.qqq.backend.core.model.metadata.security;
 
 
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.TopLevelMetaDataInterface;
+
+
 /*******************************************************************************
  ** Define a type of security key (e.g., a field associated with values), that
  ** can be used to control access to records and/or fields
  *******************************************************************************/
-public class QSecurityKeyType
+public class QSecurityKeyType implements TopLevelMetaDataInterface
 {
    private String name;
    private String allAccessKeyName;
@@ -134,4 +138,15 @@ public class QSecurityKeyType
       return (this);
    }
 
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public void addSelfToInstance(QInstance qInstance)
+   {
+      qInstance.addSecurityKeyType(this);
+   }
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/QFieldSection.java

@@ -22,7 +22,12 @@
 package com.kingsrook.qqq.backend.core.model.metadata.tables;
 
 
+import java.util.ArrayList;
 import java.util.List;
+import java.util.Set;
+import com.kingsrook.qqq.backend.core.instances.QInstanceHelpContentManager;
+import com.kingsrook.qqq.backend.core.model.metadata.help.HelpRole;
+import com.kingsrook.qqq.backend.core.model.metadata.help.QHelpContent;
 import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon;
 import com.kingsrook.qqq.backend.core.utils.collections.MutableList;
 
@@ -44,6 +49,8 @@ public class QFieldSection
    private boolean isHidden = false;
    private Integer gridColumns;
 
+   private List<QHelpContent> helpContents;
+
 
 
    /*******************************************************************************
@@ -364,4 +371,61 @@ public class QFieldSection
       return (this);
    }
 
+
+
+   /*******************************************************************************
+    ** Getter for helpContents
+    *******************************************************************************/
+   public List<QHelpContent> getHelpContents()
+   {
+      return (this.helpContents);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for helpContents
+    *******************************************************************************/
+   public void setHelpContents(List<QHelpContent> helpContents)
+   {
+      this.helpContents = helpContents;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for helpContents
+    *******************************************************************************/
+   public QFieldSection withHelpContents(List<QHelpContent> helpContents)
+   {
+      this.helpContents = helpContents;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for adding 1 helpContent
+    *******************************************************************************/
+   public QFieldSection withHelpContent(QHelpContent helpContent)
+   {
+      if(this.helpContents == null)
+      {
+         this.helpContents = new ArrayList<>();
+      }
+
+      QInstanceHelpContentManager.putHelpContentInList(helpContent, this.helpContents);
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** remove a single helpContent based on its set of roles
+    *******************************************************************************/
+   public void removeHelpContent(Set<HelpRole> roles)
+   {
+      QInstanceHelpContentManager.removeHelpContentByRoleSetFromList(roles, this.helpContents);
+   }
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/querystats/QueryStatMetaDataProvider.java

@@ -187,7 +187,8 @@ public class QueryStatMetaDataProvider
       return (new QPossibleValueSource()
          .withType(QPossibleValueSourceType.TABLE)
          .withName(QueryStat.TABLE_NAME)
-         .withTableName(QueryStat.TABLE_NAME));
+         .withTableName(QueryStat.TABLE_NAME))
+         .withOrderByField("id", false);
    }
 
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/savedfilters/SavedFiltersMetaDataProvider.java

@@ -88,7 +88,8 @@ public class SavedFiltersMetaDataProvider
          .withName(SavedFilter.TABLE_NAME)
          .withType(QPossibleValueSourceType.TABLE)
          .withTableName(SavedFilter.TABLE_NAME)
-         .withValueFormatAndFields(PVSValueFormatAndFields.LABEL_ONLY);
+         .withValueFormatAndFields(PVSValueFormatAndFields.LABEL_ONLY)
+         .withOrderByField("label");
    }
 
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/scripts/ScriptsMetaDataProvider.java

@@ -303,19 +303,24 @@ public class ScriptsMetaDataProvider
    {
       instance.addPossibleValueSource(new QPossibleValueSource()
          .withName(Script.TABLE_NAME)
-         .withTableName(Script.TABLE_NAME));
+         .withTableName(Script.TABLE_NAME)
+         .withOrderByField("name"));
 
       instance.addPossibleValueSource(new QPossibleValueSource()
          .withName(ScriptRevision.TABLE_NAME)
-         .withTableName(ScriptRevision.TABLE_NAME));
+         .withTableName(ScriptRevision.TABLE_NAME)
+         .withOrderByField("scriptId")
+         .withOrderByField("sequenceNo", false));
 
       instance.addPossibleValueSource(new QPossibleValueSource()
          .withName(ScriptType.TABLE_NAME)
-         .withTableName(ScriptType.TABLE_NAME));
+         .withTableName(ScriptType.TABLE_NAME)
+         .withOrderByField("name"));
 
       instance.addPossibleValueSource(new QPossibleValueSource()
          .withName(ScriptLog.TABLE_NAME)
-         .withTableName(ScriptLog.TABLE_NAME));
+         .withTableName(ScriptLog.TABLE_NAME)
+         .withOrderByField("id", false));
 
       instance.addPossibleValueSource(new QPossibleValueSource()
          .withName(ScriptTypeFileMode.NAME)
@@ -378,7 +383,7 @@ public class ScriptsMetaDataProvider
    /*******************************************************************************
     **
     *******************************************************************************/
-   private QTableMetaData defineTableTriggerTable(String backendName) throws QException
+   public QTableMetaData defineTableTriggerTable(String backendName) throws QException
    {
       QTableMetaData tableMetaData = defineStandardTable(backendName, TableTrigger.TABLE_NAME, TableTrigger.class)
          .withRecordLabelFields("id")

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/tables/QQQTablesMetaDataProvider.java

@@ -127,7 +127,8 @@ public class QQQTablesMetaDataProvider
       return (new QPossibleValueSource()
          .withType(QPossibleValueSourceType.TABLE)
          .withName(QQQTable.TABLE_NAME)
-         .withTableName(QQQTable.TABLE_NAME));
+         .withTableName(QQQTable.TABLE_NAME))
+         .withOrderByField("label");
    }
 
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/authentication/implementations/Auth0AuthenticationModule.java

@@ -33,7 +33,6 @@ import java.util.List;
 import java.util.Map;
 import java.util.Optional;
 import java.util.Set;
-import java.util.UUID;
 import com.auth0.client.auth.AuthAPI;
 import com.auth0.exception.Auth0Exception;
 import com.auth0.json.auth.TokenHolder;
@@ -52,6 +51,7 @@ import com.kingsrook.qqq.backend.core.actions.tables.GetAction;
 import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
 import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
 import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction;
+import com.kingsrook.qqq.backend.core.context.CapturedContext;
 import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.exceptions.AccessTokenException;
 import com.kingsrook.qqq.backend.core.exceptions.QAuthenticationException;
@@ -68,11 +68,11 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
 import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
 import com.kingsrook.qqq.backend.core.model.metadata.authentication.Auth0AuthenticationMetaData;
-import com.kingsrook.qqq.backend.core.model.metadata.authentication.QAuthenticationMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.security.QSecurityKeyType;
 import com.kingsrook.qqq.backend.core.model.session.QSession;
 import com.kingsrook.qqq.backend.core.model.session.QUser;
 import com.kingsrook.qqq.backend.core.modules.authentication.QAuthenticationModuleInterface;
+import com.kingsrook.qqq.backend.core.modules.authentication.implementations.model.UserSession;
 import com.kingsrook.qqq.backend.core.state.InMemoryStateProvider;
 import com.kingsrook.qqq.backend.core.state.SimpleStateKey;
 import com.kingsrook.qqq.backend.core.state.StateProviderInterface;
@@ -80,7 +80,6 @@ import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
 import com.kingsrook.qqq.backend.core.utils.JsonUtils;
 import com.kingsrook.qqq.backend.core.utils.StringUtils;
 import com.kingsrook.qqq.backend.core.utils.ValueUtils;
-import org.apache.http.HttpStatus;
 import org.apache.http.client.entity.UrlEncodedFormEntity;
 import org.apache.http.client.methods.CloseableHttpResponse;
 import org.apache.http.client.methods.HttpPost;
@@ -90,9 +89,23 @@ import org.apache.http.message.BasicNameValuePair;
 import org.apache.http.util.EntityUtils;
 import org.json.JSONArray;
 import org.json.JSONObject;
+import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
 
 
 /*******************************************************************************
+ ** QQQ AuthenticationModule for working with Auth0.
+ **
+ ** createSession can be called with the following fields in its context:
+ **
+ ** System-User session use-case:
+ ** 1: Takes in an "accessToken" (but doesn't store a userSession record).
+ ** 1b: legacy frontend use-case does the same as system-user!
+ **
+ ** Web User session use-cases:
+ ** 2: creates a new session (userSession record) by taking an "accessToken"
+ ** 3: looks up an existing session (userSession record) by taking a "sessionUUID"
+ ** 4: takes an "apiKey" (looked up in metaData.AccessTokenTableName - refreshing accessToken with auth0 if needed).
+ ** 5: takes a "basicAuthString" (encoded username:password), which make a new accessToken in auth0
  **
  *******************************************************************************/
 public class Auth0AuthenticationModule implements QAuthenticationModuleInterface
@@ -104,14 +117,17 @@ public class Auth0AuthenticationModule implements QAuthenticationModuleInterface
    /////////////////////////////////////////////////////////////////////////////////////////////////////////////
    public static final int ID_TOKEN_VALIDATION_INTERVAL_SECONDS = 1800;
 
-   public static final String AUTH0_ACCESS_TOKEN_KEY = "sessionId";
-   public static final String API_KEY                = "apiKey";
-   public static final String BASIC_AUTH_KEY         = "basicAuthString";
+   public static final String ACCESS_TOKEN_KEY = "accessToken";
+   public static final String API_KEY          = "apiKey"; // todo - look for users of this, see if we can change to use this constant; maybe move constants up?
+   public static final String SESSION_UUID_KEY = "sessionUUID";
+   public static final String BASIC_AUTH_KEY   = "basicAuthString"; // todo - look for users of this, see if we can change to use this constant; maybe move constants up?
 
-   public static final String TOKEN_NOT_PROVIDED_ERROR = "Access Token was not provided";
-   public static final String COULD_NOT_DECODE_ERROR   = "Unable to decode access token";
-   public static final String EXPIRED_TOKEN_ERROR      = "Token has expired";
-   public static final String INVALID_TOKEN_ERROR      = "An invalid token was provided";
+   public static final String DO_STORE_USER_SESSION_KEY = "doStoreUserSession";
+
+   static final String TOKEN_NOT_PROVIDED_ERROR = "Access Token was not provided";
+   static final String COULD_NOT_DECODE_ERROR   = "Unable to decode access token";
+   static final String EXPIRED_TOKEN_ERROR      = "Token has expired";
+   static final String INVALID_TOKEN_ERROR      = "An invalid token was provided";
 
 
    ////////////////////////////////////////////////////////////////////////////////////////////////////////////
@@ -149,94 +165,121 @@ public class Auth0AuthenticationModule implements QAuthenticationModuleInterface
    @Override
    public QSession createSession(QInstance qInstance, Map<String, String> context) throws QAuthenticationException
    {
-      Auth0AuthenticationMetaData metaData = (Auth0AuthenticationMetaData) qInstance.getAuthentication();
-
-      ///////////////////////////////////////////////////////////
-      // check if we are processing a Basic Auth Session first //
-      ///////////////////////////////////////////////////////////
-      if(context.containsKey(BASIC_AUTH_KEY))
-      {
-         AuthAPI auth = AuthAPI.newBuilder(metaData.getBaseUrl(), metaData.getClientId(), metaData.getClientSecret()).build();
-         try
-         {
-            /////////////////////////////////////////////////
-            // decode the credentials from the header auth //
-            /////////////////////////////////////////////////
-            String base64Credentials = context.get(BASIC_AUTH_KEY).trim();
-            String accessToken       = getAccessTokenFromBase64BasicAuthCredentials(metaData, auth, base64Credentials);
-            context.put(AUTH0_ACCESS_TOKEN_KEY, accessToken);
-         }
-         catch(Auth0Exception e)
-         {
-            ////////////////
-            // ¯\_(ツ)_/¯ //
-            ////////////////
-            String message = "Error handling basic authentication: " + e.getMessage();
-            LOG.error(message, e);
-            throw (new QAuthenticationException(message));
-         }
-      }
-
-      ////////////////////////////////////////////////////////////////////
-      // get the jwt id or qqq translated token from the context object //
-      ////////////////////////////////////////////////////////////////////
-      String accessToken = context.get(AUTH0_ACCESS_TOKEN_KEY);
-
-      /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-      // check to see if the session id is a UUID, if so, that means we need to look up the 'actual' token in the access_token table //
-      /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-      if(accessToken != null && StringUtils.isUUID(accessToken))
-      {
-         accessToken = lookupActualAccessToken(metaData, accessToken);
-      }
-
-      ////////////////////////////////////////////////////////
-      // if access token is still null, look for an api key //
-      ////////////////////////////////////////////////////////
-      if(accessToken == null)
-      {
-         String apiKey = context.get(API_KEY);
-         if(apiKey != null)
-         {
-            accessToken = getAccessTokenFromApiKey(metaData, apiKey);
-         }
-      }
-
-      if(accessToken == null)
-      {
-         LOG.warn(TOKEN_NOT_PROVIDED_ERROR);
-         throw (new QAuthenticationException(TOKEN_NOT_PROVIDED_ERROR));
-      }
-
-      //////////////////////////////////////////////////////////////////////////////////////
-      // decode the token locally to make sure it is valid and to look at when it expires //
-      //////////////////////////////////////////////////////////////////////////////////////
       try
       {
+         Auth0AuthenticationMetaData metaData = (Auth0AuthenticationMetaData) qInstance.getAuthentication();
+
+         String accessToken = null;
+         if(CollectionUtils.containsKeyWithNonNullValue(context, SESSION_UUID_KEY))
+         {
+            /////////////////////////////////////////////////////////////////////////////////////////
+            // process a sessionUUID - looks up userSession record - cannot create token this way. //
+            /////////////////////////////////////////////////////////////////////////////////////////
+            String sessionUUID = context.get(SESSION_UUID_KEY);
+            LOG.info("Creating session from sessionUUID (userSession)", logPair("sessionUUID", maskForLog(sessionUUID)));
+            if(sessionUUID != null)
+            {
+               accessToken = getAccessTokenFromSessionUUID(metaData, sessionUUID);
+            }
+         }
+         else if(CollectionUtils.containsKeyWithNonNullValue(context, ACCESS_TOKEN_KEY))
+         {
+            //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+            // if the context contains an access token, then create a new session based on that token.                          //
+            // todo#authHeader - this else/if should maybe be first, but while we have frontend passing both, we want it second //
+            //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+            accessToken = context.get(ACCESS_TOKEN_KEY);
+            QSession qSession = buildAndValidateSession(qInstance, accessToken);
+
+            ////////////////////////////////////////////////////////////////
+            // build & store userSession db record, if requested to do so //
+            ////////////////////////////////////////////////////////////////
+            if(CollectionUtils.containsKeyWithNonNullValue(context, DO_STORE_USER_SESSION_KEY))
+            {
+               insertUserSession(qInstance, accessToken, qSession);
+               LOG.info("Creating session based on input accessToken and creating a userSession", logPair("userId", qSession.getUser().getIdReference()));
+            }
+            else
+            {
+               ///////////////////////////////////////////////
+               // todo#authHeader - remove all this logging //
+               ///////////////////////////////////////////////
+               String userName = qSession.getUser() != null ? qSession.getUser().getFullName() : null;
+               if(userName != null && !userName.contains("System User"))
+               {
+                  LOG.info("Creating session based on input accessToken but not creating a userSession", logPair("userName", qSession.getUser().getFullName()));
+               }
+            }
+
+            return (qSession);
+         }
+         else if(CollectionUtils.containsKeyWithNonNullValue(context, BASIC_AUTH_KEY))
+         {
+            //////////////////////////////////////////////////////////////////////////////////////
+            // Process a basic auth (username:password)                                         //
+            // by getting an access token from auth0 (re-using from state provider if possible) //
+            //////////////////////////////////////////////////////////////////////////////////////
+            AuthAPI auth = AuthAPI.newBuilder(metaData.getBaseUrl(), metaData.getClientId(), metaData.getClientSecret()).build();
+            try
+            {
+               /////////////////////////////////////////////////
+               // decode the credentials from the header auth //
+               /////////////////////////////////////////////////
+               String base64Credentials = context.get(BASIC_AUTH_KEY).trim();
+               LOG.info("Creating session from basicAuthentication", logPair("base64Credentials", maskForLog(base64Credentials)));
+               accessToken = getAccessTokenFromBase64BasicAuthCredentials(metaData, auth, base64Credentials);
+            }
+            catch(Auth0Exception e)
+            {
+               ////////////////
+               // ¯\_(ツ)_/¯ //
+               ////////////////
+               String message = "Error handling basic authentication: " + e.getMessage();
+               LOG.error(message, e);
+               throw (new QAuthenticationException(message));
+            }
+         }
+         else if(CollectionUtils.containsKeyWithNonNullValue(context, API_KEY))
+         {
+            ///////////////////////////////////////////////////////////////////////////////////////
+            // process an api key - looks up client application token (creating token if needed) //
+            ///////////////////////////////////////////////////////////////////////////////////////
+            String apiKey = context.get(API_KEY);
+            LOG.info("Creating session from apiKey (accessTokenTable)", logPair("apiKey", maskForLog(apiKey)));
+            if(apiKey != null)
+            {
+               accessToken = getAccessTokenFromApiKey(metaData, apiKey);
+            }
+         }
+
+         /* todo confirm this is deprecated
+         /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+         // check to see if the session id is a UUID, if so, that means we need to look up the 'actual' token in the access_token table //
+         /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+         if(accessToken != null && StringUtils.isUUID(accessToken))
+         {
+            accessToken = lookupActualAccessToken(metaData, accessToken);
+         }
+         */
+
+         ///////////////////////////////////////////
+         // if token wasn't found by now, give up //
+         ///////////////////////////////////////////
+         if(accessToken == null)
+         {
+            LOG.warn(TOKEN_NOT_PROVIDED_ERROR);
+            throw (new QAuthenticationException(TOKEN_NOT_PROVIDED_ERROR));
+         }
+
          /////////////////////////////////////////////////////
          // try to build session to see if still valid      //
          // then call method to check more session validity //
          /////////////////////////////////////////////////////
-         QSession qSession = buildQSessionFromToken(accessToken, qInstance);
-         if(isSessionValid(qInstance, qSession))
-         {
-            return (qSession);
-         }
-
-         ///////////////////////////////////////////////////////////////////////////////////////
-         // if we make it here it means we have never validated this token or its been a long //
-         // enough duration so we need to re-verify the token                                 //
-         ///////////////////////////////////////////////////////////////////////////////////////
-         qSession = revalidateTokenAndBuildSession(qInstance, accessToken);
-
-         ////////////////////////////////////////////////////////////////////
-         // put now into state so we dont check until next interval passes //
-         ///////////////////////////////////////////////////////////////////
-         StateProviderInterface spi = getStateProvider();
-         SimpleStateKey<String> key = new SimpleStateKey<>(qSession.getIdReference());
-         spi.put(key, Instant.now());
-
-         return (qSession);
+         return buildAndValidateSession(qInstance, accessToken);
+      }
+      catch(QAuthenticationException qae)
+      {
+         throw (qae);
       }
       catch(JWTDecodeException jde)
       {
@@ -272,6 +315,61 @@ public class Auth0AuthenticationModule implements QAuthenticationModuleInterface
 
 
 
+   /*******************************************************************************
+    ** Insert a session as a new record into userSession table
+    *******************************************************************************/
+   private void insertUserSession(QInstance qInstance, String accessToken, QSession qSession) throws QException
+   {
+      CapturedContext capturedContext = QContext.capture();
+      try
+      {
+         QContext.init(qInstance, null);
+         QContext.setQSession(getChickenAndEggSession());
+
+         UserSession userSession = new UserSession()
+            .withUuid(qSession.getUuid())
+            .withUserId(qSession.getUser().getIdReference())
+            .withAccessToken(accessToken);
+
+         new InsertAction().execute(new InsertInput(UserSession.TABLE_NAME).withRecordEntity(userSession));
+      }
+      finally
+      {
+         QContext.init(capturedContext);
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private QSession buildAndValidateSession(QInstance qInstance, String accessToken) throws JwkException
+   {
+      QSession qSession = buildQSessionFromToken(accessToken, qInstance);
+      if(isSessionValid(qInstance, qSession))
+      {
+         return (qSession);
+      }
+
+      //////////////////////////////////////////////////////////////////////////////////////////
+      // if we make it here it means we have never validated this token or it has been a long //
+      // enough duration so we need to re-verify the token                                    //
+      //////////////////////////////////////////////////////////////////////////////////////////
+      qSession = revalidateTokenAndBuildSession(qInstance, accessToken);
+
+      /////////////////////////////////////////////////////////////////////
+      // put now into state so we don't check until next interval passes //
+      /////////////////////////////////////////////////////////////////////
+      StateProviderInterface spi = getStateProvider();
+      SimpleStateKey<String> key = new SimpleStateKey<>(qSession.getIdReference());
+      spi.put(key, Instant.now());
+
+      return (qSession);
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/
@@ -299,7 +397,7 @@ public class Auth0AuthenticationModule implements QAuthenticationModuleInterface
       byte[] credDecoded = Base64.getDecoder().decode(base64Credentials);
       String credentials = new String(credDecoded, StandardCharsets.UTF_8);
 
-      String accessToken = getAccessTokenFromAuth0(metaData, auth, credentials);
+      String accessToken = getAccessTokenForUsernameAndPasswordFromAuth0(metaData, auth, credentials);
       stateProvider.put(accessTokenStateKey, accessToken);
       stateProvider.put(timestampStateKey, Instant.now());
       return (accessToken);
@@ -310,7 +408,7 @@ public class Auth0AuthenticationModule implements QAuthenticationModuleInterface
    /*******************************************************************************
     **
     *******************************************************************************/
-   protected String getAccessTokenFromAuth0(Auth0AuthenticationMetaData metaData, AuthAPI auth, String credentials) throws Auth0Exception
+   protected String getAccessTokenForUsernameAndPasswordFromAuth0(Auth0AuthenticationMetaData metaData, AuthAPI auth, String credentials) throws Auth0Exception
    {
       /////////////////////////////////////
       // call auth0 with a login request //
@@ -620,75 +718,11 @@ public class Auth0AuthenticationModule implements QAuthenticationModuleInterface
 
 
    /*******************************************************************************
-    ** create a new auth0 access token
+    ** make http request to Auth0 for a new access token for an application - e.g.,
+    ** with a clientId and clientSecret as params
     **
     *******************************************************************************/
-   public String createAccessToken(QAuthenticationMetaData metaData, String clientId, String clientSecret) throws AccessTokenException
-   {
-      QSession                    sessionBefore = QContext.getQSession();
-      Auth0AuthenticationMetaData auth0MetaData = (Auth0AuthenticationMetaData) metaData;
-
-      try
-      {
-         QContext.setQSession(getChickenAndEggSession());
-
-         ///////////////////////////////////////////////////////////////////////////////////////
-         // fetch the application from database, will throw accesstokenexception if not found //
-         ///////////////////////////////////////////////////////////////////////////////////////
-         QRecord clientAuth0Application = getClientAuth0Application(auth0MetaData, clientId);
-
-         /////////////////////////////////////////////////////////////////////////////////////////////////
-         // request access token from auth0 if exception is not thrown, that means 200OK, we want to    //
-         // store the actual access token in the database, and return a unique value                    //
-         // back to the user which will be what they use on subseqeunt requests (because token too big) //
-         /////////////////////////////////////////////////////////////////////////////////////////////////
-         JSONObject accessTokenData = requestAccessTokenFromAuth0(auth0MetaData, clientId, clientSecret);
-
-         Integer expiresInSeconds = accessTokenData.getInt("expires_in");
-         String  accessToken      = accessTokenData.getString("access_token");
-         String  uuid             = UUID.randomUUID().toString();
-
-         /////////////////////////////////
-         // store the details in the db //
-         /////////////////////////////////
-         QRecord accessTokenRecord = new QRecord()
-            .withValue(auth0MetaData.getClientAuth0ApplicationIdField(), clientAuth0Application.getValue("id"))
-            .withValue(auth0MetaData.getAuth0AccessTokenField(), accessToken)
-            .withValue(auth0MetaData.getQqqAccessTokenField(), uuid)
-            .withValue(auth0MetaData.getExpiresInSecondsField(), expiresInSeconds);
-         InsertInput input = new InsertInput();
-         input.setTableName(auth0MetaData.getAccessTokenTableName());
-         input.setRecords(List.of(accessTokenRecord));
-         new InsertAction().execute(input);
-
-         //////////////////////////////////
-         // update and send the response //
-         //////////////////////////////////
-         accessTokenData.put("access_token", uuid);
-         accessTokenData.remove("scope");
-         return (accessTokenData.toString());
-      }
-      catch(AccessTokenException ate)
-      {
-         throw (ate);
-      }
-      catch(Exception e)
-      {
-         throw (new AccessTokenException(e.getMessage(), e));
-      }
-      finally
-      {
-         QContext.setQSession(sessionBefore);
-      }
-   }
-
-
-
-   /*******************************************************************************
-    ** make http request to Auth0 for a new access token
-    **
-    *******************************************************************************/
-   public JSONObject requestAccessTokenFromAuth0(Auth0AuthenticationMetaData auth0MetaData, String clientId, String clientSecret) throws AccessTokenException
+   public JSONObject requestAccessTokenForClientIdAndSecretFromAuth0(Auth0AuthenticationMetaData auth0MetaData, String clientId, String clientSecret) throws AccessTokenException
    {
       ///////////////////////////////////////////////////////////////////
       // make a request to Auth0 using the client_id and client_secret //
@@ -776,6 +810,63 @@ public class Auth0AuthenticationModule implements QAuthenticationModuleInterface
 
 
 
+   /*******************************************************************************
+    ** Look up access_token from session UUID
+    **
+    *******************************************************************************/
+   private String getAccessTokenFromSessionUUID(Auth0AuthenticationMetaData metaData, String sessionUUID) throws QAuthenticationException
+   {
+      String   accessToken   = null;
+      QSession beforeSession = QContext.getQSession();
+
+      try
+      {
+         QContext.setQSession(getChickenAndEggSession());
+
+         ///////////////////////////////////////
+         // query for the user session record //
+         ///////////////////////////////////////
+         QRecord userSessionRecord = new GetAction().executeForRecord(new GetInput(UserSession.TABLE_NAME)
+            .withUniqueKey(Map.of("uuid", sessionUUID))
+            .withShouldMaskPasswords(false)
+            .withShouldOmitHiddenFields(false));
+
+         if(userSessionRecord != null)
+         {
+            accessToken = userSessionRecord.getValueString("accessToken");
+
+            ////////////////////////////////////////////////////////////
+            // decode the accessToken and make sure it is not expired //
+            ////////////////////////////////////////////////////////////
+            if(accessToken != null)
+            {
+               DecodedJWT jwt = JWT.decode(accessToken);
+               if(jwt.getExpiresAtAsInstant().isBefore(Instant.now()))
+               {
+                  throw (new QAuthenticationException("accessToken is expired"));
+               }
+            }
+         }
+      }
+      catch(QAuthenticationException qae)
+      {
+         throw (qae);
+      }
+      catch(Exception e)
+      {
+         LOG.warn("Error looking up userSession by sessionUUID", e);
+         throw (new QAuthenticationException("Error looking up userSession by sessionUUID", e));
+      }
+      finally
+      {
+         QContext.setQSession(beforeSession);
+      }
+
+      return (accessToken);
+   }
+
+
+
    /*******************************************************************************
     ** Look up access_token from api key
     **
@@ -841,7 +932,7 @@ public class Auth0AuthenticationModule implements QAuthenticationModuleInterface
                // store the actual access token in the database, and return a unique value                    //
                // back to the user which will be what they use on subsequent requests (because token too big) //
                /////////////////////////////////////////////////////////////////////////////////////////////////
-               JSONObject accessTokenData = requestAccessTokenFromAuth0(metaData, clientId, clientSecret);
+               JSONObject accessTokenData = requestAccessTokenForClientIdAndSecretFromAuth0(metaData, clientId, clientSecret);
 
                Integer expiresInSeconds = accessTokenData.getInt("expires_in");
                accessToken = accessTokenData.getString("access_token");
@@ -872,25 +963,23 @@ public class Auth0AuthenticationModule implements QAuthenticationModuleInterface
 
 
    /*******************************************************************************
-    ** Look up client_auth0_application record, return if found.
     **
     *******************************************************************************/
-   QRecord getClientAuth0Application(Auth0AuthenticationMetaData metaData, String clientId) throws QException
+   static String maskForLog(String input)
    {
-      //////////////////////////////////////////////////////////////////////////////////////
-      // try to look up existing auth0 application from database, insert one if not found //
-      //////////////////////////////////////////////////////////////////////////////////////
-      QueryInput queryInput = new QueryInput();
-      queryInput.setTableName(metaData.getClientAuth0ApplicationTableName());
-      queryInput.setFilter(new QQueryFilter(new QFilterCriteria(metaData.getAuth0ClientIdField(), QCriteriaOperator.EQUALS, clientId)));
-      QueryOutput queryOutput = new QueryAction().execute(queryInput);
-
-      if(CollectionUtils.nullSafeHasContents(queryOutput.getRecords()))
+      if(input == null)
       {
-         return (queryOutput.getRecords().get(0));
+         return (null);
       }
 
-      throw (new AccessTokenException("This client has not been configured to use the API.", HttpStatus.SC_UNAUTHORIZED));
+      if(input.length() < 8)
+      {
+         return ("******");
+      }
+      else
+      {
+         return (input.substring(0, 6) + "******");
+      }
    }
 
 }