Merge branch 'rel/0.24.0'

Feature/ce 2260 add ability to send to fe and extensive

.circleci/adjust-pom-version.sh

@@ -1,23 +1,51 @@
 #!/bin/bash
 
-if [ -z "$CIRCLE_BRANCH" ] && [ -z "$CIRCLE_TAG" ]; then
-   echo "Error:  env vars CIRCLE_BRANCH and CIRCLE_TAG were not set."
-   exit 1;
-fi
+############################################################################
+## adjust-pom.version.sh
+## During CircleCI builds - edit the qqq parent pom.xml, to set the
+## <revision> value such that:
+## - feature-branch builds, tagged as snapshot-*, deploy with a version
+##   number that includes that tag's name (minus the snapshot- part)
+## - integration-branch builds deploy with a version number that includes
+##   the branch name slugified
+## - we never deploy -SNAPSHOT versions any more - because we don't believe
+##   it is ever valid to not know exactly what versions you are getting
+##   (perhaps because we are too loose with our versioning?)
+############################################################################
 
-if [ "$CIRCLE_BRANCH" == "dev" ] || [ "$CIRCLE_BRANCH" == "staging" ] || [ "$CIRCLE_BRANCH" == "main" ] || [ \! -z $(echo "$CIRCLE_TAG" | grep "^version-") ]; then
-   echo "On a primary branch or tag [${CIRCLE_BRANCH}${CIRCLE_TAG}] - will not edit the pom version.";
+POM=$(dirname $0)/../pom.xml
+echo "On branch: $CIRCLE_BRANCH, tag: $CIRCLE_TAG..."
+
+######################################################################
+## ## only do anything if the committed pom has a -SNAPSHOT version ##
+######################################################################
+REVISION=$(grep '<revision>' $POM | sed 's/.*<revision>//;s/<.*//');
+echo "<revision> in pom.xml is: $REVISION"
+if [ \! $(echo "$REVISION" | grep SNAPSHOT) ]; then
+   echo "Not on a SNAPSHOT revision, so nothing to do here."
    exit 0;
 fi
 
-if [ -n "$CIRCLE_BRANCH" ]; then
-   SLUG=$(echo $CIRCLE_BRANCH | sed 's/[^a-zA-Z0-9]/-/g')
-else
-   SLUG=$(echo $CIRCLE_TAG | sed 's/^snapshot-//g')
+##################################################################################
+## ## figure out if we need a SLUG:  a snapshot- tag, or an integration/ branch ##
+##################################################################################
+SLUG=""
+if [ $(echo "$CIRCLE_TAG" | grep ^snapshot-) ]; then
+   SLUG=$(echo "$CIRCLE_TAG" | sed "s/^snapshot-//")-
+   echo "Using slug [$SLUG] from tag [$CIRCLE_TAG]"
+
+elif [ $(echo "$CIRCLE_BRANCH" |  grep ^integration/) ]; then
+   SLUG=$(echo "$CIRCLE_BRANCH" | sed "s,/,-,g")-
+   echo "Using slug [$SLUG] from branch [$CIRCLE_BRANCH]"
 fi
 
-POM=$(dirname $0)/../pom.xml
+################################################################
+## ## build the replcaement for -SNAPSHOT, and update the pom ##
+################################################################
+TIMESTAMP=$(date +%Y%m%d-%H%M%S)
+REPLACEMENT=${SLUG}${TIMESTAMP}
 
-echo "Updating $POM <revision> to: $SLUG-SNAPSHOT"
-sed -i "s/<revision>.*/<revision>$SLUG-SNAPSHOT<\/revision>/" $POM
+echo "Updating $POM -SNAPSHOT to: -$REPLACEMENT"
+sed -i "s/-SNAPSHOT<\/revision>/-$REPLACEMENT<\/revision>/" $POM
 git diff $POM
+

.circleci/config.yml

@@ -2,6 +2,7 @@ version: 2.1
 
 orbs:
   localstack: localstack/platform@2.1
+  browser-tools: circleci/browser-tools@1.4.7
 
 commands:
   store_jacoco_site:
@@ -38,6 +39,8 @@ commands:
       - restore_cache:
           keys:
             - v1-dependencies-{{ checksum "pom.xml" }}
+      - browser-tools/install-chrome
+      - browser-tools/install-chromedriver
       - run:
           name: Write .env
           command: |
@@ -79,6 +82,19 @@ commands:
             - ~/.m2
           key: v1-dependencies-{{ checksum "pom.xml" }}
 
+  check_middleware_api_versions:
+      steps:
+        - checkout
+        - restore_cache:
+            keys:
+              - v1-dependencies-{{ checksum "pom.xml" }}
+        - run:
+            name: Build and Run ValidateApiVersions
+            command: |
+              mvn -s .circleci/mvn-settings.xml -T4 install -DskipTests
+              mvn -s .circleci/mvn-settings.xml -pl qqq-middleware-javalin package appassembler:assemble -DskipTests
+              qqq-middleware-javalin/target/appassembler/bin/ValidateApiVersions -r $(pwd)
+
   mvn_jar_deploy:
     steps:
       - checkout
@@ -114,14 +130,9 @@ commands:
           command: |
             cd docs
             asciidoctor -a docinfo=shared index.adoc
-
-  upload_docs_site:
-    steps:
-      - run:
-          name: scp html to justinsgotskinnylegs.com
-          command: |
-            cd docs
-            scp index.html dkelkhoff@45.79.44.221:/mnt/first-volume/dkelkhoff/nginx/html/justinsgotskinnylegs.com/qqq-docs.html
+      - store_artifacts:
+          path: docs/index.html
+          when: always
 
 jobs:
   mvn_test:
@@ -130,6 +141,7 @@ jobs:
       ## - localstack/startup
       - install_java17
       - mvn_verify
+      - check_middleware_api_versions
 
   mvn_deploy:
     executor: localstack/default
@@ -137,6 +149,7 @@ jobs:
       ## - localstack/startup
       - install_java17
       - mvn_verify
+      - check_middleware_api_versions
       - mvn_jar_deploy
 
   publish_asciidoc:
@@ -144,7 +157,6 @@ jobs:
     steps:
       - install_asciidoctor
       - run_asciidoctor
-      - upload_docs_site
 
 workflows:
   test_only:

docs/Introduction.adoc

@@ -136,17 +136,13 @@ This speaks to the fact that this "code" is not executable code - but rather is
 **** The Filter button in the Query Screen will present a menu listing all fields from the table for the user to build ad-hoc queries against the table.
 The data-types specified for the fields (in the meta-data) dictate what operators QQQ allows the user to use against fields (e.g., Strings offer "contains" vs Numbers offer "greater than").
 **** Values for records from the table will be formatted for presentation based on the meta-data (such as a numeric field being shown with commas if it represents a quantity, or formatted as currency).
-...
-
-[start=2]
-. *Meta Data* - declarative code - java object instances (potentially which could be read from `.yaml` files or other data sources in a future version of QQQ), which tell QQQ about the backend systems, tables, processes, reports, widgets, etc, that make up the application.
-For example:
-* Details about the database you are using, and how to connect to it.
-* A database table's name, fields, their types, its keys, and basic business rules (required fields, read-only fields, field lengths).
-* The description of web API - its URL and authentication mechanism.
-* A table/path within a web API, and the fields returned in the JSON at that endpoint.
-* The specification of a custom workflow (process), including what screens are needed, with input & output values, and references to the custom application code for processing the data.
-* Details about a chart that summarizes data from a table for presentation as a dashboard widget.
+* Other kinds of information that you tell QQQ about in the form of meta-data objects includes:
+** Details about the database you are using, and how to connect to it.
+** A database table's name, fields, their types, its keys, and basic business rules (required fields, read-only fields, field lengths).
+** The specification of a custom workflow (process), including what screens are needed, with input & output values, and references to the custom application code for processing the data.
+** Details about a chart that summarizes data from a table for presentation as a dashboard widget.
+** The description of web API - its URL and authentication mechanism.
+** A table/path within a web API, and the fields returned in the JSON at that endpoint.
 // the section below is kinda dumb.  like, it says you have to write application code, but
 // then it just talks about how your app code gets for-free the same shit that QQQ does.
 // it should instead say more about what your custom app code is or does.
@@ -164,7 +160,8 @@ For example:
 // * The multi-threaded, paged producer/consumer pattern used in standard framework actions is how all custom application actions are also invoked.
 // ** For example, the standard QQQ Bulk Edit action uses the same streamed-ETL process that custom application processes can use.
 // Meaning your custom processes can take full advantage of the same complex frontend, middleware, and backend structural pieces, and you can just focus on your unique busines logic needs.
-2. *Application code* - to customize beyond what the QQQ framework does out-of-the box, and to provide application-specific business-logic.
+
+. *Application code* - to customize beyond what the QQQ framework does out-of-the box, and to provide application-specific business-logic.
 QQQ provides its programmers the same classes that it internally uses for record access, resulting in a unified application model.
 For example:
 

docs/actions/QueryAction.adoc

@@ -155,9 +155,9 @@ new QFilterOrderBy()
 ----
 
 ==== QueryJoin
-* `joinTable` - *String, required* - Name of the table that is being joined in to the existing query.
+* `joinTable` - *String, required (though inferrable)* - Name of the table that is being joined in to the existing query.
 ** Will be inferred from *joinMetaData*, if *joinTable* is not set when *joinMetaData* gets set.
-* `baseTableOrAlias` - *String, required* - Name of a table (or an alias) already defined in the query, to which the *joinTable* will be joined.
+* `baseTableOrAlias` - *String, required (though inferrable)* - Name of a table (or an alias) already defined in the query, to which the *joinTable* will be joined.
 ** Will be inferred from *joinMetaData*, if *baseTableOrAlias* is not set when *joinMetaData* gets set (which will only use the leftTableName from the joinMetaData - never an alias).
 * `joinMetaData` - *QJoinMetaData object* - Optional specification of a {link-join} in the current QInstance.
 If not set, will be looked up at runtime based on *baseTableOrAlias* and *joinTable*.
@@ -165,21 +165,78 @@ If not set, will be looked up at runtime based on *baseTableOrAlias* and *joinTa
 * `alias` - *String* - Optional (unless multiple instances of the same table are being joined together, when it becomes required).
 Behavior based on SQL `FROM` clause aliases.
 If given, must be used as the part before the dot in field name specifications throughout the rest of the query input.
-* `select` - *boolean, default: false* - Specify whether fields from the *rightTable* should be selected by the query.
+* `select` - *boolean, default: false* - Specify whether fields from the *joinTable* should be selected by the query.
 If *true*, then the `QRecord` objects returned by this query will have values with corresponding to the (table-or-alias `.` field-name) form.
 * `type` - *Enum of INNER, LEFT, RIGHT, FULL, default: INNER* - specifies the SQL-style type of join being performed.
 
 [source,java]
-.QueryJoin definition examples:
+.Basic QueryJoin usage example:
 ----
-// selecting from an "orderLine" table - then join to its corresponding "order" table
+// selecting from an "orderLine" table, joined to its corresponding (parent) "order" table
 queryInput.withTableName("orderLine");
 queryInput.withQueryJoin(new QueryJoin("order").withSelect(true));
 ...
 queryOutput.getRecords().get(0).getValueBigDecimal("order.grandTotal");
+----
 
+[source,java]
+."V" shaped query - selecting from one parent table, and two children joined to it:
+----
+// TODO this needs verified for accuracy, though is a reasonable starting point as-is
+// selecting from an "order" table, and two children of it, orderLine and customer
+queryInput.withTableName("order");
+queryInput.withQueryJoin(new QueryJoin("orderLine").withSelect(true));
+queryInput.withQueryJoin(new QueryJoin("customer").withSelect(true));
+...
+QRecord joinedRecord = queryOutput.getRecords().get(0);
+joinedRecord.getValueString("orderNo");
+joinedRecord.getValueString("orderLine.sku");
+joinedRecord.getValueString("customer.firstName");
+----
+
+[source,java]
+."Chain" shaped query - selecting from one parent table, a child table, and a grandchild:
+----
+// TODO this needs verified for accuracy, though is a reasonable starting point as-is
+// selecting from an "order" table, with a "customer" child table, and an "address" sub-table
+queryInput.withTableName("order");
+queryInput.withQueryJoin(new QueryJoin("customer").withSelect(true));
+queryInput.withQueryJoin(new QueryJoin("address").withSelect(true));
+...
+QRecord joinedRecord = queryOutput.getRecords().get(0);
+joinedRecord.getValueString("orderNo");
+joinedRecord.getValueString("customer.firstName");
+joinedRecord.getValueString("address.street1");
+----
+
+[source,java]
+.QueryJoin usage example where two tables have two different joins between them:
+----
+// TODO this needs verified for accuracy, though is a reasonable starting point as-is
+// here there's a "fulfillmentPlan" table, which points at the order table (many-to-one,
+// as an order's plan can change over time, and we keep old plans around).
+// This join is named:  fulfillmentPlanJoinOrder
+//
+// The other join is "order" pointing at its current "fulfillmentPlan"
+// This join is named:  orderJoinCurrentFulfillmentPlan
+
+// to select an order along with its current fulfillment plan:
+queryInput.withTableName("order");
+queryInput.withQueryJoin(new QueryJoin(instance.getJoin("orderJoinCurrentFulfillmentPlan"))
+   .withSelect(true));
+
+// to select an order, and all fulfillment plans for an order (1 or more records):
+queryInput.withTableName("order");
+queryInput.withQueryJoin(new QueryJoin(instance.getJoin("fulfillmentPlanJoinOrder"))
+   .withSelect(true));
+----
+
+[source,java]
+.QueryJoin usage example for table with two joins to the same child table, selecting from both:
+----
 // given an "order" table with 2 foreign keys to a customer table (billToCustomerId and shipToCustomerId)
 // Note, we must supply the JoinMetaData to the QueryJoin, to drive what fields to join on in each case.
+// we must also define an alias for each of the QueryJoins
 queryInput.withTableName("order");
 queryInput.withQueryJoins(List.of(
    new QueryJoin(instance.getJoin("orderJoinShipToCustomer")
@@ -190,11 +247,18 @@ queryInput.withQueryJoins(List.of(
        .withSelect(true))));
 ...
 record.getValueString("billToCustomer.firstName")
-   + " placed an order for "
+   + " paid for an order, to be sent to "
    + record.getValueString("shipToCustomer.firstName")
 
 ----
 
+[source,java]
+.Implicit QueryJoin, where unambiguous and required by QQueryFilter
+----
+// TODO finish and verify
+queryInput.withTableName("order");
+----
+
 === QueryOutput
 * `records` - *List of QRecord* - List of 0 or more records that match the query filter.
 ** _Note:  If a *recordPipe* was supplied to the QueryInput, then calling `queryOutput.getRecords()` will result in an `IllegalStateException` being thrown - as the records were placed into the pipe as they were fetched, and cannot all be accessed as a single list._

docs/index.adoc

@@ -6,7 +6,10 @@
 
 include::Introduction.adoc[leveloffset=+1]
 
-== Meta Data
+== Meta Data Production
+include::metaData/MetaDataProduction.adoc[leveloffset=+1]
+
+== Meta Data Types
 // Organizational units
 include::metaData/QInstance.adoc[leveloffset=+1]
 include::metaData/Backends.adoc[leveloffset=+1]

docs/metaData/Fields.adoc

@@ -56,6 +56,37 @@ if the value in the field is longer than the `maxLength`, then one of the follow
 
 ----
 
+===== ValueRangeBehavior
+Used on Numeric fields.  Specifies min and/or max allowed values for the field.
+For each of min and max, the following attributes can be set:
+
+* `minValue` / `maxValue` - the number that is the limit.
+* `minAllowEqualTo` / `maxAllowEqualTo` - boolean (default true).  Controls if < (>) or ≤ (≥).
+* `minBehavior` / `maxBehavior` - enum of `ERROR` (default) or `CLIP`.
+** If `ERROR`, then a value not within the range causes an error, and the value does not get stored.
+** else if `CLIP`, then a value not within the range gets "clipped" to either be the min/max (if allowEqualTo),
+or to the min/max plus/minus the clipAmount
+* `minClipAmount` / `maxClipAmount` - Default 1.  Used when behavior is `CLIP` (only applies when
+not allowEqualTo).
+
+[source,java]
+.Examples of using ValueRangeBehavior
+----
+    new QFieldMetaData("noOfShoes", QFieldType.INTEGER)
+      .withBehavior(new ValueRangeBehavior().withMinValue(0));
+
+    new QFieldMetaData("price", QFieldType.BIG_DECIMAL)
+      .withBehavior(new ValueRangeBehavior()
+         // set the min value to be >= 0, and an error if an input is < 0.
+         .withMinValue(BigDecimal.ZERO)
+         .withMinAllowEqualTo(true)
+         .withMinBehavior(ERROR)
+         // set the max value to be < 100 - but effectively, clip larger values to 99.99
+         // here we use the .withMax() method that takes 4 params vs. calling 4 .withMax*() methods.
+         .withMax(new BigDecimal("100.00"), false, CLIP, new BigDecimal("0.01"))
+    );
+----
+
 ===== DynamicDefaultValueBehavior
 Used to set a dynamic default value to a field when it is being inserted or updated.
 For example, instead of having a hard-coded `defaultValue` specified in the field meta-data,

docs/metaData/MetaDataProduction.adoc

@@ -0,0 +1,413 @@
+[#MetaDataProduction]
+include::../variables.adoc[]
+
+The first thing that an application built using QQQ needs to do is to define its meta data.
+This basically means the construction of a `QInstance` object, which is populated with the
+meta data objects defining the backend(s), tables, processes, possible-value sources, joins,
+authentication provider, etc, that make your application.
+
+There are various styles that can be used for how you define your meta data, and for the
+most part they can be mixed and matched.  They will be presented here based on the historical
+evolution of how they were added to QQQ, where we generally believe that better techniques have
+been added over time.  So, you may wish to skip the earlier techniques, and jump straight to
+the end of this section.  However, it can always be instructive to learn about the past, so,
+read at your own pace.
+
+== Omni-Provider
+At the most basic level, the way to populate a `QInstance` is the simple and direct approach of creating
+one big class, possibly with just one big method, and just doing all the work directly inline there.
+
+This may (clearly) violate several good engineering principles.  However, it does have the benefit of
+being simple - if all of your meta-data is defined in one place, it can be pretty simple to find where
+that place is.  So - especially in a small project, this technique may be worth continuing to consider.
+
+Re: "doing all the work" as mentioned above - what work are we talking about?  At a minimum, we need
+to construct the following meta-data objects, and pass them into our `QInstance`:
+
+* `QAuthenticationMetaData` - how (or if!) users will be authenticated to the application.
+* `QBackendMeataData` - a backend data store.
+* `QTableMetaData` - a table (within the backend).
+* `QAppMetaData` - an organizational unit to present the other elements in a UI.
+
+Here's what a single-method omni-provider could look like:
+
+[source,java]
+.About the simplest possible single-file meta-data provider
+----
+public QInstance defineQInstance()
+{
+    QInstance qInstance = new QInstance();
+
+    qInstance.setAuthentication(new QAuthenticationMetaData()
+       .withName("anonymous")
+       .withType(QAuthenticationType.FULLY_ANONYMOUS));
+
+    qInstance.addBackend(new QBackendMetaData()
+       .withBackendType(MemoryBackendModule.class)
+       .withName("memoryBackend"));
+
+    qInstance.addTable(new QTableMetaData()
+        .withName("myTable")
+        .withPrimaryKeyField("id")
+        .withBackendName("memoryBackend")
+        .withField(new QFieldMetaData("id", QFieldType.INTEGER)));
+
+    qInstance.addApp(new QAppMetaData()
+       .withName("myApp")
+       .withSectionOfChildren(new QAppSection().withName("mySection"),
+          qInstance.getTable("myTable")))
+
+   return (qInstance);
+}
+----
+
+== Multi-method Omni-Provider
+
+The next evolution of meta-data production comes by just applying some basic better-engineering
+principles, and splitting up from a single method that constructs all the things, to at least
+using unique methods to construct each thing, then calling those methods to add their results
+to the QInstance.
+
+[source,java]
+.Multi-method omni- meta-data provider
+----
+public QInstance defineQInstance()
+{
+    QInstance qInstance = new QInstance();
+    qInstance.setAuthentication(defineAuthenticationMetaData());
+    qInstance.addBackend(defineBackendMetaData());
+    qInstance.addTable(defineMyTableMetaData());
+    qInstance.addApp(defineMyAppMetaData(qInstance));
+    return qInstance;
+}
+
+public QAuthenticationMetaData defineAuthenticationMetaData()
+{
+    return new QAuthenticationMetaData()
+       .withName("anonymous")
+       .withType(QAuthenticationType.FULLY_ANONYMOUS);
+}
+
+public QBackendMetaData defineBackendMetaData()
+{
+    return new QBackendMetaData()
+       .withBackendType(MemoryBackendModule.class)
+       .withName("memoryBackend");
+}
+
+// implementations of defineMyTableMetaData() and defineMyAppMetaData(qInstance)
+// left as an exercise for the reader
+----
+
+== Multi-class Providers
+
+Then the next logical evolution would be to put each of these single meta-data producing
+objects into its own class, along with calls to those classes.  This gets us away from the
+"5000 line" single-class, and lets us stop using the word "omni":
+
+[source,java]
+.Multi-class meta-data providers
+----
+public QInstance defineQInstance()
+{
+    QInstance qInstance = new QInstance();
+    qInstance.setAuthentication(new AuthMetaDataProvider().defineAuthenticationMetaData());
+    qInstance.addBackend(new BackendMetaDataProvider().defineBackendMetaData());
+    qInstance.addTable(new MyTableMetaDataProvider().defineTableMetaData());
+    qInstance.addApp(new MyAppMetaDataProvider().defineAppMetaData(qInstance));
+    return qInstance;
+}
+
+public class AuthMetaDataProvider
+{
+    public QAuthenticationMetaData defineAuthenticationMetaData()
+    {
+        return new QAuthenticationMetaData()
+           .withName("anonymous")
+           .withType(QAuthenticationType.FULLY_ANONYMOUS);
+    }
+}
+
+public class BackendMetaDataProvider
+{
+    public QBackendMetaData defineBackendMetaData()
+    {
+        return new QBackendMetaData()
+           .withBackendType(MemoryBackendModule.class)
+           .withName("memoryBackend");
+    }
+}
+
+// implementations of MyTableMetaDataProvider and MyAppMetaDataProvider
+// left as an exercise for the reader
+----
+
+== MetaDataProducerInterface
+
+As the size of your application grows, if you're doing per-object meta-data providers, you may find it
+burdensome, when adding a new object to your instance, to have to write code for it in two places -
+that is - a new class to produce that meta-data object, AND a single line of code to add that object
+to your `QInstance`.  As such, a mechanism exists to let you avoid that line-of-code for adding the object
+to the `QInstance`.
+
+This mechanism involves adding the `MetaDataProducerInterface` to all of your classes that produce a
+meta-data object.  This interface is generic, with a type parameter that will typically be the type of
+meta-data object you are producing, such as `QTableMetaData`, `QProcessMetaData`, or `QWidgetMetaData`,
+(technically, any class which implements `TopLevelMetaData`).  Implementers of the interface are then
+required to override just one method: `T produce(QInstance qInstance) throws QException;`
+
+Once you have your `MetaDataProducerInterface` classes defined, then there's a one-time call needed
+to add all of the objects produced by these classes to your `QInstance` - as shown here:
+
+[source,java]
+.Using MetaDataProducerInterface
+----
+public QInstance defineQInstance()
+{
+    QInstance qInstance = new QInstance();
+    MetaDataProducerHelper.processAllMetaDataProducersInPackage(qInstance,
+      "com.mydomain.myapplication");
+    return qInstance;
+}
+
+public class AuthMetaDataProducer implements MetaDataProducerInterface<QAuthenticationMetaData>
+{
+    @Override
+    public QAuthenticationMetaData produce(QInstance qInstance)
+    {
+        return new QAuthenticationMetaData()
+           .withName("anonymous")
+           .withType(QAuthenticationType.FULLY_ANONYMOUS);
+    }
+}
+
+public class BackendMetaDataProducer implements MetaDataProducerInterface<QBackendMetaData>
+{
+    @Override
+    public QBackendMetaData defineBackendMetaData()
+    {
+        return new QBackendMetaData()
+           .withBackendType(MemoryBackendModule.class)
+           .withName("memoryBackend");
+    }
+}
+
+// implementations of MyTableMetaDataProvider and MyAppMetaDataProvider
+// left as an exercise for the reader
+----
+
+=== MetaDataProducerMultiOutput
+It is worth mentioning, that sometimes it might feel like a bridge that's a bit too far, to make
+every single one of your meta-data objects require its own class.  Some may argue that it's best
+to do it that way - single responsibility principle, etc.  But, if you're producing, say, 5 widgets
+that are all related, and it's only a handful of lines of code for each one, maybe you'd rather
+produce them all in the same class.  Or maybe when you define a table, you'd like to define its
+joins and widgets at the same time.
+
+This approach can be accomplished by making the type argument for your `MetaDataProducerInterface` be
+`MetaDataProducerMultiOutput` - a simple class that just wraps a list of other `MetaDataProducerOutput`
+objects.
+
+[source,java]
+.Returning a MetaDataProducerMultiOutput
+----
+public class MyMultiProducer implements MetaDataProducerInterface<MetaDataProducerMultiOutput>
+{
+    @Override
+    public MetaDataProducerMultiOutput produce(QInstance qInstance)
+    {
+       MetaDataProducerMultiOutput output = new MetaDataProducerMultiOutput();
+
+       output.add(new QPossibleValueSource()...);
+       output.add(new QJoinMetaData()...);
+       output.add(new QJoinMetaData()...);
+       output.add(new QWidgetMetaData()...);
+       output.add(new QTableMetaData()...);
+
+       return (output);
+    }
+}
+----
+
+
+== Aside: TableMetaData with RecordEntities
+At this point, let's take a brief aside to dig deeper into the creation of a `QTableMeta` object.
+Tables, being probably the most important meta-data type in QQQ, have a lot of information that can
+be specified in their meta-data object.
+
+At the same time, if you're writing any custom code in your QQQ application
+(e.g., any processes or table customizers), where you're working with records from tables, you may
+prefer being able to work with entity beans (e.g., java classes with typed getter & setter methods),
+rather than the default object type that QQQ's ORM actions return, the `QRecord`, which carries all
+of its values in a `Map` (where you don't get compile-time checks of field names or data types).
+QQQ has a mechanism for dealing with this - in the form of the `QRecordEntity` class.
+
+So - if you want to build your application using entity beans (which is recommended, for the compile-time
+safety that they provide in custom code), you will be writing a `QRecordEntity` class for each of your tables,
+which will look like:
+
+[source,java]
+.QRecordEntity example
+----
+public class MyTable extends QRecordEntity
+{
+   public static final String TABLE_NAME = "myTable";
+
+   @QField(isEditable = false, isPrimaryKey = true)
+   private Integer id;
+
+   @QField()
+   private String name;
+
+   // no-arg constructor and constructor that takes a QRecord
+   // getters & setters (and optional fluent setters)
+}
+----
+
+The point of introducing this topic here and now is, that a `QRecordEntity` can be used to shortcut to
+defining some of the attributes in a `QTableMetaData` object.  Specifically, in a `MetaDataProducer<QTableMetaData>`
+you may say:
+
+[source,java]
+.QTableMetaDataProducer using a QRecordEntity
+----
+public QTableMetaData produce(QInstance qInstance) throws QExcpetion
+{
+  return new QTableMetaData()
+     .withName(MyTable.TABLE_NAME)
+     .withFieldsFromEntity(MyTable.class)
+     .withBackendName("memoryBackend");
+}
+----
+
+That `withFieldsFromEntity` call is one of the biggest benefits of this technique.  It allows you to avoid defining
+all of the fields in you table in two places (the entity and the table meta-data).
+
+== MetaData Producing Annotations for Entities
+
+If you are using `QRecordEntity` classes that correspond to your tables, then you can take advantage of some
+additional annotations on those classes, to produce more related meta-data objects associated with those tables.
+The point of this is to eliminate boilerplate, and simplify / speed up the process of getting a new table
+built and deployed in your application.
+
+Furthermore, the case can be made that it is beneficial to keep the meta-data definition for a table as close
+as possible to the entity that corresponds to the table.  This enables modifications to the table (e.g., adding
+a new field/column) to only require edits in one java source file, rather than necessarily requiring edits
+in two files.
+
+=== @QMetaDataProducingEntity
+This is an annotation meant to be placed on a `QRecordEntity` subclass, which you would like to be
+processed by an invocation of `MetaDataProducerHelper`, to automatically produce some meta-data
+objects.
+
+This annotation supports:
+
+* Creating table meta-data for the corresponding record entity table. Enabled by setting `produceTableMetaData=true`.
+** One may customize the table meta data that is produced automatically by supplying a class that extends
+`MetaDataCustomizerInterface` in the annotation attribute `tableMetaDataCustomizer`.
+** In addition to (or as an alternative to) the per-table `MetaDataCustomizerInterface` that can be specified
+in `@QMetaDataProducingEntity.tableMetaDataCustomzier`, when an application calls
+`MetaDataProducerHelper.processAllMetaDataProducersInPackage`, an additional `MetaDataCustomizerInterface` can be
+given, to apply a common set of adjustments to all tales being generated by the call.
+* Making a possible-value-source out of the table.  Enabled by setting `producePossibleValueSource=true`.
+* Processing child tables to create joins and childRecordList widgets
+
+=== @ChildTable
+This is an annotation used as a value that goes inside a `@QMetadataProducingEntity` annotation, to define
+child-tables, e.g., for producing joins and childRecordList widgets related to the table defined in the entity class.
+
+==== @ChildJoin
+This is an annotation used as a value inside a `@ChildTable` inside a `@QMetadataProducingEntity` annotation,
+to control the generation of a `QJoinMetaData`, as a `ONE_TO_MANY` type join from the table represented by
+the annotated entity, to the table referenced in the `@ChildTable` annotation.
+
+==== @ChildRecordListWidget
+This is an annotation used as a value that goes inside a `@QMetadataProducingEntity` annotation, to control
+the generation of a QWidgetMetaData - for a ChildRecordList widget.
+
+[source,java]
+.QRecordEntity with meta-data producing annotations and a table MetaDataCustomizer
+----
+@QMetaDataProducingEntity(
+   produceTableMetaData = true,
+   tableMetaDataCustomizer = MyTable.TableMetaDataCustomizer.class,
+   producePossibleValueSource = true,
+   childTables = {
+      @ChildTable(
+         childTableEntityClass = MyChildTable.class,
+         childJoin = @ChildJoin(enabled = true),
+         childRecordListWidget = @ChildRecordListWidget(enabled = true, label = "Children"))
+   }
+)
+public class MyTable extends QRecordEntity
+{
+   public static final String TABLE_NAME = "myTable";
+
+   public static class TableMetaDataCustomizer implements MetaDataCustomizerInterface<QTableMetaData>
+   {
+      @Override
+      public QTableMetaData customizeMetaData(QInstance qInstance, QTableMetaData table) throws QException
+      {
+         String childJoinName = QJoinMetaData.makeInferredJoinName(TABLE_NAME, MyChildTable.TABLE_NAME);
+
+         table
+            .withUniqueKey(new UniqueKey("name"))
+            .withIcon(new QIcon().withName("table_bar"))
+            .withRecordLabelFormat("%s")
+            .withRecordLabelFields("name")
+
+            .withSection(new QFieldSection("identity", new QIcon().withName("badge"), Tier.T1,
+               List.of("id", "name")))
+            // todo additional sections for other fields
+            .withSection(new QFieldSection("children", new QIcon().withName("account_tree"), Tier.T2)
+               .withWidgetName(childJoinName))
+
+            .withExposedJoin(new ExposedJoin()
+               .withLabel("Children")
+               .withJoinPath(List.of(childJoinName))
+               .withJoinTable(MyChildTable.TABLE_NAME));
+
+         return (table);
+      }
+   }
+
+   @QField(isEditable = false, isPrimaryKey = true)
+   private Integer id;
+
+   // remaining fields, constructors, getters & setters left as an exercise for the reader and/or the IDE
+}
+----
+
+The class given in the example above, if processed by the `MetaDataProducerHelper`, would add the following
+meta-data objects to your `QInstance`:
+
+* A `QTableMetaData` named `myTable`, with all fields annotated as `@QField` from the `QRecordEntity` class,
+and with additional attributes as set in the `TableMetaDataCustomizer` inner class.
+* A `QPossibleValueSource` named `myTable`, of type `TABLE`, with `myTable` as its backing table.
+* A `QJoinMetaData` named `myTableJoinMyChildTable`, as a `ONE_TO_MANY` type, between those two tables.
+* A `QWidgetMetaData` named `myTableJoinMyChildTable`, as a `CHILD_RECORD_LIST` type, that will show a list of
+records from `myChildTable` as a widget, when viewing a record from `myTable`.
+
+== Other MetaData Producing Annotations
+
+Similar to these annotations for a `RecordEntity`, a similar one exists for a `PossibleValueEnum` class,
+to automatically write the meta-data to use that enum as a possible value source in your application:
+
+=== @QMetaDataProducingPossibleValueEnum
+This is an annotation to go on a `PossibleValueEnum` class, which you would like to be
+processed by MetaDataProducerHelper, to automatically produce a PossibleValueSource meta-data
+based on the enum.
+
+[source,java]
+.PossibleValueEnum with meta-data producing annotation
+----
+@QMetaDataProducingPossibleValueEnum(producePossibleValueSource = true)
+public enum MyOptionsEnum implements PossibleValueEnum<Integer>
+{
+   // values and methods left as exercise for reader
+}
+----
+The enum given in the example above, if processed by the `MetaDataProducerHelper`, would add the following
+meta-data object to your `QInstance`:
+
+* A `QPossibleValueSource` named `MyOptionsEnum`, of type `ENUM`, with `MyOptionsEnum` as its backing enum.

docs/metaData/Processes.adoc

@@ -38,6 +38,13 @@ See {link-permissionRules} for details.
 *** 1) by a single call to `.withStepList(List<QStepMetaData>)`, which internally adds each step into the `steps` map.
 *** 2) by multiple calls to `.addStep(QStepMetaData)`, which adds a step to both the `stepList` and `steps` map.
 ** If a process also needs optional steps (for a <<_custom_process_flow>>), they should be added by a call to `.addOptionalStep(QStepMetaData)`, which only places them in the `steps` map.
+* `stepFlow` - *enum, default LINEAR* - specifies the the flow-control logic between steps.  Possible values are:
+** `LINEAR` - steps are executed in-order, through the `stepList`.
+A backend step _can_ customize the `nextStepName` or re-order the `stepList`, if needed.
+In a frontend step, a user may be given the option to go _back_ to a previous step as well.
+** `STATE_MACHINE` - steps are executed as a Fine State Machine, starting with the first step in `stepList`,
+but then proceeding based on the `nextStepName` specified by the previous step.
+Thus allowing much more flexible flows.
 * `schedule` - *<<QScheduleMetaData>>* - set up the process to run automatically on the specified schedule.
 See below for details.
 * `minInputRecords` - *Integer* - #not used...#
@@ -67,6 +74,11 @@ For processes with a user-interface, they must define one or more "screens" in t
 * `formFields` - *List of String* - list of field names used by the screen as form-inputs.
 * `viewFields` - *List of String* - list of field names used by the screen as visible outputs.
 * `recordListFields` - *List of String* - list of field names used by the screen in a record listing.
+* `format` - *Optional String* - directive for a frontend to use specialized formatting for the display of the process.
+** Consult frontend documentation for supported values and their meanings.
+* `backStepName` - *Optional String* - For processes using `LINEAR` flow, if this value is given,
+then the frontend should offer a control that the user can take (e.g., a button) to move back to an
+earlier step in the process.
 
 ==== QFrontendComponentMetaData
 
@@ -90,10 +102,13 @@ Expects a process value named `html`.
 Expects process values named `downloadFileName` and `serverFilePath`.
 ** `GOOGLE_DRIVE_SELECT_FOLDER` - Special form that presents a UI from Google Drive, where the user can select a folder (e.g., as a target for uploading files in a subsequent backend step).
 ** `BULK_EDIT_FORM` - For use by the standard QQQ Bulk Edit process.
+** `BULK_LOAD_FILE_MAPPING_FORM`, `BULK_LOAD_VALUE_MAPPING_FORM`, or `BULK_LOAD_PROFILE_FORM` - For use by the standard QQQ Bulk Load process.
 ** `VALIDATION_REVIEW_SCREEN` - For use by the QQQ Streamed ETL With Frontend process family of processes.
 Displays a component prompting the user to run full validation or to skip it, or, if full validation has been ran, then showing the results of that validation.
 ** `PROCESS_SUMMARY_RESULTS` - For use by the QQQ Streamed ETL With Frontend process family of processes.
 Displays the summary results of running the process.
+** `WIDGET` - Render a QQQ Widget.
+Requires that `widgetName` be given as a value for the component.
 ** `RECORD_LIST` - _Deprecated.
 Showed a grid with a list of records as populated by the process._
 * `values` - *Map of String → Serializable* - Key=value pairs, with different expectations based on the component's `type`.
@@ -116,6 +131,27 @@ It can be used, however, for example, to cause a `defaultValue` to be applied to
 It can also be used to cause the process to throw an error, if a field is marked as `isRequired`, but a value is not present.
 ** `recordListMetaData` - *RecordListMetaData object* - _Not used at this time._
 
+==== QStateMachineStep
+
+Processes that use `flow = STATE_MACHINE` should use process steps of type `QStateMachineStep`.
+
+A common pattern seen in state-machine processes, is that they will present a frontend-step to a user,
+then always run a given backend-step in response to that screen which the user submitted.
+Inside that backend-step, custom application logic will determine the next state to go to,
+which is typically another frontend-step (which would then submit data to its corresponding backend-step,
+and continue the FSM).
+
+To help facilitate this pattern, factory methods exist on `QStateMachineStep`,
+for constructing the commonly-expected types of state-machine steps:
+
+* `frontendThenBackend(name, frontendStep, backendStep)` - for the frontend-then-backend pattern described above.
+* `backendOnly(name, backendStep)` - for a state that only has a backend step.
+This might be useful as a “reset” step, to run before restarting a state-loop.
+* `frontendOnly(name, frontendStep)` - for a state that only has a frontend step,
+which would always be followed by another state, which must be specified as the `defaultNextStepName`
+on the `QStateMachineStep`.
+
+
 ==== BasepullConfiguration
 
 A "Basepull" process is a common pattern where an application needs to perform some action on all new (or updated) records from a particular data source.
@@ -218,12 +254,10 @@ But for some cases, doing page-level transactions can reduce long-transactions a
 * `withSchedule(QScheduleMetaData schedule)` - Add a <<QScheduleMetaData>> to the process.
 
 [#_custom_process_flow]
-==== Custom Process Flow
-As referenced in the definition of the <<_QProcessMetaData_Properties,QProcessMetaData Properties>>, by default, a process
-will execute each of its steps in-order, as defined in the `stepList` property.
-However, a Backend Step can customize this flow #todo - write more clearly here...
-
-There are generally 2 method to call (in a `BackendStep`) to do a dynamic flow:
+==== How to customize a Linear process flow
+As referenced in the definition of the <<_QProcessMetaData_Properties,QProcessMetaData Properties>>, by default,
+(with `flow = LINEAR`) a process will execute each of its steps in-order, as defined in the `stepList` property.
+However, a Backend Step can customize this flow as follows:
 
 * `RunBackendStepOutput.setOverrideLastStepName(String stepName)`
 ** QQQ's `RunProcessAction` keeps track of which step it "last" ran, e.g., to tell it which one to run next.
@@ -239,7 +273,7 @@ does need to be found in the new `stepNameList` - otherwise, the framework will
 for figuring out where to go next.
 
 [source,java]
-.Example of a defining process that can use a flexible flow:
+.Example of a defining process that can use a customized linear flow:
 ----
 // for a case like this, it would be recommended to define all step names in constants:
 public final static String STEP_START = "start";
@@ -324,4 +358,21 @@ public static class StartStep implements BackendStep
 }
 ----
 
+[#_process_back]
+==== How to allow a process to go back
+
+The simplest option to allow a process to present a "Back" button to users,
+thus allowing them to move backward through a process
+(e.g., from a review screen back to an earlier input screen), is to set the property `backStepName`
+on a `QFrontendStepMetaData`.
+
+If the step that is executed after the user hits "Back" is a backend step, then within that
+step, `runBackendStepInput.getIsStepBack()` will return `true` (but ONLY within that first step after
+the user hits "Back").  It may be necessary within individual processes to be aware that the user
+has chosen to go back, to reset certain values in the process's state.
+
+Alternatively, if a frontend step's "Back" behavior needs to be dynamic (e.g., sometimes not available,
+or sometimes targeting different steps in the process), then in a backend step that runs before the
+frontend step, a call to `runBackendStepOutput.getProcessState().setBackStepName()` can be made,
+to customize the value which would otherwise come from the `QFrontendStepMetaData`.
 

docs/metaData/QInstance.adoc

@@ -29,11 +29,21 @@ service.routes(qJavalinImplementation.getRoutes());
 service.start();
 ----
 
-*QBackendMetaData Setup Methods:*
+*QInstance Setup:*
 
 These are the methods that one is most likely to use when setting up (defining) a `QInstance` object:
 
-* asdf
+* `add(TopLevelMetaDataInterface metaData)` - Generic method that takes most of the meta-data subtypes that can be added
+to an instance, such as `QBackendMetaData`, `QTableMetaData`, `QProcessMetaData`, etc.
+There are also type-specific methods (e.g., `addTable`, `addProcess`, etc), which one can call instead - this would just
+be a matter of personal preference.
 
-*QBackendMetaData Usage Methods:*
+*QInstance Usage:*
 
+Generally you will set up a `QInstance` in your application's startup flow, and then place it in the server (e.g., javalin).
+But, if, during application-code runtime, you need access to any of the meta-data in the instance, you access it
+via the `QContext` object's static `getInstance()` method.  This can be useful, for example, to get a list of the defined
+tables in the application, or fields in a table, or details about a field, etc.
+
+It is generally considered risky and/or not a good idea at all to modify the `QInstance` after it has been validated and
+a server is running.  Future versions of QQQ may in fact restrict modifications to the instance after validation.

pom.xml

@@ -34,8 +34,10 @@
       <module>qqq-backend-module-api</module>
       <module>qqq-backend-module-filesystem</module>
       <module>qqq-backend-module-rdbms</module>
+      <module>qqq-backend-module-sqlite</module>
       <module>qqq-backend-module-mongodb</module>
       <module>qqq-language-support-javascript</module>
+      <module>qqq-openapi</module>
       <module>qqq-middleware-picocli</module>
       <module>qqq-middleware-javalin</module>
       <module>qqq-middleware-lambda</module>
@@ -46,7 +48,7 @@
    </modules>
 
    <properties>
-      <revision>0.23.0-SNAPSHOT</revision>
+      <revision>0.24.0</revision>
 
       <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
       <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>

qqq-backend-core/pom.xml

@@ -100,7 +100,12 @@
       <dependency>
          <groupId>org.dhatim</groupId>
          <artifactId>fastexcel</artifactId>
-         <version>0.12.15</version>
+         <version>0.18.4</version>
+      </dependency>
+      <dependency>
+         <groupId>org.dhatim</groupId>
+         <artifactId>fastexcel-reader</artifactId>
+         <version>0.18.4</version>
       </dependency>
       <dependency>
          <groupId>org.apache.poi</groupId>
@@ -112,6 +117,14 @@
          <artifactId>poi-ooxml</artifactId>
          <version>5.2.5</version>
       </dependency>
+
+      <!-- adding to help FastExcel -->
+      <dependency>
+         <groupId>commons-io</groupId>
+         <artifactId>commons-io</artifactId>
+         <version>2.16.0</version>
+      </dependency>
+
       <dependency>
          <groupId>com.auth0</groupId>
          <artifactId>auth0</artifactId>

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/async/AsyncRecordPipeLoop.java

@@ -186,7 +186,7 @@ public class AsyncRecordPipeLoop
 
       if(recordCount > 0)
       {
-         LOG.info("End of job summary", logPair("recordCount", recordCount), logPair("jobName", jobName), logPair("millis", endTime - jobStartTime), logPair("recordsPerSecond", 1000d * (recordCount / (.001d + (endTime - jobStartTime)))));
+         LOG.debug("End of job summary", logPair("recordCount", recordCount), logPair("jobName", jobName), logPair("millis", endTime - jobStartTime), logPair("recordsPerSecond", 1000d * (recordCount / (.001d + (endTime - jobStartTime)))));
       }
 
       return (recordCount);

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/audits/AuditAction.java

@@ -44,6 +44,7 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetOutput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertOutput;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.metadata.security.MultiRecordSecurityLock;
 import com.kingsrook.qqq.backend.core.model.metadata.security.RecordSecurityLock;
 import com.kingsrook.qqq.backend.core.model.metadata.security.RecordSecurityLockFilters;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
@@ -173,26 +174,53 @@ public class AuditAction extends AbstractQActionFunction<AuditInput, AuditOutput
       Map<String, Serializable> securityKeyValues = new HashMap<>();
       for(RecordSecurityLock recordSecurityLock : RecordSecurityLockFilters.filterForReadLocks(CollectionUtils.nonNullList(table.getRecordSecurityLocks())))
       {
-         Serializable keyValue = record == null ? null : record.getValue(recordSecurityLock.getFieldName());
-
-         if(keyValue == null && oldRecord.isPresent())
-         {
-            LOG.debug("Table with a securityLock, but value not found in field", logPair("table", table.getName()), logPair("field", recordSecurityLock.getFieldName()));
-            keyValue = oldRecord.get().getValue(recordSecurityLock.getFieldName());
-         }
-
-         if(keyValue == null)
-         {
-            LOG.debug("Table with a securityLock, but value not found in field", logPair("table", table.getName()), logPair("field", recordSecurityLock.getFieldName()), logPair("oldRecordIsPresent", oldRecord.isPresent()));
-         }
-
-         securityKeyValues.put(recordSecurityLock.getSecurityKeyType(), keyValue);
+         getRecordSecurityKeyValues(table, record, oldRecord, recordSecurityLock, securityKeyValues);
       }
       return securityKeyValues;
    }
 
 
 
+   /***************************************************************************
+    ** recursive implementation of getRecordSecurityKeyValues, for dealing with
+    ** multi-locks
+    ***************************************************************************/
+   private static void getRecordSecurityKeyValues(QTableMetaData table, QRecord record, Optional<QRecord> oldRecord, RecordSecurityLock recordSecurityLock, Map<String, Serializable> securityKeyValues)
+   {
+      //////////////////////////////////////////////////////
+      // special case with recursive call for multi-locks //
+      //////////////////////////////////////////////////////
+      if(recordSecurityLock instanceof MultiRecordSecurityLock multiRecordSecurityLock)
+      {
+         for(RecordSecurityLock subLock : RecordSecurityLockFilters.filterForReadLocks(CollectionUtils.nonNullList(multiRecordSecurityLock.getLocks())))
+         {
+            getRecordSecurityKeyValues(table, record, oldRecord, subLock, securityKeyValues);
+         }
+
+         return;
+      }
+
+      ///////////////////////////////////////////
+      // by default, deal with non-multi locks //
+      ///////////////////////////////////////////
+      Serializable keyValue = record == null ? null : record.getValue(recordSecurityLock.getFieldName());
+
+      if(keyValue == null && oldRecord.isPresent())
+      {
+         LOG.debug("Table with a securityLock, but value not found in field", logPair("table", table.getName()), logPair("field", recordSecurityLock.getFieldName()));
+         keyValue = oldRecord.get().getValue(recordSecurityLock.getFieldName());
+      }
+
+      if(keyValue == null)
+      {
+         LOG.debug("Table with a securityLock, but value not found in field", logPair("table", table.getName()), logPair("field", recordSecurityLock.getFieldName()), logPair("oldRecordIsPresent", oldRecord.isPresent()));
+      }
+
+      securityKeyValues.put(recordSecurityLock.getSecurityKeyType(), keyValue);
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/
@@ -218,21 +246,16 @@ public class AuditAction extends AbstractQActionFunction<AuditInput, AuditOutput
                   throw (new QException("Requested audit for an unrecognized table name: " + auditSingleInput.getAuditTableName()));
                }
 
-               ///////////////////////////////////////////////////
-               // validate security keys on the table are given //
-               ///////////////////////////////////////////////////
-               for(RecordSecurityLock recordSecurityLock : RecordSecurityLockFilters.filterForReadLocks(CollectionUtils.nonNullList(table.getRecordSecurityLocks())))
+               ///////////////////////////////////////////////////////
+               // validate security keys on the table are given     //
+               // originally, this case threw...                    //
+               // but i think it's better to record the audit, just //
+               // missing its security key value, then to fail...   //
+               // but, maybe should be configurable, etc...         //
+               ///////////////////////////////////////////////////////
+               if(!validateSecurityKeys(auditSingleInput, table))
                {
-                  if(auditSingleInput.getSecurityKeyValues() == null || !auditSingleInput.getSecurityKeyValues().containsKey(recordSecurityLock.getSecurityKeyType()))
-                  {
-                     ///////////////////////////////////////////////////////
-                     // originally, this case threw...                    //
-                     // but i think it's better to record the audit, just //
-                     // missing its security key value, then to fail...   //
-                     ///////////////////////////////////////////////////////
-                     // throw (new QException("Missing securityKeyValue [" + recordSecurityLock.getSecurityKeyType() + "] in audit request for table " + auditSingleInput.getAuditTableName()));
-                     LOG.info("Missing securityKeyValue in audit request", logPair("table", auditSingleInput.getAuditTableName()), logPair("securityKey", recordSecurityLock.getSecurityKeyType()));
-                  }
+                  LOG.debug("Missing securityKeyValue in audit request", logPair("table", auditSingleInput.getAuditTableName()), logPair("auditMessage", auditSingleInput.getMessage()), logPair("recordId", auditSingleInput.getRecordId()));
                }
 
                ////////////////////////////////////////////////
@@ -310,6 +333,70 @@ public class AuditAction extends AbstractQActionFunction<AuditInput, AuditOutput
 
 
 
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   static boolean validateSecurityKeys(AuditSingleInput auditSingleInput, QTableMetaData table)
+   {
+      boolean allAreValid = true;
+      for(RecordSecurityLock recordSecurityLock : RecordSecurityLockFilters.filterForReadLocks(CollectionUtils.nonNullList(table.getRecordSecurityLocks())))
+      {
+         boolean lockIsValid = validateSecurityKeysForLock(auditSingleInput, recordSecurityLock);
+         if(!lockIsValid)
+         {
+            allAreValid = false;
+         }
+      }
+
+      return (allAreValid);
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   private static boolean validateSecurityKeysForLock(AuditSingleInput auditSingleInput, RecordSecurityLock recordSecurityLock)
+   {
+      if(recordSecurityLock instanceof MultiRecordSecurityLock multiRecordSecurityLock)
+      {
+         boolean allSubLocksAreValid = true;
+         boolean anySubLocksAreValid = false;
+         for(RecordSecurityLock lock : RecordSecurityLockFilters.filterForReadLocks(CollectionUtils.nonNullList(multiRecordSecurityLock.getLocks())))
+         {
+            boolean subLockIsValid = validateSecurityKeysForLock(auditSingleInput, lock);
+            if(subLockIsValid)
+            {
+               anySubLocksAreValid = true;
+            }
+            else
+            {
+               allSubLocksAreValid = false;
+            }
+         }
+
+         if(multiRecordSecurityLock.getOperator().equals(MultiRecordSecurityLock.BooleanOperator.OR))
+         {
+            return (anySubLocksAreValid);
+         }
+         else if(multiRecordSecurityLock.getOperator().equals(MultiRecordSecurityLock.BooleanOperator.AND))
+         {
+            return (allSubLocksAreValid);
+         }
+      }
+      else
+      {
+         if(auditSingleInput.getSecurityKeyValues() == null || !auditSingleInput.getSecurityKeyValues().containsKey(recordSecurityLock.getSecurityKeyType()))
+         {
+            return (false);
+         }
+      }
+
+      return (true);
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/RecordCustomizerUtilityInterface.java

@@ -27,6 +27,7 @@ import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
+import java.util.Optional;
 import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.logging.QLogger;
 import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
@@ -160,4 +161,18 @@ public interface RecordCustomizerUtilityInterface
       return (oldRecordMap);
    }
 
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   static <T extends Serializable> T getValueFromRecordOrOldRecord(String fieldName, QRecord record, Serializable primaryKey, Optional<Map<Serializable, QRecord>> oldRecordMap)
+   {
+      T value = (T) record.getValue(fieldName);
+      if(value == null && primaryKey != null && oldRecordMap.isPresent() && oldRecordMap.get().containsKey(primaryKey))
+      {
+         value = (T) oldRecordMap.get().get(primaryKey).getValue(fieldName);
+      }
+      return value;
+   }
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/TableCustomizerInterface.java

@@ -22,15 +22,19 @@
 package com.kingsrook.qqq.backend.core.actions.customizers;
 
 
+import java.io.Serializable;
 import java.util.List;
+import java.util.Map;
 import java.util.Optional;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.logging.QLogger;
+import com.kingsrook.qqq.backend.core.model.actions.AbstractActionInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.QueryOrGetInputInterface;
 import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
 import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
 
 
@@ -47,7 +51,6 @@ public interface TableCustomizerInterface
 {
    QLogger LOG = QLogger.getLogger(TableCustomizerInterface.class);
 
-
    /*******************************************************************************
     ** custom actions to run after a query (or get!) takes place.
     **
@@ -77,8 +80,15 @@ public interface TableCustomizerInterface
     *******************************************************************************/
    default List<QRecord> preInsert(InsertInput insertInput, List<QRecord> records, boolean isPreview) throws QException
    {
-      LOG.info("A default implementation of preInsert is running...  Probably not expected!", logPair("tableName", insertInput.getTableName()));
-      return (records);
+      try
+      {
+         return (preInsertOrUpdate(insertInput, records, isPreview, Optional.empty()));
+      }
+      catch(NotImplementedHereException e)
+      {
+         LOG.info("A default implementation of preInsert is running...  Probably not expected!", logPair("tableName", insertInput.getTableName()));
+         return (records);
+      }
    }
 
 
@@ -104,8 +114,15 @@ public interface TableCustomizerInterface
     *******************************************************************************/
    default List<QRecord> postInsert(InsertInput insertInput, List<QRecord> records) throws QException
    {
-      LOG.info("A default implementation of postInsert is running...  Probably not expected!", logPair("tableName", insertInput.getTableName()));
-      return (records);
+      try
+      {
+         return (postInsertOrUpdate(insertInput, records, Optional.empty()));
+      }
+      catch(NotImplementedHereException e)
+      {
+         LOG.info("A default implementation of postInsert is running...  Probably not expected!", logPair("tableName", insertInput.getTableName()));
+         return (records);
+      }
    }
 
 
@@ -130,8 +147,15 @@ public interface TableCustomizerInterface
     *******************************************************************************/
    default List<QRecord> preUpdate(UpdateInput updateInput, List<QRecord> records, boolean isPreview, Optional<List<QRecord>> oldRecordList) throws QException
    {
-      LOG.info("A default implementation of preUpdate is running...  Probably not expected!", logPair("tableName", updateInput.getTableName()));
-      return (records);
+      try
+      {
+         return (preInsertOrUpdate(updateInput, records, isPreview, oldRecordList));
+      }
+      catch(NotImplementedHereException e)
+      {
+         LOG.info("A default implementation of preUpdate is running...  Probably not expected!", logPair("tableName", updateInput.getTableName()));
+         return (records);
+      }
    }
 
 
@@ -151,8 +175,15 @@ public interface TableCustomizerInterface
     *******************************************************************************/
    default List<QRecord> postUpdate(UpdateInput updateInput, List<QRecord> records, Optional<List<QRecord>> oldRecordList) throws QException
    {
-      LOG.info("A default implementation of postUpdate is running...  Probably not expected!", logPair("tableName", updateInput.getTableName()));
-      return (records);
+      try
+      {
+         return (postInsertOrUpdate(updateInput, records, oldRecordList));
+      }
+      catch(NotImplementedHereException e)
+      {
+         LOG.info("A default implementation of postUpdate is running...  Probably not expected!", logPair("tableName", updateInput.getTableName()));
+         return (records);
+      }
    }
 
 
@@ -199,4 +230,59 @@ public interface TableCustomizerInterface
       return (records);
    }
 
+
+   /***************************************************************************
+    ** Optional method to override in a customizer that does the same thing for
+    ** both preInsert & preUpdate.
+    ***************************************************************************/
+   default List<QRecord> preInsertOrUpdate(AbstractActionInput input, List<QRecord> records, boolean isPreview, Optional<List<QRecord>> oldRecordList) throws QException
+   {
+      throw NotImplementedHereException.instance;
+   }
+
+
+   /***************************************************************************
+    ** Optional method to override in a customizer that does the same thing for
+    ** both postInsert & postUpdate.
+    ***************************************************************************/
+   default List<QRecord> postInsertOrUpdate(AbstractActionInput input, List<QRecord> records, Optional<List<QRecord>> oldRecordList) throws QException
+   {
+      throw NotImplementedHereException.instance;
+   }
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   default Optional<Map<Serializable, QRecord>> oldRecordListToMap(String primaryKeyField, Optional<List<QRecord>> oldRecordList)
+   {
+      if(oldRecordList.isPresent())
+      {
+         return (Optional.of(CollectionUtils.listToMap(oldRecordList.get(), r -> r.getValue(primaryKeyField))));
+      }
+      else
+      {
+         return (Optional.empty());
+      }
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   class NotImplementedHereException extends QException
+   {
+      private static NotImplementedHereException instance = new NotImplementedHereException();
+
+
+
+      /***************************************************************************
+       **
+       ***************************************************************************/
+      private NotImplementedHereException()
+      {
+         super("Not implemented here");
+      }
+   }
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/dashboard/widgets/AlertWidgetRenderer.java

@@ -23,11 +23,11 @@ package com.kingsrook.qqq.backend.core.actions.dashboard.widgets;
 
 
 import com.kingsrook.qqq.backend.core.exceptions.QException;
-import com.kingsrook.qqq.backend.core.model.MetaDataProducerInterface;
 import com.kingsrook.qqq.backend.core.model.actions.widgets.RenderWidgetInput;
 import com.kingsrook.qqq.backend.core.model.actions.widgets.RenderWidgetOutput;
 import com.kingsrook.qqq.backend.core.model.dashboard.widgets.AlertData;
 import com.kingsrook.qqq.backend.core.model.dashboard.widgets.WidgetType;
+import com.kingsrook.qqq.backend.core.model.metadata.MetaDataProducerInterface;
 import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
 import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
 import com.kingsrook.qqq.backend.core.model.metadata.dashboard.QWidgetMetaData;
@@ -40,9 +40,9 @@ import com.kingsrook.qqq.backend.core.model.metadata.dashboard.QWidgetMetaData;
  ** - alertType - name of entry in AlertType enum (ERROR, WARNING, SUCCESS)
  ** - alertHtml - html to display inside the alert (other than its icon)
  *******************************************************************************/
-public class ProcessAlertWidget extends AbstractWidgetRenderer implements MetaDataProducerInterface<QWidgetMetaData>
+public class AlertWidgetRenderer extends AbstractWidgetRenderer implements MetaDataProducerInterface<QWidgetMetaData>
 {
-   public static final String NAME = "ProcessAlertWidget";
+   public static final String NAME = "AlertWidgetRenderer";
 
 
 

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/dashboard/widgets/ChildRecordListRenderer.java

@@ -37,6 +37,8 @@ import com.kingsrook.qqq.backend.core.actions.values.QValueFormatter;
 import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.exceptions.QNotFoundException;
+import com.kingsrook.qqq.backend.core.instances.QInstanceValidator;
+import com.kingsrook.qqq.backend.core.instances.validation.plugins.QInstanceValidatorPluginInterface;
 import com.kingsrook.qqq.backend.core.logging.QLogger;
 import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetInput;
@@ -51,12 +53,15 @@ import com.kingsrook.qqq.backend.core.model.actions.widgets.RenderWidgetOutput;
 import com.kingsrook.qqq.backend.core.model.dashboard.widgets.ChildRecordListData;
 import com.kingsrook.qqq.backend.core.model.dashboard.widgets.WidgetType;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
 import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
 import com.kingsrook.qqq.backend.core.model.metadata.dashboard.AbstractWidgetMetaDataBuilder;
 import com.kingsrook.qqq.backend.core.model.metadata.dashboard.QWidgetMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.dashboard.QWidgetMetaDataInterface;
 import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinOn;
 import com.kingsrook.qqq.backend.core.model.metadata.joins.QJoinMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
 import com.kingsrook.qqq.backend.core.utils.JsonUtils;
 import com.kingsrook.qqq.backend.core.utils.StringUtils;
 import com.kingsrook.qqq.backend.core.utils.ValueUtils;
@@ -83,7 +88,9 @@ public class ChildRecordListRenderer extends AbstractWidgetRenderer
          .withIsCard(true)
          .withCodeReference(new QCodeReference(ChildRecordListRenderer.class))
          .withType(WidgetType.CHILD_RECORD_LIST.getType())
-         .withDefaultValue("joinName", join.getName())));
+         .withDefaultValue("joinName", join.getName())
+         .withValidatorPlugin(new ChildRecordListWidgetValidator())
+      ));
    }
 
 
@@ -168,6 +175,7 @@ public class ChildRecordListRenderer extends AbstractWidgetRenderer
          widgetMetaData.withDefaultValue("manageAssociationName", manageAssociationName);
          return (this);
       }
+
    }
 
 
@@ -194,7 +202,7 @@ public class ChildRecordListRenderer extends AbstractWidgetRenderer
          }
          else if(input.getWidgetMetaData().getDefaultValues().containsKey("maxRows"))
          {
-            maxRows = ValueUtils.getValueAsInteger(input.getWidgetMetaData().getDefaultValues().containsKey("maxRows"));
+            maxRows = ValueUtils.getValueAsInteger(input.getWidgetMetaData().getDefaultValues().get("maxRows"));
          }
 
          //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
@@ -299,8 +307,18 @@ public class ChildRecordListRenderer extends AbstractWidgetRenderer
                   }
                }
             }
+
+            if(widgetValues.containsKey("defaultValuesForNewChildRecordsFromParentFields"))
+            {
+               @SuppressWarnings("unchecked")
+               Map<String, String> defaultValuesForNewChildRecordsFromParentFields = (Map<String, String>) widgetValues.get("defaultValuesForNewChildRecordsFromParentFields");
+               widgetData.setDefaultValuesForNewChildRecordsFromParentFields(defaultValuesForNewChildRecordsFromParentFields);
+            }
          }
 
+         widgetData.setAllowRecordEdit(BooleanUtils.isTrue(ValueUtils.getValueAsBoolean(input.getQueryParams().get("allowRecordEdit"))));
+         widgetData.setAllowRecordDelete(BooleanUtils.isTrue(ValueUtils.getValueAsBoolean(input.getQueryParams().get("allowRecordDelete"))));
+
          return (new RenderWidgetOutput(widgetData));
       }
       catch(Exception e)
@@ -310,4 +328,68 @@ public class ChildRecordListRenderer extends AbstractWidgetRenderer
       }
    }
 
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   private static class ChildRecordListWidgetValidator implements QInstanceValidatorPluginInterface<QWidgetMetaDataInterface>
+   {
+
+      /***************************************************************************
+       **
+       ***************************************************************************/
+      @Override
+      public void validate(QWidgetMetaDataInterface widgetMetaData, QInstance qInstance, QInstanceValidator qInstanceValidator)
+      {
+         String prefix = "Widget " + widgetMetaData.getName() + ": ";
+
+         //////////////////////////////////
+         // make sure join name is given //
+         //////////////////////////////////
+         String joinName = ValueUtils.getValueAsString(CollectionUtils.nonNullMap(widgetMetaData.getDefaultValues()).get("joinName"));
+         if(qInstanceValidator.assertCondition(StringUtils.hasContent(joinName), prefix + "defaultValue for joinName must be given"))
+         {
+            ///////////////////////////
+            // make sure join exists //
+            ///////////////////////////
+            QJoinMetaData join = qInstance.getJoin(joinName);
+            if(qInstanceValidator.assertCondition(join != null, prefix + "No join named " + joinName + " exists in the instance"))
+            {
+               //////////////////////////////////////////////////////////////////////////////////
+               // if there's a manageAssociationName, make sure the table has that association //
+               //////////////////////////////////////////////////////////////////////////////////
+               String manageAssociationName = ValueUtils.getValueAsString(widgetMetaData.getDefaultValues().get("manageAssociationName"));
+               if(StringUtils.hasContent(manageAssociationName))
+               {
+                  validateAssociationName(prefix, manageAssociationName, join, qInstance, qInstanceValidator);
+               }
+            }
+         }
+      }
+
+
+
+      /***************************************************************************
+       **
+       ***************************************************************************/
+      private void validateAssociationName(String prefix, String manageAssociationName, QJoinMetaData join, QInstance qInstance, QInstanceValidator qInstanceValidator)
+      {
+         ///////////////////////////////////
+         // make sure join's table exists //
+         ///////////////////////////////////
+         QTableMetaData table = qInstance.getTable(join.getLeftTable());
+         if(table == null)
+         {
+            qInstanceValidator.getErrors().add(prefix + "Unable to validate manageAssociationName, as table [" + join.getLeftTable() + "] on left-side table of join [" + join.getName() + "] does not exist.");
+         }
+         else
+         {
+            if(CollectionUtils.nonNullList(table.getAssociations()).stream().noneMatch(a -> manageAssociationName.equals(a.getName())))
+            {
+               qInstanceValidator.getErrors().add(prefix + "an association named [" + manageAssociationName + "] does not exist on table [" + join.getLeftTable() + "]");
+            }
+         }
+      }
+   }
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/dashboard/widgets/RecordListWidgetRenderer.java

@@ -0,0 +1,251 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.actions.dashboard.widgets;
+
+
+import java.net.URLEncoder;
+import java.nio.charset.Charset;
+import java.util.HashMap;
+import com.kingsrook.qqq.backend.core.actions.tables.CountAction;
+import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
+import com.kingsrook.qqq.backend.core.actions.values.QValueFormatter;
+import com.kingsrook.qqq.backend.core.context.QContext;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.instances.QInstanceValidator;
+import com.kingsrook.qqq.backend.core.instances.validation.plugins.QInstanceValidatorPluginInterface;
+import com.kingsrook.qqq.backend.core.logging.QLogger;
+import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.FilterUseCase;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput;
+import com.kingsrook.qqq.backend.core.model.actions.widgets.RenderWidgetInput;
+import com.kingsrook.qqq.backend.core.model.actions.widgets.RenderWidgetOutput;
+import com.kingsrook.qqq.backend.core.model.dashboard.widgets.ChildRecordListData;
+import com.kingsrook.qqq.backend.core.model.dashboard.widgets.WidgetType;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
+import com.kingsrook.qqq.backend.core.model.metadata.dashboard.AbstractWidgetMetaDataBuilder;
+import com.kingsrook.qqq.backend.core.model.metadata.dashboard.QWidgetMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.dashboard.QWidgetMetaDataInterface;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
+import com.kingsrook.qqq.backend.core.utils.JsonUtils;
+import com.kingsrook.qqq.backend.core.utils.StringUtils;
+import com.kingsrook.qqq.backend.core.utils.ValueUtils;
+import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
+
+
+/*******************************************************************************
+ ** Generic widget to display a list of records.
+ **
+ ** Note, closely related to (and copied from ChildRecordListRenderer.
+ ** opportunity to share more code with that in the future??
+ *******************************************************************************/
+public class RecordListWidgetRenderer extends AbstractWidgetRenderer
+{
+   private static final QLogger LOG = QLogger.getLogger(RecordListWidgetRenderer.class);
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public static Builder widgetMetaDataBuilder(String widgetName)
+   {
+      return (new Builder(new QWidgetMetaData()
+         .withName(widgetName)
+         .withIsCard(true)
+         .withCodeReference(new QCodeReference(RecordListWidgetRenderer.class))
+         .withType(WidgetType.CHILD_RECORD_LIST.getType())
+         .withValidatorPlugin(new RecordListWidgetValidator())
+      ));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public static class Builder extends AbstractWidgetMetaDataBuilder
+   {
+
+      /*******************************************************************************
+       ** Constructor
+       **
+       *******************************************************************************/
+      public Builder(QWidgetMetaData widgetMetaData)
+      {
+         super(widgetMetaData);
+      }
+
+
+
+      /*******************************************************************************
+       **
+       *******************************************************************************/
+      public Builder withLabel(String label)
+      {
+         widgetMetaData.setLabel(label);
+         return (this);
+      }
+
+
+
+      /*******************************************************************************
+       **
+       *******************************************************************************/
+      public Builder withMaxRows(Integer maxRows)
+      {
+         widgetMetaData.withDefaultValue("maxRows", maxRows);
+         return (this);
+      }
+
+
+
+      /*******************************************************************************
+       **
+       *******************************************************************************/
+      public Builder withTableName(String tableName)
+      {
+         widgetMetaData.withDefaultValue("tableName", tableName);
+         return (this);
+      }
+
+
+
+      /*******************************************************************************
+       **
+       *******************************************************************************/
+      public Builder withFilter(QQueryFilter filter)
+      {
+         widgetMetaData.withDefaultValue("filter", filter);
+         return (this);
+      }
+
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public RenderWidgetOutput render(RenderWidgetInput input) throws QException
+   {
+      try
+      {
+         Integer maxRows = null;
+         if(StringUtils.hasContent(input.getQueryParams().get("maxRows")))
+         {
+            maxRows = ValueUtils.getValueAsInteger(input.getQueryParams().get("maxRows"));
+         }
+         else if(input.getWidgetMetaData().getDefaultValues().containsKey("maxRows"))
+         {
+            maxRows = ValueUtils.getValueAsInteger(input.getWidgetMetaData().getDefaultValues().get("maxRows"));
+         }
+
+         QQueryFilter filter = ((QQueryFilter) input.getWidgetMetaData().getDefaultValues().get("filter")).clone();
+         filter.interpretValues(new HashMap<>(input.getQueryParams()), FilterUseCase.DEFAULT);
+         filter.setLimit(maxRows);
+
+         String         tableName = ValueUtils.getValueAsString(input.getWidgetMetaData().getDefaultValues().get("tableName"));
+         QTableMetaData table     = QContext.getQInstance().getTable(tableName);
+
+         QueryInput queryInput = new QueryInput();
+         queryInput.setTableName(tableName);
+         queryInput.setShouldTranslatePossibleValues(true);
+         queryInput.setShouldGenerateDisplayValues(true);
+         queryInput.setFilter(filter);
+         QueryOutput queryOutput = new QueryAction().execute(queryInput);
+
+         QValueFormatter.setBlobValuesToDownloadUrls(table, queryOutput.getRecords());
+
+         int totalRows = queryOutput.getRecords().size();
+         if(maxRows != null && (queryOutput.getRecords().size() == maxRows))
+         {
+            /////////////////////////////////////////////////////////////////////////////////////
+            // if the input said to only do some max, and the # of results we got is that max, //
+            // then do a count query, for displaying 1-n of <count>                            //
+            /////////////////////////////////////////////////////////////////////////////////////
+            CountInput countInput = new CountInput();
+            countInput.setTableName(tableName);
+            countInput.setFilter(filter);
+            totalRows = new CountAction().execute(countInput).getCount();
+         }
+
+         String tablePath   = QContext.getQInstance().getTablePath(tableName);
+         String viewAllLink = tablePath == null ? null : (tablePath + "?filter=" + URLEncoder.encode(JsonUtils.toJson(filter), Charset.defaultCharset()));
+
+         ChildRecordListData widgetData = new ChildRecordListData(input.getQueryParams().get("widgetLabel"), queryOutput, table, tablePath, viewAllLink, totalRows);
+
+         return (new RenderWidgetOutput(widgetData));
+      }
+      catch(Exception e)
+      {
+         LOG.warn("Error rendering record list widget", e, logPair("widgetName", () -> input.getWidgetMetaData().getName()));
+         throw (e);
+      }
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   private static class RecordListWidgetValidator implements QInstanceValidatorPluginInterface<QWidgetMetaDataInterface>
+   {
+
+      /***************************************************************************
+       **
+       ***************************************************************************/
+      @Override
+      public void validate(QWidgetMetaDataInterface widgetMetaData, QInstance qInstance, QInstanceValidator qInstanceValidator)
+      {
+         String prefix = "Widget " + widgetMetaData.getName() + ": ";
+
+         //////////////////////////////////////////////
+         // make sure table name is given and exists //
+         //////////////////////////////////////////////
+         QTableMetaData table = null;
+         String tableName = ValueUtils.getValueAsString(CollectionUtils.nonNullMap(widgetMetaData.getDefaultValues()).get("tableName"));
+         if(qInstanceValidator.assertCondition(StringUtils.hasContent(tableName), prefix + "defaultValue for tableName must be given"))
+         {
+            ////////////////////////////
+            // make sure table exists //
+            ////////////////////////////
+            table = qInstance.getTable(tableName);
+            qInstanceValidator.assertCondition(table != null, prefix + "No table named " + tableName + " exists in the instance");
+         }
+
+         ////////////////////////////////////////////////////////////////////////////////////
+         // make sure filter is given and is valid (only check that if table is given too) //
+         ////////////////////////////////////////////////////////////////////////////////////
+         QQueryFilter filter = ((QQueryFilter) widgetMetaData.getDefaultValues().get("filter"));
+         if(qInstanceValidator.assertCondition(filter != null, prefix + "defaultValue for filter must be given") && table != null)
+         {
+            qInstanceValidator.validateQueryFilter(qInstance, prefix, table, filter, null);
+         }
+      }
+
+   }
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/metadata/AllowAllMetaDataFilter.java

@@ -0,0 +1,92 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.actions.metadata;
+
+
+import com.kingsrook.qqq.backend.core.model.actions.metadata.MetaDataInput;
+import com.kingsrook.qqq.backend.core.model.metadata.dashboard.QWidgetMetaDataInterface;
+import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+
+
+/*******************************************************************************
+ ** a default implementation of MetaDataFilterInterface, that allows all the things
+ *******************************************************************************/
+public class AllowAllMetaDataFilter implements MetaDataFilterInterface
+{
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   @Override
+   public boolean allowTable(MetaDataInput input, QTableMetaData table)
+   {
+      return (true);
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   @Override
+   public boolean allowProcess(MetaDataInput input, QProcessMetaData process)
+   {
+      return (true);
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   @Override
+   public boolean allowReport(MetaDataInput input, QReportMetaData report)
+   {
+      return (true);
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   @Override
+   public boolean allowApp(MetaDataInput input, QAppMetaData app)
+   {
+      return (true);
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   @Override
+   public boolean allowWidget(MetaDataInput input, QWidgetMetaDataInterface widget)
+   {
+      return (true);
+   }
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/metadata/MetaDataAction.java

@@ -28,13 +28,18 @@ import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
 import com.kingsrook.qqq.backend.core.actions.ActionHelper;
+import com.kingsrook.qqq.backend.core.actions.customizers.QCodeLoader;
 import com.kingsrook.qqq.backend.core.actions.permissions.PermissionCheckResult;
 import com.kingsrook.qqq.backend.core.actions.permissions.PermissionsHelper;
 import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.exceptions.QRuntimeException;
+import com.kingsrook.qqq.backend.core.logging.QLogger;
 import com.kingsrook.qqq.backend.core.model.actions.metadata.MetaDataInput;
 import com.kingsrook.qqq.backend.core.model.actions.metadata.MetaDataOutput;
 import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
 import com.kingsrook.qqq.backend.core.model.metadata.dashboard.QWidgetMetaDataInterface;
 import com.kingsrook.qqq.backend.core.model.metadata.frontend.AppTreeNode;
 import com.kingsrook.qqq.backend.core.model.metadata.frontend.QFrontendAppMetaData;
@@ -49,6 +54,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
+import com.kingsrook.qqq.backend.core.utils.memoization.Memoization;
 
 
 /*******************************************************************************
@@ -57,6 +63,12 @@ import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
  *******************************************************************************/
 public class MetaDataAction
 {
+   private static final QLogger LOG = QLogger.getLogger(MetaDataAction.class);
+
+   private static Memoization<QInstance, MetaDataFilterInterface> metaDataFilterMemoization = new Memoization<>();
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/
@@ -64,10 +76,10 @@ public class MetaDataAction
    {
       ActionHelper.validateSession(metaDataInput);
 
-      // todo pre-customization - just get to modify the request?
-      MetaDataOutput metaDataOutput = new MetaDataOutput();
+      MetaDataOutput           metaDataOutput = new MetaDataOutput();
+      Map<String, AppTreeNode> treeNodes      = new LinkedHashMap<>();
 
-      Map<String, AppTreeNode> treeNodes = new LinkedHashMap<>();
+      MetaDataFilterInterface filter = getMetaDataFilter();
 
       /////////////////////////////////////
       // map tables to frontend metadata //
@@ -78,6 +90,11 @@ public class MetaDataAction
          String         tableName = entry.getKey();
          QTableMetaData table     = entry.getValue();
 
+         if(!filter.allowTable(metaDataInput, table))
+         {
+            continue;
+         }
+
          PermissionCheckResult permissionResult = PermissionsHelper.getPermissionCheckResult(metaDataInput, table);
          if(permissionResult.equals(PermissionCheckResult.DENY_HIDE))
          {
@@ -102,6 +119,11 @@ public class MetaDataAction
          String           processName = entry.getKey();
          QProcessMetaData process     = entry.getValue();
 
+         if(!filter.allowProcess(metaDataInput, process))
+         {
+            continue;
+         }
+
          PermissionCheckResult permissionResult = PermissionsHelper.getPermissionCheckResult(metaDataInput, process);
          if(permissionResult.equals(PermissionCheckResult.DENY_HIDE))
          {
@@ -122,6 +144,11 @@ public class MetaDataAction
          String          reportName = entry.getKey();
          QReportMetaData report     = entry.getValue();
 
+         if(!filter.allowReport(metaDataInput, report))
+         {
+            continue;
+         }
+
          PermissionCheckResult permissionResult = PermissionsHelper.getPermissionCheckResult(metaDataInput, report);
          if(permissionResult.equals(PermissionCheckResult.DENY_HIDE))
          {
@@ -142,6 +169,11 @@ public class MetaDataAction
          String                   widgetName = entry.getKey();
          QWidgetMetaDataInterface widget     = entry.getValue();
 
+         if(!filter.allowWidget(metaDataInput, widget))
+         {
+            continue;
+         }
+
          PermissionCheckResult permissionResult = PermissionsHelper.getPermissionCheckResult(metaDataInput, widget);
          if(permissionResult.equals(PermissionCheckResult.DENY_HIDE))
          {
@@ -174,9 +206,19 @@ public class MetaDataAction
             continue;
          }
 
-         apps.put(appName, new QFrontendAppMetaData(app, metaDataOutput));
-         treeNodes.put(appName, new AppTreeNode(app));
+         if(!filter.allowApp(metaDataInput, app))
+         {
+            continue;
+         }
 
+         //////////////////////////////////////
+         // build the frontend-app meta-data //
+         //////////////////////////////////////
+         QFrontendAppMetaData frontendAppMetaData = new QFrontendAppMetaData(app, metaDataOutput);
+
+         /////////////////////////////////////////
+         // add children (if they're permitted) //
+         /////////////////////////////////////////
          if(CollectionUtils.nullSafeHasContents(app.getChildren()))
          {
             for(QAppChildMetaData child : app.getChildren())
@@ -190,9 +232,42 @@ public class MetaDataAction
                   }
                }
 
-               apps.get(appName).addChild(new AppTreeNode(child));
+               //////////////////////////////////////////////////////////////////////////////////////////////////////
+               // if the child was filtered away, so it isn't in its corresponding map, then don't include it here //
+               //////////////////////////////////////////////////////////////////////////////////////////////////////
+               if(child instanceof QTableMetaData table && !tables.containsKey(table.getName()))
+               {
+                  continue;
+               }
+               if(child instanceof QProcessMetaData process && !processes.containsKey(process.getName()))
+               {
+                  continue;
+               }
+               if(child instanceof QReportMetaData report && !reports.containsKey(report.getName()))
+               {
+                  continue;
+               }
+               if(child instanceof QAppMetaData childApp && !apps.containsKey(childApp.getName()))
+               {
+                  // continue;
+               }
+
+               frontendAppMetaData.addChild(new AppTreeNode(child));
             }
          }
+
+         //////////////////////////////////////////////////////////////////////////////////////////////////////
+         // if the app ended up having no children, then discard it                                          //
+         // todo - i think this was wrong, because it didn't take into account ... something nested maybe... //
+         //////////////////////////////////////////////////////////////////////////////////////////////////////
+         if(CollectionUtils.nullSafeIsEmpty(frontendAppMetaData.getChildren()) && CollectionUtils.nullSafeIsEmpty(frontendAppMetaData.getWidgets()))
+         {
+            // LOG.debug("Discarding empty app", logPair("name", frontendAppMetaData.getName()));
+            // continue;
+         }
+
+         apps.put(appName, frontendAppMetaData);
+         treeNodes.put(appName, new AppTreeNode(app));
       }
       metaDataOutput.setApps(apps);
 
@@ -228,6 +303,33 @@ public class MetaDataAction
 
 
 
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   private MetaDataFilterInterface getMetaDataFilter()
+   {
+      return metaDataFilterMemoization.getResult(QContext.getQInstance(), i ->
+      {
+         MetaDataFilterInterface filter                  = null;
+         QCodeReference          metaDataFilterReference = QContext.getQInstance().getMetaDataFilter();
+         if(metaDataFilterReference != null)
+         {
+            filter = QCodeLoader.getAdHoc(MetaDataFilterInterface.class, metaDataFilterReference);
+            LOG.debug("Using new meta-data filter of type: " + filter.getClass().getSimpleName());
+         }
+
+         if(filter == null)
+         {
+            filter = new AllowAllMetaDataFilter();
+            LOG.debug("Using new default (allow-all) meta-data filter");
+         }
+
+         return (filter);
+      }).orElseThrow(() -> new QRuntimeException("Error getting metaDataFilter"));
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/metadata/MetaDataFilterInterface.java

@@ -0,0 +1,64 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.actions.metadata;
+
+
+import com.kingsrook.qqq.backend.core.model.actions.metadata.MetaDataInput;
+import com.kingsrook.qqq.backend.core.model.metadata.dashboard.QWidgetMetaDataInterface;
+import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+
+
+/*******************************************************************************
+ **
+ *******************************************************************************/
+public interface MetaDataFilterInterface
+{
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   boolean allowTable(MetaDataInput input, QTableMetaData table);
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   boolean allowProcess(MetaDataInput input, QProcessMetaData process);
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   boolean allowReport(MetaDataInput input, QReportMetaData report);
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   boolean allowApp(MetaDataInput input, QAppMetaData app);
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   boolean allowWidget(MetaDataInput input, QWidgetMetaDataInterface widget);
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/processes/RunBackendStepAction.java

@@ -40,6 +40,7 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput;
 import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
+import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReferenceLambda;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionInputMetaData;
@@ -257,11 +258,20 @@ public class RunBackendStepAction
       {
          runBackendStepOutput.seedFromRequest(runBackendStepInput);
 
-         Class<?> codeClass  = Class.forName(code.getName());
-         Object   codeObject = codeClass.getConstructor().newInstance();
+         Object codeObject;
+         if(code instanceof QCodeReferenceLambda<?> qCodeReferenceLambda)
+         {
+            codeObject = qCodeReferenceLambda.getLambda();
+         }
+         else
+         {
+            Class<?> codeClass = Class.forName(code.getName());
+            codeObject = codeClass.getConstructor().newInstance();
+         }
+
          if(!(codeObject instanceof BackendStep backendStepCodeObject))
          {
-            throw (new QException("The supplied code [" + codeClass.getName() + "] is not an instance of BackendStep"));
+            throw (new QException("The supplied codeReference [" + code + "] is not a reference to a BackendStep"));
          }
 
          backendStepCodeObject.run(runBackendStepInput, runBackendStepOutput);

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/processes/RunProcessAction.java

@@ -28,9 +28,11 @@ import java.time.temporal.ChronoUnit;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
+import java.util.Objects;
 import java.util.Optional;
 import java.util.UUID;
 import com.kingsrook.qqq.backend.core.actions.ActionHelper;
+import com.kingsrook.qqq.backend.core.actions.customizers.QCodeLoader;
 import com.kingsrook.qqq.backend.core.actions.dashboard.widgets.NoCodeWidgetRenderer;
 import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
 import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
@@ -52,15 +54,18 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
 import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.processes.NoCodeWidgetFrontendComponentMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendComponentMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendStepMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.processes.QStateMachineStep;
 import com.kingsrook.qqq.backend.core.model.metadata.processes.QStepMetaData;
 import com.kingsrook.qqq.backend.core.model.session.QSession;
 import com.kingsrook.qqq.backend.core.processes.implementations.basepull.BasepullConfiguration;
+import com.kingsrook.qqq.backend.core.processes.tracing.ProcessTracerInterface;
 import com.kingsrook.qqq.backend.core.state.InMemoryStateProvider;
 import com.kingsrook.qqq.backend.core.state.StateProviderInterface;
 import com.kingsrook.qqq.backend.core.state.StateType;
@@ -69,6 +74,7 @@ import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
 import com.kingsrook.qqq.backend.core.utils.StringUtils;
 import com.kingsrook.qqq.backend.core.utils.ValueUtils;
 import org.apache.commons.lang.BooleanUtils;
+import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
 
 
 /*******************************************************************************
@@ -85,12 +91,16 @@ public class RunProcessAction
    public static final String BASEPULL_TIMESTAMP_FIELD  = "basepullTimestampField";
    public static final String BASEPULL_CONFIGURATION    = "basepullConfiguration";
 
+   public static final String PROCESS_TRACER_CODE_REFERENCE_FIELD = "processTracerCodeReference";
+
    ////////////////////////////////////////////////////////////////////////////////////////////////
    // indicator that the timestamp field should be updated - e.g., the execute step is finished. //
    ////////////////////////////////////////////////////////////////////////////////////////////////
    public static final String BASEPULL_READY_TO_UPDATE_TIMESTAMP_FIELD = "basepullReadyToUpdateTimestamp";
    public static final String BASEPULL_DID_QUERY_USING_TIMESTAMP_FIELD = "basepullDidQueryUsingTimestamp";
 
+   private ProcessTracerInterface processTracer;
+
 
 
    /*******************************************************************************
@@ -117,9 +127,17 @@ public class RunProcessAction
       }
       runProcessOutput.setProcessUUID(runProcessInput.getProcessUUID());
 
+      traceStartOrResume(runProcessInput, process);
+
       UUIDAndTypeStateKey stateKey     = new UUIDAndTypeStateKey(UUID.fromString(runProcessInput.getProcessUUID()), StateType.PROCESS_STATUS);
       ProcessState        processState = primeProcessState(runProcessInput, stateKey, process);
 
+      /////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      // these should always be clear when we're starting a run - so make sure they haven't leaked from previous //
+      /////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      processState.clearNextStepName();
+      processState.clearBackStepName();
+
       /////////////////////////////////////////////////////////
       // if process is 'basepull' style, keep track of 'now' //
       /////////////////////////////////////////////////////////
@@ -134,90 +152,11 @@ public class RunProcessAction
 
       try
       {
-         String lastStepName = runProcessInput.getStartAfterStep();
-
-         STEP_LOOP:
-         while(true)
+         switch(Objects.requireNonNull(process.getStepFlow(), "Process [" + process.getName() + "] has a null stepFlow."))
          {
-            ///////////////////////////////////////////////////////////////////////////////////////////////////////
-            // always refresh the step list - as any step that runs can modify it (in the process state).        //
-            // this is why we don't do a loop over the step list - as we'd get ConcurrentModificationExceptions. //
-            ///////////////////////////////////////////////////////////////////////////////////////////////////////
-            List<QStepMetaData> stepList = getAvailableStepList(processState, process, lastStepName);
-            if(stepList.isEmpty())
-            {
-               break;
-            }
-
-            QStepMetaData step = stepList.get(0);
-            lastStepName = step.getName();
-
-            if(step instanceof QFrontendStepMetaData frontendStep)
-            {
-               ////////////////////////////////////////////////////////////////
-               // Handle what to do with frontend steps, per request setting //
-               ////////////////////////////////////////////////////////////////
-               switch(runProcessInput.getFrontendStepBehavior())
-               {
-                  case BREAK ->
-                  {
-                     LOG.trace("Breaking process [" + process.getName() + "] at frontend step (as requested by caller): " + step.getName());
-                     processFrontendStepFieldDefaultValues(processState, frontendStep);
-                     processFrontendComponents(processState, frontendStep);
-                     processState.setNextStepName(step.getName());
-                     break STEP_LOOP;
-                  }
-                  case SKIP ->
-                  {
-                     LOG.trace("Skipping frontend step [" + step.getName() + "] in process [" + process.getName() + "] (as requested by caller)");
-
-                     //////////////////////////////////////////////////////////////////////
-                     // much less error prone in case this code changes in the future... //
-                     //////////////////////////////////////////////////////////////////////
-                     // noinspection UnnecessaryContinue
-                     continue;
-                  }
-                  case FAIL ->
-                  {
-                     LOG.trace("Throwing error for frontend step [" + step.getName() + "] in process [" + process.getName() + "] (as requested by caller)");
-                     throw (new QException("Failing process at step " + step.getName() + " (as requested, to fail on frontend steps)"));
-                  }
-                  default -> throw new IllegalStateException("Unexpected value: " + runProcessInput.getFrontendStepBehavior());
-               }
-            }
-            else if(step instanceof QBackendStepMetaData backendStepMetaData)
-            {
-               ///////////////////////
-               // Run backend steps //
-               ///////////////////////
-               LOG.debug("Running backend step [" + step.getName() + "] in process [" + process.getName() + "]");
-               RunBackendStepOutput runBackendStepOutput = runBackendStep(runProcessInput, process, runProcessOutput, stateKey, backendStepMetaData, process, processState);
-
-               /////////////////////////////////////////////////////////////////////////////////////////
-               // if the step returned an override lastStepName, use that to determine how we proceed //
-               /////////////////////////////////////////////////////////////////////////////////////////
-               if(runBackendStepOutput.getOverrideLastStepName() != null)
-               {
-                  LOG.debug("Process step [" + lastStepName + "] returned an overrideLastStepName [" + runBackendStepOutput.getOverrideLastStepName() + "]!");
-                  lastStepName = runBackendStepOutput.getOverrideLastStepName();
-               }
-
-               /////////////////////////////////////////////////////////////////////////////////////////////
-               // similarly, if the step produced an updatedFrontendStepList, propagate that data outward //
-               /////////////////////////////////////////////////////////////////////////////////////////////
-               if(runBackendStepOutput.getUpdatedFrontendStepList() != null)
-               {
-                  LOG.debug("Process step [" + lastStepName + "] generated an updatedFrontendStepList [" + runBackendStepOutput.getUpdatedFrontendStepList().stream().map(s -> s.getName()).toList() + "]!");
-                  runProcessOutput.setUpdatedFrontendStepList(runBackendStepOutput.getUpdatedFrontendStepList());
-               }
-            }
-            else
-            {
-               //////////////////////////////////////////////////
-               // in case we have a different step type, throw //
-               //////////////////////////////////////////////////
-               throw (new QException("Unsure how to run a step of type: " + step.getClass().getName()));
-            }
+            case LINEAR -> runLinearStepLoop(process, processState, stateKey, runProcessInput, runProcessOutput);
+            case STATE_MACHINE -> runStateMachineStep(runProcessInput.getStartAfterStep(), process, processState, stateKey, runProcessInput, runProcessOutput, 0);
+            default -> throw (new QException("Unhandled process step flow: " + process.getStepFlow()));
          }
 
          ///////////////////////////////////////////////////////////////////////////
@@ -237,6 +176,7 @@ public class RunProcessAction
          ////////////////////////////////////////////////////////////
          // upon exception (e.g., one thrown by a step), throw it. //
          ////////////////////////////////////////////////////////////
+         traceBreakOrFinish(runProcessInput, runProcessOutput, qe);
          throw (qe);
       }
       catch(Exception e)
@@ -244,6 +184,7 @@ public class RunProcessAction
          ////////////////////////////////////////////////////////////
          // upon exception (e.g., one thrown by a step), throw it. //
          ////////////////////////////////////////////////////////////
+         traceBreakOrFinish(runProcessInput, runProcessOutput, e);
          throw (new QException("Error running process", e));
       }
       finally
@@ -254,11 +195,317 @@ public class RunProcessAction
          runProcessOutput.setProcessState(processState);
       }
 
+      traceBreakOrFinish(runProcessInput, runProcessOutput, null);
+
       return (runProcessOutput);
    }
 
 
 
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   private void runLinearStepLoop(QProcessMetaData process, ProcessState processState, UUIDAndTypeStateKey stateKey, RunProcessInput runProcessInput, RunProcessOutput runProcessOutput) throws Exception
+   {
+      String lastStepName = runProcessInput.getStartAfterStep();
+      String startAtStep  = runProcessInput.getStartAtStep();
+
+      while(true)
+      {
+         ///////////////////////////////////////////////////////////////////////////////////////////////////////
+         // always refresh the step list - as any step that runs can modify it (in the process state).        //
+         // this is why we don't do a loop over the step list - as we'd get ConcurrentModificationExceptions. //
+         // deal with if we were told, from the input, to start After a step, or start At a step.             //
+         ///////////////////////////////////////////////////////////////////////////////////////////////////////
+         List<QStepMetaData> stepList;
+         if(startAtStep == null)
+         {
+            stepList = getAvailableStepList(processState, process, lastStepName, false);
+         }
+         else
+         {
+            stepList = getAvailableStepList(processState, process, startAtStep, true);
+
+            ///////////////////////////////////////////////////////////////////////////////////
+            // clear this field - so after we run a step, we'll then loop in last-step mode. //
+            ///////////////////////////////////////////////////////////////////////////////////
+            startAtStep = null;
+
+            ///////////////////////////////////////////////////////////////////////////////////
+            // if we're going to run a backend step now, let it see that this is a step-back //
+            ///////////////////////////////////////////////////////////////////////////////////
+            processState.setIsStepBack(true);
+         }
+
+         if(stepList.isEmpty())
+         {
+            break;
+         }
+
+         QStepMetaData step = stepList.get(0);
+         lastStepName = step.getName();
+
+         if(step instanceof QFrontendStepMetaData frontendStep)
+         {
+            LoopTodo loopTodo = prepareForFrontendStep(runProcessInput, process, frontendStep, processState);
+            if(loopTodo == LoopTodo.BREAK)
+            {
+               break;
+            }
+         }
+         else if(step instanceof QBackendStepMetaData backendStepMetaData)
+         {
+            RunBackendStepOutput runBackendStepOutput = runBackendStep(process, processState, stateKey, runProcessInput, runProcessOutput, backendStepMetaData, step);
+
+            /////////////////////////////////////////////////////////////////////////////////////////
+            // if the step returned an override lastStepName, use that to determine how we proceed //
+            /////////////////////////////////////////////////////////////////////////////////////////
+            if(runBackendStepOutput.getOverrideLastStepName() != null)
+            {
+               LOG.debug("Process step [" + lastStepName + "] returned an overrideLastStepName [" + runBackendStepOutput.getOverrideLastStepName() + "]!");
+               lastStepName = runBackendStepOutput.getOverrideLastStepName();
+            }
+         }
+         else
+         {
+            //////////////////////////////////////////////////
+            // in case we have a different step type, throw //
+            //////////////////////////////////////////////////
+            throw (new QException("Unsure how to run a step of type: " + step.getClass().getName()));
+         }
+
+         ////////////////////////////////////////////////////////////////////////////////////////
+         // only let this value be set for the original back step - don't let it stick around. //
+         // if a process wants to keep track of this itself, it can, but in a different slot.  //
+         ////////////////////////////////////////////////////////////////////////////////////////
+         processState.setIsStepBack(false);
+      }
+
+      ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      // in case we broke from the loop above (e.g., by going directly into a frontend step), once again make sure to lower this flag. //
+      ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      processState.setIsStepBack(false);
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   private enum LoopTodo
+   {
+      BREAK,
+      CONTINUE
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   private LoopTodo prepareForFrontendStep(RunProcessInput runProcessInput, QProcessMetaData process, QFrontendStepMetaData step, ProcessState processState) throws QException
+   {
+      ////////////////////////////////////////////////////////////////
+      // Handle what to do with frontend steps, per request setting //
+      ////////////////////////////////////////////////////////////////
+      switch(runProcessInput.getFrontendStepBehavior())
+      {
+         case BREAK ->
+         {
+            LOG.trace("Breaking process [" + process.getName() + "] at frontend step (as requested by caller): " + step.getName());
+            processFrontendStepFieldDefaultValues(processState, step);
+            processFrontendComponents(processState, step);
+            processState.setNextStepName(step.getName());
+
+            if(StringUtils.hasContent(step.getBackStepName()) && processState.getBackStepName().isEmpty())
+            {
+               processState.setBackStepName(step.getBackStepName());
+            }
+
+            return LoopTodo.BREAK;
+         }
+         case SKIP ->
+         {
+            LOG.trace("Skipping frontend step [" + step.getName() + "] in process [" + process.getName() + "] (as requested by caller)");
+            return LoopTodo.CONTINUE;
+         }
+         case FAIL ->
+         {
+            LOG.trace("Throwing error for frontend step [" + step.getName() + "] in process [" + process.getName() + "] (as requested by caller)");
+            throw (new QException("Failing process at step " + step.getName() + " (as requested, to fail on frontend steps)"));
+         }
+         default -> throw new IllegalStateException("Unexpected value: " + runProcessInput.getFrontendStepBehavior());
+      }
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   private void runStateMachineStep(String lastStepName, QProcessMetaData process, ProcessState processState, UUIDAndTypeStateKey stateKey, RunProcessInput runProcessInput, RunProcessOutput runProcessOutput, int stackDepth) throws Exception
+   {
+      //////////////////////////////
+      // check for stack-overflow //
+      //////////////////////////////
+      Integer maxStateMachineProcessStepFlowStackDepth = Objects.requireNonNullElse(runProcessInput.getValueInteger("maxStateMachineProcessStepFlowStackDepth"), 20);
+      if(stackDepth > maxStateMachineProcessStepFlowStackDepth)
+      {
+         throw (new QException("StateMachine process recurred too many times (exceeded maxStateMachineProcessStepFlowStackDepth of " + maxStateMachineProcessStepFlowStackDepth + ")"));
+      }
+
+      //////////////////////////////////
+      // figure out what step to run: //
+      //////////////////////////////////
+      QStepMetaData step = null;
+      if(!StringUtils.hasContent(lastStepName))
+      {
+         ////////////////////////////////////////////////////////////////////
+         // if no lastStepName is given, start at the process's first step //
+         ////////////////////////////////////////////////////////////////////
+         if(CollectionUtils.nullSafeIsEmpty(process.getStepList()))
+         {
+            throw (new QException("Process [" + process.getName() + "] does not have a step list defined."));
+         }
+         step = process.getStepList().get(0);
+      }
+      else
+      {
+         /////////////////////////////////////
+         // else run the given lastStepName //
+         /////////////////////////////////////
+         processState.clearNextStepName();
+         processState.clearBackStepName();
+         step = process.getStep(lastStepName);
+         if(step == null)
+         {
+            throw (new QException("Could not find step by name [" + lastStepName + "]"));
+         }
+      }
+
+      /////////////////////////////////////////////////////////////////////////
+      // for the flow of:                                                    //
+      // we were on a frontend step (as a sub-step of a state machine step), //
+      // and now we're here to run that state-step's backend step -          //
+      // find the state-machine step containing this frontend step.          //
+      /////////////////////////////////////////////////////////////////////////
+      String skipSubStepsUntil = null;
+      if(step instanceof QFrontendStepMetaData frontendStepMetaData)
+      {
+         QStateMachineStep stateMachineStep = getStateMachineStepContainingSubStep(process, frontendStepMetaData.getName());
+         if(stateMachineStep == null)
+         {
+            throw (new QException("Could not find stateMachineStep that contains last-frontend step: " + frontendStepMetaData.getName()));
+         }
+         step = stateMachineStep;
+
+         //////////////////////////////////////////////////////////////////////////////////
+         // set this flag, to know to skip this frontend step in the sub-step loop below //
+         //////////////////////////////////////////////////////////////////////////////////
+         skipSubStepsUntil = frontendStepMetaData.getName();
+      }
+
+      if(!(step instanceof QStateMachineStep stateMachineStep))
+      {
+         throw (new QException("Have a non-stateMachineStep in a process using stateMachine flow... " + step.getClass().getName()));
+      }
+
+      ///////////////////////
+      // run the sub-steps //
+      ///////////////////////
+      boolean ranAnySubSteps = false;
+      for(QStepMetaData subStep : stateMachineStep.getSubSteps())
+      {
+         ///////////////////////////////////////////////////////////////////////////////////////////////
+         // ok, well, skip them if this flag is set (and clear the flag once we've hit this sub-step) //
+         ///////////////////////////////////////////////////////////////////////////////////////////////
+         if(skipSubStepsUntil != null)
+         {
+            if(skipSubStepsUntil.equals(subStep.getName()))
+            {
+               skipSubStepsUntil = null;
+            }
+            continue;
+         }
+
+         ranAnySubSteps = true;
+         if(subStep instanceof QFrontendStepMetaData frontendStep)
+         {
+            LoopTodo loopTodo = prepareForFrontendStep(runProcessInput, process, frontendStep, processState);
+            if(loopTodo == LoopTodo.BREAK)
+            {
+               return;
+            }
+         }
+         else if(subStep instanceof QBackendStepMetaData backendStepMetaData)
+         {
+            RunBackendStepOutput runBackendStepOutput = runBackendStep(process, processState, stateKey, runProcessInput, runProcessOutput, backendStepMetaData, step);
+            Optional<String>     nextStepName         = runBackendStepOutput.getProcessState().getNextStepName();
+
+            if(nextStepName.isEmpty() && StringUtils.hasContent(stateMachineStep.getDefaultNextStepName()))
+            {
+               nextStepName = Optional.of(stateMachineStep.getDefaultNextStepName());
+            }
+
+            if(nextStepName.isPresent())
+            {
+               //////////////////////////////////////////////////////////////////////////////////////////////////////
+               // if we've been given a next-step-name, go to that step now.                                       //
+               // it might be a backend-only stateMachineStep, in which case, we should run that backend step now. //
+               // or it might be a frontend-then-backend step, in which case, we want to go to that frontend step. //
+               // if we weren't given a next-step-name, then we should stay in the same state - either to finish   //
+               // its sub-steps, or, to fall out of the loop and end the process.                                  //
+               //////////////////////////////////////////////////////////////////////////////////////////////////////
+               processState.clearNextStepName();
+               processState.clearBackStepName();
+               runStateMachineStep(nextStepName.get(), process, processState, stateKey, runProcessInput, runProcessOutput, stackDepth + 1);
+               return;
+            }
+         }
+         else
+         {
+            //////////////////////////////////////////////////
+            // in case we have a different step type, throw //
+            //////////////////////////////////////////////////
+            throw (new QException("Unsure how to run a step of type: " + step.getClass().getName()));
+         }
+      }
+
+      if(!ranAnySubSteps)
+      {
+         if(StringUtils.hasContent(stateMachineStep.getDefaultNextStepName()))
+         {
+            runStateMachineStep(stateMachineStep.getDefaultNextStepName(), process, processState, stateKey, runProcessInput, runProcessOutput, stackDepth + 1);
+         }
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public QStateMachineStep getStateMachineStepContainingSubStep(QProcessMetaData process, String stepName)
+   {
+      for(QStepMetaData step : process.getAllSteps().values())
+      {
+         if(step instanceof QStateMachineStep stateMachineStep)
+         {
+            for(QStepMetaData subStep : stateMachineStep.getSubSteps())
+            {
+               if(subStep.getName().equals(stepName))
+               {
+                  return (stateMachineStep);
+               }
+            }
+         }
+      }
+
+      return (null);
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/
@@ -336,12 +583,12 @@ public class RunProcessAction
          ///////////////////////////////////////////////////
          runProcessInput.seedFromProcessState(optionalProcessState.get());
 
-         ///////////////////////////////////////////////////////////////////////////////////////////////////
-         // if we're restoring an old state, we can discard a previously stored updatedFrontendStepList - //
-         // it is only needed on the transitional edge from a backend-step to a frontend step, but not    //
-         // in the other directly                                                                         //
-         ///////////////////////////////////////////////////////////////////////////////////////////////////
-         optionalProcessState.get().setUpdatedFrontendStepList(null);
+         /////////////////////////////////////////////////////////////////////////////////////////////////////
+         // if we're restoring an old state, we can discard a previously stored processMetaDataAdjustment - //
+         // it is only needed on the transitional edge from a backend-step to a frontend step, but not      //
+         // in the other directly                                                                           //
+         /////////////////////////////////////////////////////////////////////////////////////////////////////
+         optionalProcessState.get().setProcessMetaDataAdjustment(null);
 
          ///////////////////////////////////////////////////////////////////////////
          // if there were values from the caller, put those (back) in the request //
@@ -356,16 +603,40 @@ public class RunProcessAction
       }
 
       ProcessState processState = optionalProcessState.get();
-      processState.clearNextStepName();
       return processState;
    }
 
 
 
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   private RunBackendStepOutput runBackendStep(QProcessMetaData process, ProcessState processState, UUIDAndTypeStateKey stateKey, RunProcessInput runProcessInput, RunProcessOutput runProcessOutput, QBackendStepMetaData backendStepMetaData, QStepMetaData step) throws Exception
+   {
+      ///////////////////////
+      // Run backend steps //
+      ///////////////////////
+      LOG.debug("Running backend step [" + step.getName() + "] in process [" + process.getName() + "]");
+      RunBackendStepOutput runBackendStepOutput = runBackendStep(runProcessInput, process, runProcessOutput, stateKey, backendStepMetaData, process, processState);
+
+      //////////////////////////////////////////////////////////////////////////////////////////////
+      // similarly, if the step produced a processMetaDataAdjustment, propagate that data outward //
+      //////////////////////////////////////////////////////////////////////////////////////////////
+      if(runBackendStepOutput.getProcessMetaDataAdjustment() != null)
+      {
+         LOG.debug("Process step [" + step.getName() + "] generated a ProcessMetaDataAdjustment [" + runBackendStepOutput.getProcessMetaDataAdjustment() + "]!");
+         runProcessOutput.setProcessMetaDataAdjustment(runBackendStepOutput.getProcessMetaDataAdjustment());
+      }
+
+      return runBackendStepOutput;
+   }
+
+
+
    /*******************************************************************************
     ** Run a single backend step.
     *******************************************************************************/
-   protected RunBackendStepOutput runBackendStep(RunProcessInput runProcessInput, QProcessMetaData process, RunProcessOutput runProcessOutput, UUIDAndTypeStateKey stateKey, QBackendStepMetaData backendStep, QProcessMetaData qProcessMetaData, ProcessState processState) throws Exception
+   RunBackendStepOutput runBackendStep(RunProcessInput runProcessInput, QProcessMetaData process, RunProcessOutput runProcessOutput, UUIDAndTypeStateKey stateKey, QBackendStepMetaData backendStep, QProcessMetaData qProcessMetaData, ProcessState processState) throws Exception
    {
       RunBackendStepInput runBackendStepInput = new RunBackendStepInput(processState);
       runBackendStepInput.setProcessName(process.getName());
@@ -373,6 +644,7 @@ public class RunProcessAction
       runBackendStepInput.setCallback(runProcessInput.getCallback());
       runBackendStepInput.setFrontendStepBehavior(runProcessInput.getFrontendStepBehavior());
       runBackendStepInput.setAsyncJobCallback(runProcessInput.getAsyncJobCallback());
+      runBackendStepInput.setProcessTracer(processTracer);
 
       runBackendStepInput.setTableName(process.getTableName());
       if(!StringUtils.hasContent(runBackendStepInput.getTableName()))
@@ -394,9 +666,13 @@ public class RunProcessAction
          runBackendStepInput.setBasepullLastRunTime((Instant) runProcessInput.getValues().get(BASEPULL_LAST_RUNTIME_KEY));
       }
 
+      traceStepStart(runBackendStepInput);
+
       RunBackendStepOutput runBackendStepOutput = new RunBackendStepAction().execute(runBackendStepInput);
       storeState(stateKey, runBackendStepOutput.getProcessState());
 
+      traceStepFinish(runBackendStepInput, runBackendStepOutput);
+
       if(runBackendStepOutput.getException() != null)
       {
          runProcessOutput.setException(runBackendStepOutput.getException());
@@ -410,8 +686,10 @@ public class RunProcessAction
 
    /*******************************************************************************
     ** Get the list of steps which are eligible to run.
+    **
+    ** lastStep will be included in the list, or not, based on includeLastStep.
     *******************************************************************************/
-   private List<QStepMetaData> getAvailableStepList(ProcessState processState, QProcessMetaData process, String lastStep) throws QException
+   static List<QStepMetaData> getAvailableStepList(ProcessState processState, QProcessMetaData process, String lastStep, boolean includeLastStep) throws QException
    {
       if(lastStep == null)
       {
@@ -438,6 +716,10 @@ public class RunProcessAction
             if(stepName.equals(lastStep))
             {
                foundLastStep = true;
+               if(includeLastStep)
+               {
+                  validStepNames.add(stepName);
+               }
             }
          }
          return (stepNamesToSteps(process, validStepNames));
@@ -449,7 +731,7 @@ public class RunProcessAction
    /*******************************************************************************
     **
     *******************************************************************************/
-   private List<QStepMetaData> stepNamesToSteps(QProcessMetaData process, List<String> stepNames) throws QException
+   private static List<QStepMetaData> stepNamesToSteps(QProcessMetaData process, List<String> stepNames) throws QException
    {
       List<QStepMetaData> result = new ArrayList<>();
 
@@ -533,13 +815,14 @@ public class RunProcessAction
       {
          QSession         session         = QContext.getQSession();
          QBackendMetaData backendMetaData = QContext.getQInstance().getBackend(process.getVariantBackend());
-         if(session.getBackendVariants() == null || !session.getBackendVariants().containsKey(backendMetaData.getVariantOptionsTableTypeValue()))
+         String           variantTypeKey  = backendMetaData.getBackendVariantsConfig().getVariantTypeKey();
+         if(session.getBackendVariants() == null || !session.getBackendVariants().containsKey(variantTypeKey))
          {
             LOG.warn("Could not find Backend Variant information for Backend '" + backendMetaData.getName() + "'");
          }
          else
          {
-            basepullKeyValue += "-" + session.getBackendVariants().get(backendMetaData.getVariantOptionsTableTypeValue());
+            basepullKeyValue += "-" + session.getBackendVariants().get(variantTypeKey);
          }
       }
 
@@ -668,4 +951,153 @@ public class RunProcessAction
       runProcessInput.getValues().put(BASEPULL_TIMESTAMP_FIELD, basepullConfiguration.getTimestampField());
       runProcessInput.getValues().put(BASEPULL_CONFIGURATION, basepullConfiguration);
    }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   private void setupProcessTracer(RunProcessInput runProcessInput, QProcessMetaData process)
+   {
+      try
+      {
+         if(process.getProcessTracerCodeReference() != null)
+         {
+            processTracer = QCodeLoader.getAdHoc(ProcessTracerInterface.class, process.getProcessTracerCodeReference());
+         }
+
+         Serializable processTracerCodeReference = runProcessInput.getValue(PROCESS_TRACER_CODE_REFERENCE_FIELD);
+         if(processTracerCodeReference != null)
+         {
+            if(processTracerCodeReference instanceof QCodeReference codeReference)
+            {
+               processTracer = QCodeLoader.getAdHoc(ProcessTracerInterface.class, codeReference);
+            }
+         }
+      }
+      catch(Exception e)
+      {
+         LOG.warn("Error setting up processTracer", e, logPair("processName", runProcessInput.getProcessName()));
+      }
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   private void traceStartOrResume(RunProcessInput runProcessInput, QProcessMetaData process)
+   {
+      setupProcessTracer(runProcessInput, process);
+
+      try
+      {
+         if(processTracer != null)
+         {
+            if(StringUtils.hasContent(runProcessInput.getStartAfterStep()) || StringUtils.hasContent(runProcessInput.getStartAtStep()))
+            {
+               processTracer.handleProcessResume(runProcessInput);
+            }
+            else
+            {
+               processTracer.handleProcessStart(runProcessInput);
+            }
+         }
+      }
+      catch(Exception e)
+      {
+         LOG.info("Error in traceStart", e, logPair("processName", runProcessInput.getProcessName()));
+      }
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   private void traceBreakOrFinish(RunProcessInput runProcessInput, RunProcessOutput runProcessOutput, Exception processException)
+   {
+      try
+      {
+         if(processTracer != null)
+         {
+            ProcessState processState = runProcessOutput.getProcessState();
+            boolean      isBreak      = true;
+
+            /////////////////////////////////////////////////////////////
+            // if there's no next step, that means the process is done //
+            /////////////////////////////////////////////////////////////
+            if(processState.getNextStepName().isEmpty())
+            {
+               isBreak = false;
+            }
+            else
+            {
+               /////////////////////////////////////////////////////////////////
+               // or if the next step is the last index, then we're also done //
+               /////////////////////////////////////////////////////////////////
+               String nextStepName  = processState.getNextStepName().get();
+               int    nextStepIndex = processState.getStepList().indexOf(nextStepName);
+               if(nextStepIndex == processState.getStepList().size() - 1)
+               {
+                  isBreak = false;
+               }
+            }
+
+            if(isBreak)
+            {
+               processTracer.handleProcessBreak(runProcessInput, runProcessOutput, processException);
+            }
+            else
+            {
+               processTracer.handleProcessFinish(runProcessInput, runProcessOutput, processException);
+            }
+         }
+      }
+      catch(Exception e)
+      {
+         LOG.info("Error in traceProcessFinish", e, logPair("processName", runProcessInput.getProcessName()));
+      }
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   private void traceStepStart(RunBackendStepInput runBackendStepInput)
+   {
+      try
+      {
+         if(processTracer != null)
+         {
+            processTracer.handleStepStart(runBackendStepInput);
+         }
+      }
+      catch(Exception e)
+      {
+         LOG.info("Error in traceStepFinish", e, logPair("processName", runBackendStepInput.getProcessName()), logPair("stepName", runBackendStepInput.getStepName()));
+      }
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   private void traceStepFinish(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput)
+   {
+      try
+      {
+         if(processTracer != null)
+         {
+            processTracer.handleStepFinish(runBackendStepInput, runBackendStepOutput);
+         }
+      }
+      catch(Exception e)
+      {
+         LOG.info("Error in traceStepFinish", e, logPair("processName", runBackendStepInput.getProcessName()), logPair("stepName", runBackendStepInput.getStepName()));
+      }
+   }
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/CsvExportStreamer.java

@@ -108,7 +108,7 @@ public class CsvExportStreamer implements ExportStreamerInterface
       }
       catch(Exception e)
       {
-         throw (new QReportingException("Error starting CSV report"));
+         throw (new QReportingException("Error starting CSV report", e));
       }
    }
 

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/GenerateReportAction.java

@@ -71,6 +71,7 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryJoin;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.FieldAndJoinTable;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
 import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportDataSource;
@@ -567,7 +568,7 @@ public class GenerateReportAction extends AbstractQActionFunction<ReportInput, R
             // all pivotFields that are possible value sources are implicitly translated //
             ///////////////////////////////////////////////////////////////////////////////
             QTableMetaData    mainTable         = QContext.getQInstance().getTable(dataSource.getSourceTable());
-            FieldAndJoinTable fieldAndJoinTable = getFieldAndJoinTable(mainTable, summaryFieldName);
+            FieldAndJoinTable fieldAndJoinTable = FieldAndJoinTable.get(mainTable, summaryFieldName);
             if(fieldAndJoinTable.field().getPossibleValueSourceName() != null)
             {
                fieldsToTranslatePossibleValues.add(summaryFieldName);
@@ -580,32 +581,6 @@ public class GenerateReportAction extends AbstractQActionFunction<ReportInput, R
 
 
 
-   /*******************************************************************************
-    **
-    *******************************************************************************/
-   public static FieldAndJoinTable getFieldAndJoinTable(QTableMetaData mainTable, String fieldName) throws QException
-   {
-      if(fieldName.indexOf('.') > -1)
-      {
-         String joinTableName = fieldName.replaceAll("\\..*", "");
-         String joinFieldName = fieldName.replaceAll(".*\\.", "");
-
-         QTableMetaData joinTable = QContext.getQInstance().getTable(joinTableName);
-         if(joinTable == null)
-         {
-            throw (new QException("Unrecognized join table name: " + joinTableName));
-         }
-
-         return new FieldAndJoinTable(joinTable.getField(joinFieldName), joinTable);
-      }
-      else
-      {
-         return new FieldAndJoinTable(mainTable.getField(fieldName), mainTable);
-      }
-   }
-
-
-
    /*******************************************************************************
     **
     *******************************************************************************/
@@ -756,7 +731,7 @@ public class GenerateReportAction extends AbstractQActionFunction<ReportInput, R
          SummaryKey key = new SummaryKey();
          for(String summaryFieldName : view.getSummaryFields())
          {
-            FieldAndJoinTable fieldAndJoinTable = getFieldAndJoinTable(table, summaryFieldName);
+            FieldAndJoinTable fieldAndJoinTable = FieldAndJoinTable.get(table, summaryFieldName);
             Serializable      summaryValue      = record.getValue(summaryFieldName);
             if(fieldAndJoinTable.field().getPossibleValueSourceName() != null)
             {
@@ -811,7 +786,7 @@ public class GenerateReportAction extends AbstractQActionFunction<ReportInput, R
             //////////////////////////////////////////////////////
             // todo - memoize this, if we ever need to optimize //
             //////////////////////////////////////////////////////
-            FieldAndJoinTable fieldAndJoinTable = getFieldAndJoinTable(table, fieldName);
+            FieldAndJoinTable fieldAndJoinTable = FieldAndJoinTable.get(table, fieldName);
             field = fieldAndJoinTable.field();
          }
          catch(Exception e)
@@ -956,7 +931,7 @@ public class GenerateReportAction extends AbstractQActionFunction<ReportInput, R
       List<QFieldMetaData> fields = new ArrayList<>();
       for(String summaryFieldName : view.getSummaryFields())
       {
-         FieldAndJoinTable fieldAndJoinTable = getFieldAndJoinTable(table, summaryFieldName);
+         FieldAndJoinTable fieldAndJoinTable = FieldAndJoinTable.get(table, summaryFieldName);
          fields.add(new QFieldMetaData(summaryFieldName, fieldAndJoinTable.field().getType()).withLabel(fieldAndJoinTable.field().getLabel())); // todo do we need the type?  if so need table as input here
       }
       for(QReportField column : view.getColumns())
@@ -1208,27 +1183,4 @@ public class GenerateReportAction extends AbstractQActionFunction<ReportInput, R
    {
    }
 
-
-
-   /*******************************************************************************
-    **
-    *******************************************************************************/
-   public record FieldAndJoinTable(QFieldMetaData field, QTableMetaData joinTable)
-   {
-
-      /*******************************************************************************
-       **
-       *******************************************************************************/
-      public String getLabel(QTableMetaData mainTable)
-      {
-         if(mainTable.getName().equals(joinTable.getName()))
-         {
-            return (field.getLabel());
-         }
-         else
-         {
-            return (joinTable.getLabel() + ": " + field.getLabel());
-         }
-      }
-   }
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/CountAction.java

@@ -32,6 +32,7 @@ import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.logging.QLogger;
 import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountOutput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
 import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 import com.kingsrook.qqq.backend.core.model.querystats.QueryStat;
@@ -82,6 +83,22 @@ public class CountAction
 
 
 
+   /*******************************************************************************
+    ** shorthand way to call for the most common use-case, when you just want the
+    ** count to be returned, and you just want to pass in a table name and filter.
+    *******************************************************************************/
+   public static Integer execute(String tableName, QQueryFilter filter) throws QException
+   {
+      CountAction countAction = new CountAction();
+      CountInput  countInput  = new CountInput();
+      countInput.setTableName(tableName);
+      countInput.setFilter(filter);
+      CountOutput countOutput = countAction.execute(countInput);
+      return (countOutput.getCount());
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/DeleteAction.java

@@ -82,6 +82,11 @@ public class DeleteAction
    {
       ActionHelper.validateSession(deleteInput);
 
+      if(deleteInput.getTableName() == null)
+      {
+         throw (new QException("Table name was not specified in delete input"));
+      }
+
       QTableMetaData table               = deleteInput.getTable();
       String         primaryKeyFieldName = table.getPrimaryKeyField();
       QFieldMetaData primaryKeyField     = table.getField(primaryKeyFieldName);
@@ -320,7 +325,7 @@ public class DeleteAction
       QTableMetaData table               = deleteInput.getTable();
       List<QRecord>  primaryKeysNotFound = validateRecordsExistAndCanBeAccessed(deleteInput, oldRecordList.get());
 
-      ValidateRecordSecurityLockHelper.validateSecurityFields(table, oldRecordList.get(), ValidateRecordSecurityLockHelper.Action.DELETE);
+      ValidateRecordSecurityLockHelper.validateSecurityFields(table, oldRecordList.get(), ValidateRecordSecurityLockHelper.Action.DELETE, deleteInput.getTransaction());
 
       ///////////////////////////////////////////////////////////////////////////
       // after all validations, run the pre-delete customizer, if there is one //

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/GetAction.java

@@ -238,6 +238,11 @@ public class GetAction
     *******************************************************************************/
    public static QRecord execute(String tableName, Serializable primaryKey) throws QException
    {
+      if(primaryKey instanceof QQueryFilter)
+      {
+         LOG.warn("Unexpected use of QQueryFilter instead of primary key in GetAction call");
+      }
+
       GetAction getAction = new GetAction();
       GetInput  getInput  = new GetInput(tableName).withPrimaryKey(primaryKey);
       return getAction.executeForRecord(getInput);

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/InsertAction.java

@@ -67,6 +67,7 @@ import com.kingsrook.qqq.backend.core.model.statusmessages.QWarningMessage;
 import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleDispatcher;
 import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleInterface;
 import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
+import com.kingsrook.qqq.backend.core.utils.StringUtils;
 import com.kingsrook.qqq.backend.core.utils.ValueUtils;
 import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
 
@@ -110,6 +111,12 @@ public class InsertAction extends AbstractQActionFunction<InsertInput, InsertOut
    public InsertOutput execute(InsertInput insertInput) throws QException
    {
       ActionHelper.validateSession(insertInput);
+
+      if(!StringUtils.hasContent(insertInput.getTableName()))
+      {
+         throw (new QException("Table name was not specified in insert input"));
+      }
+
       QTableMetaData table = insertInput.getTable();
 
       if(table == null)
@@ -122,7 +129,7 @@ public class InsertAction extends AbstractQActionFunction<InsertInput, InsertOut
       /////////////////////////////
       // run standard validators //
       /////////////////////////////
-      performValidations(insertInput, false);
+      performValidations(insertInput, false, false);
 
       //////////////////////////////////////////////////////
       // use the backend module to actually do the insert //
@@ -225,7 +232,7 @@ public class InsertAction extends AbstractQActionFunction<InsertInput, InsertOut
    /*******************************************************************************
     **
     *******************************************************************************/
-   public void performValidations(InsertInput insertInput, boolean isPreview) throws QException
+   public void performValidations(InsertInput insertInput, boolean isPreview, boolean didAlreadyRunCustomizer) throws QException
    {
       if(CollectionUtils.nullSafeIsEmpty(insertInput.getRecords()))
       {
@@ -237,12 +244,10 @@ public class InsertAction extends AbstractQActionFunction<InsertInput, InsertOut
       ///////////////////////////////////////////////////////////////////
       // load the pre-insert customizer and set it up, if there is one //
       // then we'll run it based on its WhenToRun value                //
+      // note - if we already ran it, then don't re-run it!            //
       ///////////////////////////////////////////////////////////////////
-      Optional<TableCustomizerInterface> preInsertCustomizer = QCodeLoader.getTableCustomizer(table, TableCustomizers.PRE_INSERT_RECORD.getRole());
-      if(preInsertCustomizer.isPresent())
-      {
-         runPreInsertCustomizerIfItIsTime(insertInput, isPreview, preInsertCustomizer, AbstractPreInsertCustomizer.WhenToRun.BEFORE_ALL_VALIDATIONS);
-      }
+      Optional<TableCustomizerInterface> preInsertCustomizer = didAlreadyRunCustomizer ? Optional.empty() : QCodeLoader.getTableCustomizer(table, TableCustomizers.PRE_INSERT_RECORD.getRole());
+      runPreInsertCustomizerIfItIsTime(insertInput, isPreview, preInsertCustomizer, AbstractPreInsertCustomizer.WhenToRun.BEFORE_ALL_VALIDATIONS);
 
       setDefaultValuesInRecords(table, insertInput.getRecords());
 
@@ -258,7 +263,7 @@ public class InsertAction extends AbstractQActionFunction<InsertInput, InsertOut
       }
 
       runPreInsertCustomizerIfItIsTime(insertInput, isPreview, preInsertCustomizer, AbstractPreInsertCustomizer.WhenToRun.BEFORE_SECURITY_CHECKS);
-      ValidateRecordSecurityLockHelper.validateSecurityFields(insertInput.getTable(), insertInput.getRecords(), ValidateRecordSecurityLockHelper.Action.INSERT);
+      ValidateRecordSecurityLockHelper.validateSecurityFields(insertInput.getTable(), insertInput.getRecords(), ValidateRecordSecurityLockHelper.Action.INSERT, insertInput.getTransaction());
 
       runPreInsertCustomizerIfItIsTime(insertInput, isPreview, preInsertCustomizer, AbstractPreInsertCustomizer.WhenToRun.AFTER_ALL_VALIDATIONS);
    }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/QueryAction.java

@@ -54,6 +54,7 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryJoin;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.data.QRecordEntity;
 import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.AdornmentType;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
@@ -266,6 +267,22 @@ public class QueryAction
 
 
 
+   /*******************************************************************************
+    ** shorthand way to call for the most common use-case, when you just want the
+    ** entities to be returned, and you just want to pass in a table name and filter.
+    *******************************************************************************/
+   public static <T extends QRecordEntity> List<T> execute(String tableName, Class<T> entityClass, QQueryFilter filter) throws QException
+   {
+      QueryAction queryAction = new QueryAction();
+      QueryInput  queryInput  = new QueryInput();
+      queryInput.setTableName(tableName);
+      queryInput.setFilter(filter);
+      QueryOutput queryOutput = queryAction.execute(queryInput);
+      return (queryOutput.getRecordEntities(entityClass));
+   }
+
+
+
    /*******************************************************************************
     ** shorthand way to call for the most common use-case, when you just want the
     ** records to be returned, and you just want to pass in a table name and filter.

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/StorageAction.java

@@ -47,7 +47,8 @@ public class StorageAction
 {
 
    /*******************************************************************************
-    **
+    ** create an output stream in the storage backend - that can be written to,
+    ** for the purpose of inserting or writing a file into storage.
     *******************************************************************************/
    public OutputStream createOutputStream(StorageInput storageInput) throws QException
    {
@@ -59,7 +60,8 @@ public class StorageAction
 
 
    /*******************************************************************************
-    **
+    ** create an input stream in the storage backend - that can be read from,
+    ** for the purpose of getting or reading a file from storage.
     *******************************************************************************/
    public InputStream getInputStream(StorageInput storageInput) throws QException
    {

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/UpdateAction.java

@@ -74,6 +74,7 @@ import com.kingsrook.qqq.backend.core.model.statusmessages.QWarningMessage;
 import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleDispatcher;
 import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleInterface;
 import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
+import com.kingsrook.qqq.backend.core.utils.StringUtils;
 import com.kingsrook.qqq.backend.core.utils.ValueUtils;
 import org.apache.commons.lang.BooleanUtils;
 import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
@@ -118,6 +119,11 @@ public class UpdateAction
    {
       ActionHelper.validateSession(updateInput);
 
+      if(!StringUtils.hasContent(updateInput.getTableName()))
+      {
+         throw (new QException("Table name was not specified in update input"));
+      }
+
       QTableMetaData table = updateInput.getTable();
 
       //////////////////////////////////////////////////////
@@ -261,7 +267,7 @@ public class UpdateAction
       }
       else
       {
-         ValidateRecordSecurityLockHelper.validateSecurityFields(table, updateInput.getRecords(), ValidateRecordSecurityLockHelper.Action.UPDATE);
+         ValidateRecordSecurityLockHelper.validateSecurityFields(table, updateInput.getRecords(), ValidateRecordSecurityLockHelper.Action.UPDATE, updateInput.getTransaction());
       }
 
       if(updateInput.getInputSource().shouldValidateRequiredFields())
@@ -374,7 +380,7 @@ public class UpdateAction
             }
          }
 
-         ValidateRecordSecurityLockHelper.validateSecurityFields(table, updateInput.getRecords(), ValidateRecordSecurityLockHelper.Action.UPDATE);
+         ValidateRecordSecurityLockHelper.validateSecurityFields(table, updateInput.getRecords(), ValidateRecordSecurityLockHelper.Action.UPDATE, updateInput.getTransaction());
 
          for(QRecord record : page)
          {

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/helpers/QueryStatManager.java

@@ -31,16 +31,12 @@ import java.util.concurrent.Executors;
 import java.util.concurrent.ScheduledExecutorService;
 import java.util.concurrent.TimeUnit;
 import java.util.function.Supplier;
-import com.kingsrook.qqq.backend.core.actions.tables.GetAction;
 import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
 import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.instances.QMetaDataVariableInterpreter;
 import com.kingsrook.qqq.backend.core.logging.QLogger;
-import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetInput;
-import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetOutput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
-import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertOutput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
@@ -54,12 +50,10 @@ import com.kingsrook.qqq.backend.core.model.querystats.QueryStatCriteriaField;
 import com.kingsrook.qqq.backend.core.model.querystats.QueryStatJoinTable;
 import com.kingsrook.qqq.backend.core.model.querystats.QueryStatOrderByField;
 import com.kingsrook.qqq.backend.core.model.session.QSession;
-import com.kingsrook.qqq.backend.core.model.tables.QQQTable;
-import com.kingsrook.qqq.backend.core.model.tables.QQQTablesMetaDataProvider;
+import com.kingsrook.qqq.backend.core.model.tables.QQQTableTableManager;
 import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
 import com.kingsrook.qqq.backend.core.utils.PrefixedDefaultThreadFactory;
 import com.kingsrook.qqq.backend.core.utils.StringUtils;
-import com.kingsrook.qqq.backend.core.utils.collections.MapBuilder;
 import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
 
 
@@ -371,7 +365,7 @@ public class QueryStatManager
                   //////////////////////
                   // set the table id //
                   //////////////////////
-                  Integer qqqTableId = getQQQTableId(queryStat.getTableName());
+                  Integer qqqTableId = QQQTableTableManager.getQQQTableId(getInstance().qInstance, queryStat.getTableName());
                   queryStat.setQqqTableId(qqqTableId);
 
                   //////////////////////////////
@@ -382,7 +376,7 @@ public class QueryStatManager
                      List<QueryStatJoinTable> queryStatJoinTableList = new ArrayList<>();
                      for(String joinTableName : queryStat.getJoinTableNames())
                      {
-                        queryStatJoinTableList.add(new QueryStatJoinTable().withQqqTableId(getQQQTableId(joinTableName)));
+                        queryStatJoinTableList.add(new QueryStatJoinTable().withQqqTableId(QQQTableTableManager.getQQQTableId(getInstance().qInstance, joinTableName)));
                      }
                      queryStat.setQueryStatJoinTableList(queryStatJoinTableList);
                   }
@@ -460,7 +454,7 @@ public class QueryStatManager
                String[] parts = fieldName.split("\\.");
                if(parts.length > 1)
                {
-                  queryStatCriteriaField.setQqqTableId(getQQQTableId(parts[0]));
+                  queryStatCriteriaField.setQqqTableId(QQQTableTableManager.getQQQTableId(getInstance().qInstance, parts[0]));
                   queryStatCriteriaField.setName(parts[1]);
                }
             }
@@ -498,7 +492,7 @@ public class QueryStatManager
                   String[] parts = fieldName.split("\\.");
                   if(parts.length > 1)
                   {
-                     queryStatOrderByField.setQqqTableId(getQQQTableId(parts[0]));
+                     queryStatOrderByField.setQqqTableId(QQQTableTableManager.getQQQTableId(getInstance().qInstance, parts[0]));
                      queryStatOrderByField.setName(parts[1]);
                   }
                }
@@ -512,44 +506,6 @@ public class QueryStatManager
             }
          }
       }
-
-
-
-      /*******************************************************************************
-       **
-       *******************************************************************************/
-      private static Integer getQQQTableId(String tableName) throws QException
-      {
-         /////////////////////////////
-         // look in the cache table //
-         /////////////////////////////
-         GetInput getInput = new GetInput();
-         getInput.setTableName(QQQTablesMetaDataProvider.QQQ_TABLE_CACHE_TABLE_NAME);
-         getInput.setUniqueKey(MapBuilder.of("name", tableName));
-         GetOutput getOutput = new GetAction().execute(getInput);
-
-         ////////////////////////
-         // upon cache miss... //
-         ////////////////////////
-         if(getOutput.getRecord() == null)
-         {
-            ///////////////////////////////////////////////////////
-            // insert the record (into the table, not the cache) //
-            ///////////////////////////////////////////////////////
-            QTableMetaData tableMetaData = getInstance().qInstance.getTable(tableName);
-            InsertInput    insertInput   = new InsertInput();
-            insertInput.setTableName(QQQTable.TABLE_NAME);
-            insertInput.setRecords(List.of(new QRecord().withValue("name", tableName).withValue("label", tableMetaData.getLabel())));
-            InsertOutput insertOutput = new InsertAction().execute(insertInput);
-
-            ///////////////////////////////////
-            // repeat the get from the cache //
-            ///////////////////////////////////
-            getOutput = new GetAction().execute(getInput);
-         }
-
-         return getOutput.getRecord().getValueInteger("id");
-      }
    }
 
 

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/helpers/UniqueKeyHelper.java

@@ -50,6 +50,7 @@ import com.kingsrook.qqq.backend.core.utils.ValueUtils;
  *******************************************************************************/
 public class UniqueKeyHelper
 {
+   private static Integer pageSize = 1000;
 
    /*******************************************************************************
     **
@@ -60,62 +61,71 @@ public class UniqueKeyHelper
       Map<List<Serializable>, Serializable> existingRecords = new HashMap<>();
       if(ukFieldNames != null)
       {
-         QueryInput queryInput = new QueryInput();
-         queryInput.setTableName(table.getName());
-         queryInput.setTransaction(transaction);
+         for(List<QRecord> page : CollectionUtils.getPages(recordList, pageSize))
+         {
+            QueryInput queryInput = new QueryInput();
+            queryInput.setTableName(table.getName());
+            queryInput.setTransaction(transaction);
 
-         QQueryFilter filter = new QQueryFilter();
-         if(ukFieldNames.size() == 1)
-         {
-            List<Serializable> values = recordList.stream()
-               .filter(r -> CollectionUtils.nullSafeIsEmpty(r.getErrors()))
-               .map(r -> r.getValue(ukFieldNames.get(0)))
-               .collect(Collectors.toList());
-            filter.addCriteria(new QFilterCriteria(ukFieldNames.get(0), QCriteriaOperator.IN, values));
-         }
-         else
-         {
-            filter.setBooleanOperator(QQueryFilter.BooleanOperator.OR);
-            for(QRecord record : recordList)
+            QQueryFilter filter = new QQueryFilter();
+            if(ukFieldNames.size() == 1)
             {
-               if(CollectionUtils.nullSafeHasContents(record.getErrors()))
+               List<Serializable> values = page.stream()
+                  .filter(r -> CollectionUtils.nullSafeIsEmpty(r.getErrors()))
+                  .map(r -> r.getValue(ukFieldNames.get(0)))
+                  .collect(Collectors.toList());
+
+               if(values.isEmpty())
                {
                   continue;
                }
 
-               QQueryFilter subFilter = new QQueryFilter();
-               filter.addSubFilter(subFilter);
-               for(String fieldName : ukFieldNames)
+               filter.addCriteria(new QFilterCriteria(ukFieldNames.get(0), QCriteriaOperator.IN, values));
+            }
+            else
+            {
+               filter.setBooleanOperator(QQueryFilter.BooleanOperator.OR);
+               for(QRecord record : page)
                {
-                  Serializable value = record.getValue(fieldName);
-                  if(value == null)
+                  if(CollectionUtils.nullSafeHasContents(record.getErrors()))
                   {
-                     subFilter.addCriteria(new QFilterCriteria(fieldName, QCriteriaOperator.IS_BLANK));
+                     continue;
                   }
-                  else
+
+                  QQueryFilter subFilter = new QQueryFilter();
+                  filter.addSubFilter(subFilter);
+                  for(String fieldName : ukFieldNames)
                   {
-                     subFilter.addCriteria(new QFilterCriteria(fieldName, QCriteriaOperator.EQUALS, value));
+                     Serializable value = record.getValue(fieldName);
+                     if(value == null)
+                     {
+                        subFilter.addCriteria(new QFilterCriteria(fieldName, QCriteriaOperator.IS_BLANK));
+                     }
+                     else
+                     {
+                        subFilter.addCriteria(new QFilterCriteria(fieldName, QCriteriaOperator.EQUALS, value));
+                     }
                   }
                }
+
+               if(CollectionUtils.nullSafeIsEmpty(filter.getSubFilters()))
+               {
+                  ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+                  // if we didn't build any sub-filters (because all records have errors in them), don't run a query w/ no clauses - continue to next page //
+                  ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+                  continue;
+               }
             }
 
-            if(CollectionUtils.nullSafeIsEmpty(filter.getSubFilters()))
+            queryInput.setFilter(filter);
+            QueryOutput queryOutput = new QueryAction().execute(queryInput);
+            for(QRecord record : queryOutput.getRecords())
             {
-               ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-               // if we didn't build any sub-filters (because all records have errors in them), don't run a query w/ no clauses - rather - return early. //
-               ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-               return (existingRecords);
-            }
-         }
-
-         queryInput.setFilter(filter);
-         QueryOutput queryOutput = new QueryAction().execute(queryInput);
-         for(QRecord record : queryOutput.getRecords())
-         {
-            Optional<List<Serializable>> keyValues = getKeyValues(table, uniqueKey, record, allowNullKeyValuesToEqual);
-            if(keyValues.isPresent())
-            {
-               existingRecords.put(keyValues.get(), record.getValue(table.getPrimaryKeyField()));
+               Optional<List<Serializable>> keyValues = getKeyValues(table, uniqueKey, record, allowNullKeyValuesToEqual);
+               if(keyValues.isPresent())
+               {
+                  existingRecords.put(keyValues.get(), record.getValue(table.getPrimaryKeyField()));
+               }
             }
          }
       }
@@ -200,4 +210,26 @@ public class UniqueKeyHelper
       }
    }
 
+
+
+   /*******************************************************************************
+    ** Getter for pageSize
+    **
+    *******************************************************************************/
+   public static Integer getPageSize()
+   {
+      return pageSize;
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for pageSize
+    **
+    *******************************************************************************/
+   public static void setPageSize(Integer pageSize)
+   {
+      UniqueKeyHelper.pageSize = pageSize;
+   }
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/helpers/ValidateRecordSecurityLockHelper.java

@@ -28,6 +28,7 @@ import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import com.kingsrook.qqq.backend.core.actions.QBackendTransaction;
 import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
 import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
@@ -83,7 +84,7 @@ public class ValidateRecordSecurityLockHelper
    /*******************************************************************************
     **
     *******************************************************************************/
-   public static void validateSecurityFields(QTableMetaData table, List<QRecord> records, Action action) throws QException
+   public static void validateSecurityFields(QTableMetaData table, List<QRecord> records, Action action, QBackendTransaction transaction) throws QException
    {
       MultiRecordSecurityLock locksToCheck = getRecordSecurityLocks(table, action);
       if(locksToCheck == null || CollectionUtils.nullSafeIsEmpty(locksToCheck.getLocks()))
@@ -101,7 +102,7 @@ public class ValidateRecordSecurityLockHelper
       // actually check lock values //
       ////////////////////////////////
       Map<Serializable, RecordWithErrors> errorRecords = new HashMap<>();
-      evaluateRecordLocks(table, records, action, locksToCheck, errorRecords, new ArrayList<>(), madeUpPrimaryKeys);
+      evaluateRecordLocks(table, records, action, locksToCheck, errorRecords, new ArrayList<>(), madeUpPrimaryKeys, transaction);
 
       /////////////////////////////////
       // propagate errors to records //
@@ -141,7 +142,7 @@ public class ValidateRecordSecurityLockHelper
     ** BUT - WRITE locks - in their case, we read the record no matter what, and in
     ** here we need to verify we have a key that allows us to WRITE the record.
     *******************************************************************************/
-   private static void evaluateRecordLocks(QTableMetaData table, List<QRecord> records, Action action, RecordSecurityLock recordSecurityLock, Map<Serializable, RecordWithErrors> errorRecords, List<Integer> treePosition, Map<Serializable, QRecord> madeUpPrimaryKeys) throws QException
+   private static void evaluateRecordLocks(QTableMetaData table, List<QRecord> records, Action action, RecordSecurityLock recordSecurityLock, Map<Serializable, RecordWithErrors> errorRecords, List<Integer> treePosition, Map<Serializable, QRecord> madeUpPrimaryKeys, QBackendTransaction transaction) throws QException
    {
       if(recordSecurityLock instanceof MultiRecordSecurityLock multiRecordSecurityLock)
       {
@@ -152,7 +153,7 @@ public class ValidateRecordSecurityLockHelper
          for(RecordSecurityLock childLock : CollectionUtils.nonNullList(multiRecordSecurityLock.getLocks()))
          {
             treePosition.add(i);
-            evaluateRecordLocks(table, records, action, childLock, errorRecords, treePosition, madeUpPrimaryKeys);
+            evaluateRecordLocks(table, records, action, childLock, errorRecords, treePosition, madeUpPrimaryKeys, transaction);
             treePosition.remove(treePosition.size() - 1);
             i++;
          }
@@ -225,6 +226,7 @@ public class ValidateRecordSecurityLockHelper
             // query will be like (fkey1=? and fkey2=?) OR (fkey1=? and fkey2=?) OR (fkey1=? and fkey2=?) //
             ////////////////////////////////////////////////////////////////////////////////////////////////
             QueryInput queryInput = new QueryInput();
+            queryInput.setTransaction(transaction);
             queryInput.setTableName(leftMostJoin.getLeftTable());
             QQueryFilter filter = new QQueryFilter().withBooleanOperator(QQueryFilter.BooleanOperator.OR);
             queryInput.setFilter(filter);

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/templates/RenderTemplateAction.java

@@ -104,10 +104,21 @@ public class RenderTemplateAction extends AbstractQActionFunction<RenderTemplate
 
 
 
+   /*******************************************************************************
+    ** Static wrapper to render a Velocity template.
+    *******************************************************************************/
+   @Deprecated(since = "Call the version that doesn't take an ActionInput")
+   public static String renderVelocity(AbstractActionInput parentActionInput, Map<String, Object> context, String code) throws QException
+   {
+      return (renderVelocity(context, code));
+   }
+
+
+
    /*******************************************************************************
     ** Most convenient static wrapper to render a Velocity template.
     *******************************************************************************/
-   public static String renderVelocity(AbstractActionInput parentActionInput, Map<String, Object> context, String code) throws QException
+   public static String renderVelocity(Map<String, Object> context, String code) throws QException
    {
       return (render(TemplateType.VELOCITY, context, code));
    }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/QCustomPossibleValueProvider.java

@@ -24,6 +24,7 @@ package com.kingsrook.qqq.backend.core.actions.values;
 
 import java.io.Serializable;
 import java.util.ArrayList;
+import java.util.Comparator;
 import java.util.List;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.model.actions.values.SearchPossibleValueSourceInput;
@@ -74,6 +75,31 @@ public interface QCustomPossibleValueProvider<T extends Serializable>
    }
 
 
+   /***************************************************************************
+    ** meant to be protected (but interface...) - for a custom PVS implementation
+    ** to complete its search (e.g., after it generates the list of PVS objects,
+    ** let this method do the filtering).
+    ***************************************************************************/
+   default List<QPossibleValue<T>> completeCustomPVSSearch(SearchPossibleValueSourceInput input, List<QPossibleValue<T>> possibleValues)
+   {
+      SearchPossibleValueSourceAction.PreparedSearchPossibleValueSourceInput preparedInput = SearchPossibleValueSourceAction.prepareSearchPossibleValueSourceInput(input);
+
+      List<QPossibleValue<T>> rs = new ArrayList<>();
+
+      for(QPossibleValue<T> possibleValue : possibleValues)
+      {
+         if(possibleValue != null && SearchPossibleValueSourceAction.doesPossibleValueMatchSearchInput(possibleValue, preparedInput))
+         {
+            rs.add(possibleValue);
+         }
+      }
+
+      rs.sort(Comparator.nullsLast(Comparator.comparing((QPossibleValue<T> pv) -> pv.getLabel())));
+
+      return (rs);
+   }
+
+
    /*******************************************************************************
     **
     *******************************************************************************/

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/QValueFormatter.java

@@ -45,6 +45,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
 import com.kingsrook.qqq.backend.core.utils.StringUtils;
 import com.kingsrook.qqq.backend.core.utils.ValueUtils;
+import org.apache.commons.lang3.BooleanUtils;
 import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
 
 
@@ -468,7 +469,8 @@ public class QValueFormatter
    {
       for(QFieldMetaData field : table.getFields().values())
       {
-         if(field.getType().equals(QFieldType.BLOB))
+         Optional<FieldAdornment> fileDownloadAdornment = field.getAdornment(AdornmentType.FILE_DOWNLOAD);
+         if(fileDownloadAdornment.isPresent())
          {
             //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
             // file name comes from:                                                                                            //
@@ -478,25 +480,14 @@ public class QValueFormatter
             // - tableLabel primaryKey fieldLabel                                                                               //
             // - and - if the FILE_DOWNLOAD adornment had a DEFAULT_EXTENSION, then it gets added (preceded by a dot)           //
             //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-            Optional<FieldAdornment>  fileDownloadAdornment = field.getAdornment(AdornmentType.FILE_DOWNLOAD);
-            Map<String, Serializable> adornmentValues       = Collections.emptyMap();
-
-            if(fileDownloadAdornment.isPresent())
-            {
-               adornmentValues = fileDownloadAdornment.get().getValues();
-            }
-            else
-            {
-               ///////////////////////////////////////////////////////
-               // don't change blobs unless they are file-downloads //
-               ///////////////////////////////////////////////////////
-               continue;
-            }
+            Map<String, Serializable> adornmentValues = fileDownloadAdornment.get().getValues();
 
             String fileNameField    = ValueUtils.getValueAsString(adornmentValues.get(AdornmentType.FileDownloadValues.FILE_NAME_FIELD));
             String fileNameFormat   = ValueUtils.getValueAsString(adornmentValues.get(AdornmentType.FileDownloadValues.FILE_NAME_FORMAT));
             String defaultExtension = ValueUtils.getValueAsString(adornmentValues.get(AdornmentType.FileDownloadValues.DEFAULT_EXTENSION));
 
+            Boolean downloadUrlDynamic = ValueUtils.getValueAsBoolean(adornmentValues.get(AdornmentType.FileDownloadValues.DOWNLOAD_URL_DYNAMIC));
+
             for(QRecord record : records)
             {
                if(!doesFieldHaveValue(field, record))
@@ -504,6 +495,11 @@ public class QValueFormatter
                   continue;
                }
 
+               if(BooleanUtils.isTrue(downloadUrlDynamic))
+               {
+                  continue;
+               }
+
                Serializable primaryKey = record.getValue(table.getPrimaryKeyField());
                String       fileName   = null;
 
@@ -521,7 +517,7 @@ public class QValueFormatter
                   {
                      @SuppressWarnings("unchecked") // instance validation should make this safe!
                      List<String> fileNameFormatFields = (List<String>) adornmentValues.get(AdornmentType.FileDownloadValues.FILE_NAME_FORMAT_FIELDS);
-                     List<String> values = fileNameFormatFields.stream().map(f -> ValueUtils.getValueAsString(record.getValue(f))).toList();
+                     List<String> values = CollectionUtils.nullSafeHasContents(fileNameFormatFields) ? fileNameFormatFields.stream().map(f -> ValueUtils.getValueAsString(record.getValue(f))).toList() : Collections.emptyList();
                      fileName = QValueFormatter.formatStringWithValues(fileNameFormat, values);
                   }
                }
@@ -542,7 +538,19 @@ public class QValueFormatter
                   }
                }
 
-               record.setValue(field.getName(), "/data/" + table.getName() + "/" + primaryKey + "/" + field.getName() + "/" + fileName);
+               ////////////////////////////////////////////////////////////////////////////////////////////////
+               // if field type is blob OR if there's a supplemental process or code-ref that needs to run - //
+               // then update its value to be a callback-url that'll give access to the bytes to download.   //
+               // implied here is that a String value (w/o supplemental code/proc) has its value stay as a   //
+               // URL, which is where the file is directly downloaded from.  And in the case of a String     //
+               // with code-to-run, then the code should run, followed by a redirect to the value URL.       //
+               ////////////////////////////////////////////////////////////////////////////////////////////////
+               if(QFieldType.BLOB.equals(field.getType())
+                  || adornmentValues.containsKey(AdornmentType.FileDownloadValues.SUPPLEMENTAL_CODE_REFERENCE)
+                  || adornmentValues.containsKey(AdornmentType.FileDownloadValues.SUPPLEMENTAL_PROCESS_NAME))
+               {
+                  record.setValue(field.getName(), AdornmentType.FileDownloadValues.makeFieldDownloadUrl(table.getName(), primaryKey, field.getName(), fileName));
+               }
                record.setDisplayValue(field.getName(), fileName);
             }
          }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/SearchPossibleValueSourceAction.java

@@ -26,8 +26,13 @@ import java.io.Serializable;
 import java.time.Instant;
 import java.time.LocalDate;
 import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Objects;
+import java.util.Set;
+import java.util.stream.Collectors;
 import com.kingsrook.qqq.backend.core.actions.customizers.QCodeLoader;
 import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
 import com.kingsrook.qqq.backend.core.context.QContext;
@@ -47,10 +52,9 @@ import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleVal
 import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSource;
 import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSourceType;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
-import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
 import com.kingsrook.qqq.backend.core.utils.StringUtils;
 import com.kingsrook.qqq.backend.core.utils.ValueUtils;
-import org.apache.commons.lang.NotImplementedException;
+import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
 
 
 /*******************************************************************************
@@ -61,6 +65,9 @@ public class SearchPossibleValueSourceAction
 {
    private static final QLogger LOG = QLogger.getLogger(SearchPossibleValueSourceAction.class);
 
+   private static final Set<String> warnedAboutUnexpectedValueField                = Collections.synchronizedSet(new HashSet<>());
+   private static final Set<String> warnedAboutUnexpectedNoOfFieldsToSearchByLabel = Collections.synchronizedSet(new HashSet<>());
+
    private QPossibleValueTranslator possibleValueTranslator;
 
 
@@ -101,47 +108,54 @@ public class SearchPossibleValueSourceAction
 
 
 
+   /***************************************************************************
+    ** record to store "computed" values as part of a possible-value search -
+    ** e.g., ids type-convered, and lower-cased labels.
+    ***************************************************************************/
+   public record PreparedSearchPossibleValueSourceInput(Collection<?> inputIdsAsCorrectType, Collection<String> lowerCaseLabels, String searchTerm) {}
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   public static PreparedSearchPossibleValueSourceInput prepareSearchPossibleValueSourceInput(SearchPossibleValueSourceInput input)
+   {
+      QPossibleValueSource possibleValueSource   = QContext.getQInstance().getPossibleValueSource(input.getPossibleValueSourceName());
+      List<?>              inputIdsAsCorrectType = convertInputIdsToPossibleValueSourceIdType(possibleValueSource, input.getIdList());
+
+      Set<String> lowerCaseLabels = null;
+      if(input.getLabelList() != null)
+      {
+         lowerCaseLabels = input.getLabelList().stream()
+            .filter(Objects::nonNull)
+            .map(l -> l.toLowerCase())
+            .collect(Collectors.toSet());
+      }
+
+      return (new PreparedSearchPossibleValueSourceInput(inputIdsAsCorrectType, lowerCaseLabels, input.getSearchTerm()));
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/
    private SearchPossibleValueSourceOutput searchPossibleValueEnum(SearchPossibleValueSourceInput input, QPossibleValueSource possibleValueSource)
    {
+      PreparedSearchPossibleValueSourceInput preparedSearchPossibleValueSourceInput = prepareSearchPossibleValueSourceInput(input);
+
       SearchPossibleValueSourceOutput output      = new SearchPossibleValueSourceOutput();
       List<Serializable>              matchingIds = new ArrayList<>();
 
-      List<?> inputIdsAsCorrectType = convertInputIdsToEnumIdType(possibleValueSource, input.getIdList());
-
       for(QPossibleValue<?> possibleValue : possibleValueSource.getEnumValues())
       {
-         boolean match = false;
-
-         if(input.getIdList() != null)
-         {
-            if(inputIdsAsCorrectType.contains(possibleValue.getId()))
-            {
-               match = true;
-            }
-         }
-         else
-         {
-            if(StringUtils.hasContent(input.getSearchTerm()))
-            {
-               match = (Objects.equals(ValueUtils.getValueAsString(possibleValue.getId()).toLowerCase(), input.getSearchTerm().toLowerCase())
-                  || possibleValue.getLabel().toLowerCase().startsWith(input.getSearchTerm().toLowerCase()));
-            }
-            else
-            {
-               match = true;
-            }
-         }
+         boolean match = doesPossibleValueMatchSearchInput(possibleValue, preparedSearchPossibleValueSourceInput);
 
          if(match)
          {
-            matchingIds.add((Serializable) possibleValue.getId());
+            matchingIds.add(possibleValue.getId());
          }
-
-         // todo - skip & limit?
-         // todo - default filter
       }
 
       List<QPossibleValue<?>> qPossibleValues = possibleValueTranslator.buildTranslatedPossibleValueList(possibleValueSource, matchingIds);
@@ -152,37 +166,95 @@ public class SearchPossibleValueSourceAction
 
 
 
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   public static boolean doesPossibleValueMatchSearchInput(QPossibleValue<?> possibleValue, PreparedSearchPossibleValueSourceInput input)
+   {
+      boolean match = false;
+
+      if(input.inputIdsAsCorrectType() != null)
+      {
+         if(input.inputIdsAsCorrectType().contains(possibleValue.getId()))
+         {
+            match = true;
+         }
+      }
+      else if(input.lowerCaseLabels() != null)
+      {
+         if(input.lowerCaseLabels().contains(possibleValue.getLabel().toLowerCase()))
+         {
+            match = true;
+         }
+      }
+      else
+      {
+         if(StringUtils.hasContent(input.searchTerm()))
+         {
+            match = (Objects.equals(ValueUtils.getValueAsString(possibleValue.getId()).toLowerCase(), input.searchTerm().toLowerCase())
+               || possibleValue.getLabel().toLowerCase().startsWith(input.searchTerm().toLowerCase()));
+         }
+         else
+         {
+            match = true;
+         }
+      }
+      return match;
+   }
+
+
+
    /*******************************************************************************
     ** The input list of ids might come through as a type that isn't the same as
     ** the type of the ids in the enum (e.g., strings from a frontend, integers
-    ** in an enum).  So, this method looks at the first id in the enum, and then
-    ** maps all the inputIds to be of the same type.
+    ** in an enum).  So, this method type-converts them.
     *******************************************************************************/
-   private List<Object> convertInputIdsToEnumIdType(QPossibleValueSource possibleValueSource, List<Serializable> inputIdList)
+   private static List<Object> convertInputIdsToPossibleValueSourceIdType(QPossibleValueSource possibleValueSource, List<Serializable> inputIdList)
    {
       List<Object> rs = new ArrayList<>();
-      if(CollectionUtils.nullSafeIsEmpty(inputIdList))
+
+      if(inputIdList == null)
+      {
+         return (null);
+      }
+      else if(inputIdList.isEmpty())
       {
          return (rs);
       }
 
-      Object anIdFromTheEnum = possibleValueSource.getEnumValues().get(0).getId();
+      QFieldType type = possibleValueSource.getIdType();
 
-      if(anIdFromTheEnum instanceof Integer)
+      for(Serializable inputId : inputIdList)
       {
-         inputIdList.forEach(id -> rs.add(ValueUtils.getValueAsInteger(id)));
-      }
-      else if(anIdFromTheEnum instanceof String)
-      {
-         inputIdList.forEach(id -> rs.add(ValueUtils.getValueAsString(id)));
-      }
-      else if(anIdFromTheEnum instanceof Boolean)
-      {
-         inputIdList.forEach(id -> rs.add(ValueUtils.getValueAsBoolean(id)));
-      }
-      else
-      {
-         LOG.warn("Unexpected type [" + anIdFromTheEnum.getClass().getSimpleName() + "] for ids in enum: " + possibleValueSource.getName());
+         Object properlyTypedId = null;
+         try
+         {
+            if(type.equals(QFieldType.INTEGER))
+            {
+               properlyTypedId = ValueUtils.getValueAsInteger(inputId);
+            }
+            else if(type.isStringLike())
+            {
+               properlyTypedId = ValueUtils.getValueAsString(inputId);
+            }
+            else if(type.equals(QFieldType.BOOLEAN))
+            {
+               properlyTypedId = ValueUtils.getValueAsBoolean(inputId);
+            }
+            else
+            {
+               LOG.warn("Unexpected type [" + type + "] for ids in enum: " + possibleValueSource.getName());
+            }
+         }
+         catch(Exception e)
+         {
+            LOG.debug("Error converting possible value id to expected id type", e, logPair("value", inputId));
+         }
+
+         if(properlyTypedId != null)
+         {
+            rs.add(properlyTypedId);
+         }
       }
 
       return (rs);
@@ -209,6 +281,53 @@ public class SearchPossibleValueSourceAction
       {
          queryFilter.addCriteria(new QFilterCriteria(table.getPrimaryKeyField(), QCriteriaOperator.IN, input.getIdList()));
       }
+      else if(input.getLabelList() != null)
+      {
+         List<String> fieldNames = new ArrayList<>();
+
+         ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+         // the 'value fields' will either be 'id' or 'label' (which means, use the fields from the tableMetaData's label fields) //
+         ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+         for(String valueField : possibleValueSource.getValueFields())
+         {
+            if("id".equals(valueField))
+            {
+               fieldNames.add(table.getPrimaryKeyField());
+            }
+            else if("label".equals(valueField))
+            {
+               if(table.getRecordLabelFields() != null)
+               {
+                  fieldNames.addAll(table.getRecordLabelFields());
+               }
+            }
+            else
+            {
+               String message = "Unexpected valueField defined in possibleValueSource when searching possibleValueSource by label (required: 'id' or 'label')";
+               if(!warnedAboutUnexpectedValueField.contains(possibleValueSource.getName()))
+               {
+                  LOG.warn(message, logPair("valueField", valueField), logPair("possibleValueSource", possibleValueSource.getName()));
+                  warnedAboutUnexpectedValueField.add(possibleValueSource.getName());
+               }
+               output.setWarning(message);
+            }
+         }
+
+         if(fieldNames.size() == 1)
+         {
+            queryFilter.addCriteria(new QFilterCriteria(fieldNames.get(0), QCriteriaOperator.IN, input.getLabelList()));
+         }
+         else
+         {
+            String message = "Unexpected number of fields found for searching possibleValueSource by label (required: 1, found: " + fieldNames.size() + ")";
+            if(!warnedAboutUnexpectedNoOfFieldsToSearchByLabel.contains(possibleValueSource.getName()))
+            {
+               LOG.warn(message);
+               warnedAboutUnexpectedNoOfFieldsToSearchByLabel.add(possibleValueSource.getName());
+            }
+            output.setWarning(message);
+         }
+      }
       else
       {
          String searchTerm = input.getSearchTerm();
@@ -269,8 +388,8 @@ public class SearchPossibleValueSourceAction
          queryFilter = input.getDefaultQueryFilter();
       }
 
-      // todo - skip & limit as params
-      queryFilter.setLimit(250);
+      queryFilter.setLimit(input.getLimit());
+      queryFilter.setSkip(input.getSkip());
 
       queryFilter.setOrderBys(possibleValueSource.getOrderByFields());
 
@@ -288,7 +407,7 @@ public class SearchPossibleValueSourceAction
          fieldName = table.getPrimaryKeyField();
       }
 
-      List<Serializable> ids = queryOutput.getRecords().stream().map(r -> r.getValue(fieldName)).toList();
+      List<Serializable>      ids             = queryOutput.getRecords().stream().map(r -> r.getValue(fieldName)).toList();
       List<QPossibleValue<?>> qPossibleValues = possibleValueTranslator.buildTranslatedPossibleValueList(possibleValueSource, ids);
       output.setResults(qPossibleValues);
 
@@ -301,7 +420,7 @@ public class SearchPossibleValueSourceAction
     **
     *******************************************************************************/
    @SuppressWarnings({ "rawtypes", "unchecked" })
-   private SearchPossibleValueSourceOutput searchPossibleValueCustom(SearchPossibleValueSourceInput input, QPossibleValueSource possibleValueSource)
+   private SearchPossibleValueSourceOutput searchPossibleValueCustom(SearchPossibleValueSourceInput input, QPossibleValueSource possibleValueSource) throws QException
    {
       try
       {
@@ -314,11 +433,10 @@ public class SearchPossibleValueSourceAction
       }
       catch(Exception e)
       {
-         // LOG.warn("Error sending [" + value + "] for field [" + field + "] through custom code for PVS [" + field.getPossibleValueSourceName() + "]", e);
+         String message = "Error searching custom possible value source [" + input.getPossibleValueSourceName() + "]";
+         LOG.warn(message, e);
+         throw (new QException(message, e));
       }
-
-      throw new NotImplementedException("Not impleemnted");
-      // return (null);
    }
 
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/AbstractMetaDataProducerBasedQQQApplication.java

@@ -0,0 +1,55 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.instances;
+
+
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.metadata.MetaDataProducerHelper;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+
+
+/*******************************************************************************
+ ** Version of AbstractQQQApplication that assumes all meta-data is produced
+ ** by MetaDataProducers in a single package.
+ *******************************************************************************/
+public abstract class AbstractMetaDataProducerBasedQQQApplication extends AbstractQQQApplication
+{
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   public abstract String getMetaDataPackageName();
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   @Override
+   public QInstance defineQInstance() throws QException
+   {
+      QInstance qInstance = new QInstance();
+      MetaDataProducerHelper.processAllMetaDataProducersInPackage(qInstance, getMetaDataPackageName());
+      return (qInstance);
+   }
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/AbstractQQQApplication.java

@@ -0,0 +1,78 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.instances;
+
+
+import java.util.ArrayList;
+import java.util.List;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.exceptions.QInstanceValidationException;
+import com.kingsrook.qqq.backend.core.instances.validation.plugins.QInstanceValidatorPluginInterface;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
+
+
+/*******************************************************************************
+ ** Base class to provide the definition of a QQQ-based application.
+ **
+ ** Essentially, just how to define its meta-data - in the form of a QInstance.
+ **
+ ** Also provides means to define the instance validation plugins to be used.
+ *******************************************************************************/
+public abstract class AbstractQQQApplication
+{
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   public abstract QInstance defineQInstance() throws QException;
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   public QInstance defineValidatedQInstance() throws QException, QInstanceValidationException
+   {
+      QInstance qInstance = defineQInstance();
+
+      QInstanceValidator.removeAllValidatorPlugins();
+      for(QInstanceValidatorPluginInterface<?> validatorPlugin : CollectionUtils.nonNullList(getValidatorPlugins()))
+      {
+         QInstanceValidator.addValidatorPlugin(validatorPlugin);
+      }
+
+      QInstanceValidator qInstanceValidator = new QInstanceValidator();
+      qInstanceValidator.validate(qInstance);
+      return (qInstance);
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   protected List<QInstanceValidatorPluginInterface<?>> getValidatorPlugins()
+   {
+      return new ArrayList<>();
+   }
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java

@@ -23,7 +23,11 @@ package com.kingsrook.qqq.backend.core.instances;
 
 
 import java.io.Serializable;
+import java.lang.reflect.Method;
+import java.lang.reflect.ParameterizedType;
+import java.lang.reflect.Type;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
@@ -31,21 +35,27 @@ import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
+import java.util.Optional;
 import java.util.Set;
 import java.util.stream.Collectors;
+import com.kingsrook.qqq.backend.core.actions.customizers.QCodeLoader;
 import com.kingsrook.qqq.backend.core.actions.metadata.JoinGraph;
 import com.kingsrook.qqq.backend.core.actions.permissions.BulkTableActionProcessPermissionChecker;
+import com.kingsrook.qqq.backend.core.actions.values.QCustomPossibleValueProvider;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.instances.enrichment.plugins.QInstanceEnricherPluginInterface;
 import com.kingsrook.qqq.backend.core.logging.QLogger;
 import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
 import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
 import com.kingsrook.qqq.backend.core.model.metadata.dashboard.QWidgetMetaDataInterface;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.AdornmentType;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.AdornmentType.FileUploadAdornment;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.DynamicDefaultValueBehavior;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.FieldAdornment;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
+import com.kingsrook.qqq.backend.core.model.metadata.joins.QJoinMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppChildMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppSection;
@@ -54,10 +64,12 @@ import com.kingsrook.qqq.backend.core.model.metadata.permissions.MetaDataWithPer
 import com.kingsrook.qqq.backend.core.model.metadata.permissions.QPermissionRules;
 import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSource;
 import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSourceType;
+import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.processes.QComponentType;
 import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendComponentMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendStepMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.processes.QStateMachineStep;
 import com.kingsrook.qqq.backend.core.model.metadata.processes.QStepMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.processes.QSupplementalProcessMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportDataSource;
@@ -74,13 +86,20 @@ import com.kingsrook.qqq.backend.core.processes.implementations.bulk.edit.BulkEd
 import com.kingsrook.qqq.backend.core.processes.implementations.bulk.edit.BulkEditTransformStep;
 import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkInsertExtractStep;
 import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkInsertLoadStep;
+import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkInsertPrepareFileMappingStep;
+import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkInsertPrepareFileUploadStep;
+import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkInsertPrepareValueMappingStep;
+import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkInsertReceiveFileMappingStep;
+import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkInsertReceiveValueMappingStep;
 import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkInsertTransformStep;
 import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.ExtractViaQueryStep;
 import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess;
 import com.kingsrook.qqq.backend.core.scheduler.QScheduleManager;
+import com.kingsrook.qqq.backend.core.utils.ClassPathUtils;
 import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
 import com.kingsrook.qqq.backend.core.utils.ListingHash;
 import com.kingsrook.qqq.backend.core.utils.StringUtils;
+import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
 
 
 /*******************************************************************************
@@ -106,6 +125,8 @@ public class QInstanceEnricher
    //////////////////////////////////////////////////////////////////////////////////////////////////
    private static final Map<String, String> labelMappings = new LinkedHashMap<>();
 
+   private static ListingHash<Class<?>, QInstanceEnricherPluginInterface<?>> enricherPlugins = new ListingHash<>();
+
 
 
    /*******************************************************************************
@@ -167,6 +188,7 @@ public class QInstanceEnricher
       }
 
       enrichJoins();
+      enrichInstance();
 
       //////////////////////////////////////////////////////////////////////////////
       // if the instance DOES have 1 or more scheduler, but no schedulable types, //
@@ -183,6 +205,16 @@ public class QInstanceEnricher
 
 
 
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   private void enrichInstance()
+   {
+      runPlugins(QInstance.class, qInstance, qInstance);
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/
@@ -247,6 +279,14 @@ public class QInstanceEnricher
                }
             }
          }
+
+         ///////////////////////////////////////////
+         // run plugins on joins if there are any //
+         ///////////////////////////////////////////
+         for(QJoinMetaData join : qInstance.getJoins().values())
+         {
+            runPlugins(QJoinMetaData.class, join, qInstance);
+         }
       }
       catch(Exception e)
       {
@@ -262,6 +302,7 @@ public class QInstanceEnricher
    private void enrichWidget(QWidgetMetaDataInterface widgetMetaData)
    {
       enrichPermissionRules(widgetMetaData);
+      runPlugins(QWidgetMetaDataInterface.class, widgetMetaData, qInstance);
    }
 
 
@@ -272,6 +313,7 @@ public class QInstanceEnricher
    private void enrichBackend(QBackendMetaData qBackendMetaData)
    {
       qBackendMetaData.enrich();
+      runPlugins(QBackendMetaData.class, qBackendMetaData, qInstance);
    }
 
 
@@ -312,6 +354,7 @@ public class QInstanceEnricher
 
       enrichPermissionRules(table);
       enrichAuditRules(table);
+      runPlugins(QTableMetaData.class, table, qInstance);
    }
 
 
@@ -402,6 +445,7 @@ public class QInstanceEnricher
       }
 
       enrichPermissionRules(process);
+      runPlugins(QProcessMetaData.class, process, qInstance);
    }
 
 
@@ -410,10 +454,27 @@ public class QInstanceEnricher
     **
     *******************************************************************************/
    private void enrichStep(QStepMetaData step)
+   {
+      enrichStep(step, false);
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   private void enrichStep(QStepMetaData step, boolean isSubStep)
    {
       if(!StringUtils.hasContent(step.getLabel()))
       {
-         step.setLabel(nameToLabel(step.getName()));
+         if(isSubStep && (step.getName().endsWith(".backend") || step.getName().endsWith(".frontend")))
+         {
+            step.setLabel(nameToLabel(step.getName().replaceFirst("\\.(backend|frontend)", "")));
+         }
+         else
+         {
+            step.setLabel(nameToLabel(step.getName()));
+         }
       }
 
       step.getInputFields().forEach(this::enrichField);
@@ -434,6 +495,13 @@ public class QInstanceEnricher
             frontendStepMetaData.getRecordListFields().forEach(this::enrichField);
          }
       }
+      else if(step instanceof QStateMachineStep stateMachineStep)
+      {
+         for(QStepMetaData subStep : CollectionUtils.nonNullList(stateMachineStep.getSubSteps()))
+         {
+            enrichStep(subStep, true);
+         }
+      }
    }
 
 
@@ -499,6 +567,8 @@ public class QInstanceEnricher
             field.withBehavior(DynamicDefaultValueBehavior.MODIFY_DATE);
          }
       }
+
+      runPlugins(QFieldMetaData.class, field, qInstance);
    }
 
 
@@ -570,6 +640,7 @@ public class QInstanceEnricher
       ensureAppSectionMembersAreAppChildren(app);
 
       enrichPermissionRules(app);
+      runPlugins(QAppMetaData.class, app, qInstance);
    }
 
 
@@ -717,6 +788,7 @@ public class QInstanceEnricher
       }
 
       enrichPermissionRules(report);
+      runPlugins(QReportMetaData.class, report, qInstance);
    }
 
 
@@ -808,7 +880,7 @@ public class QInstanceEnricher
    /*******************************************************************************
     **
     *******************************************************************************/
-   private void defineTableBulkInsert(QInstance qInstance, QTableMetaData table, String processName)
+   public void defineTableBulkInsert(QInstance qInstance, QTableMetaData table, String processName)
    {
       Map<String, Serializable> values = new HashMap<>();
       values.put(StreamedETLWithFrontendProcess.FIELD_DESTINATION_TABLE, table.getName());
@@ -820,6 +892,7 @@ public class QInstanceEnricher
             values
          )
          .withName(processName)
+         .withIcon(new QIcon().withName("library_add"))
          .withLabel(table.getLabel() + " Bulk Insert")
          .withTableName(table.getName())
          .withIsHidden(true)
@@ -850,18 +923,76 @@ public class QInstanceEnricher
          .map(QFieldMetaData::getLabel)
          .collect(Collectors.joining(", "));
 
+      QBackendStepMetaData prepareFileUploadStep = new QBackendStepMetaData()
+         .withName("prepareFileUpload")
+         .withCode(new QCodeReference(BulkInsertPrepareFileUploadStep.class));
+
       QFrontendStepMetaData uploadScreen = new QFrontendStepMetaData()
          .withName("upload")
          .withLabel("Upload File")
-         .withFormField(new QFieldMetaData("theFile", QFieldType.BLOB).withLabel(table.getLabel() + " File").withIsRequired(true))
-         .withComponent(new QFrontendComponentMetaData()
-            .withType(QComponentType.HELP_TEXT)
-            .withValue("previewText", "file upload instructions")
-            .withValue("text", "Upload a CSV file with the following columns:\n" + fieldsForHelpText))
+         .withFormField(new QFieldMetaData("theFile", QFieldType.BLOB)
+            .withFieldAdornment(FileUploadAdornment.newFieldAdornment()
+               .withValue(FileUploadAdornment.formatDragAndDrop())
+               .withValue(FileUploadAdornment.widthFull()))
+            .withLabel(table.getLabel() + " File")
+            .withIsRequired(true))
+         .withComponent(new QFrontendComponentMetaData().withType(QComponentType.HTML))
          .withComponent(new QFrontendComponentMetaData().withType(QComponentType.EDIT_FORM));
 
-      process.addStep(0, uploadScreen);
-      process.getFrontendStep("review").setRecordListFields(editableFields);
+      QBackendStepMetaData prepareFileMappingStep = new QBackendStepMetaData()
+         .withName("prepareFileMapping")
+         .withCode(new QCodeReference(BulkInsertPrepareFileMappingStep.class));
+
+      QFrontendStepMetaData fileMappingScreen = new QFrontendStepMetaData()
+         .withName("fileMapping")
+         .withLabel("File Mapping")
+         .withBackStepName("prepareFileUpload")
+         .withComponent(new QFrontendComponentMetaData().withType(QComponentType.BULK_LOAD_FILE_MAPPING_FORM))
+         .withFormField(new QFieldMetaData("hasHeaderRow", QFieldType.BOOLEAN))
+         .withFormField(new QFieldMetaData("layout", QFieldType.STRING)); // is actually PVS, but, this field is only added to help support helpContent, so :shrug:
+
+      QBackendStepMetaData receiveFileMappingStep = new QBackendStepMetaData()
+         .withName("receiveFileMapping")
+         .withCode(new QCodeReference(BulkInsertReceiveFileMappingStep.class));
+
+      QBackendStepMetaData prepareValueMappingStep = new QBackendStepMetaData()
+         .withName("prepareValueMapping")
+         .withCode(new QCodeReference(BulkInsertPrepareValueMappingStep.class));
+
+      QFrontendStepMetaData valueMappingScreen = new QFrontendStepMetaData()
+         .withName("valueMapping")
+         .withLabel("Value Mapping")
+         .withBackStepName("prepareFileMapping")
+         .withComponent(new QFrontendComponentMetaData().withType(QComponentType.BULK_LOAD_VALUE_MAPPING_FORM));
+
+      QBackendStepMetaData receiveValueMappingStep = new QBackendStepMetaData()
+         .withName("receiveValueMapping")
+         .withCode(new QCodeReference(BulkInsertReceiveValueMappingStep.class));
+
+      int i = 0;
+      process.addStep(i++, prepareFileUploadStep);
+      process.addStep(i++, uploadScreen);
+
+      process.addStep(i++, prepareFileMappingStep);
+      process.addStep(i++, fileMappingScreen);
+      process.addStep(i++, receiveFileMappingStep);
+
+      process.addStep(i++, prepareValueMappingStep);
+      process.addStep(i++, valueMappingScreen);
+      process.addStep(i++, receiveValueMappingStep);
+
+      process.getFrontendStep(StreamedETLWithFrontendProcess.STEP_NAME_REVIEW).setRecordListFields(editableFields);
+
+      //////////////////////////////////////////////////////////////////////////////////////////
+      // put the bulk-load profile form (e.g., for saving it) on the review & result screens) //
+      //////////////////////////////////////////////////////////////////////////////////////////
+      process.getFrontendStep(StreamedETLWithFrontendProcess.STEP_NAME_REVIEW)
+         .withBackStepName("prepareFileMapping")
+         .getComponents().add(0, new QFrontendComponentMetaData().withType(QComponentType.BULK_LOAD_PROFILE_FORM));
+
+      process.getFrontendStep(StreamedETLWithFrontendProcess.STEP_NAME_RESULT)
+         .getComponents().add(0, new QFrontendComponentMetaData().withType(QComponentType.BULK_LOAD_PROFILE_FORM));
+
       qInstance.addProcess(process);
    }
 
@@ -1256,6 +1387,137 @@ public class QInstanceEnricher
             }
          }
 
+         if(possibleValueSource.getIdType() == null)
+         {
+            QTableMetaData table = qInstance.getTable(possibleValueSource.getTableName());
+            if(table != null)
+            {
+               String         primaryKeyField         = table.getPrimaryKeyField();
+               QFieldMetaData primaryKeyFieldMetaData = table.getFields().get(primaryKeyField);
+               if(primaryKeyFieldMetaData != null)
+               {
+                  possibleValueSource.setIdType(primaryKeyFieldMetaData.getType());
+               }
+            }
+         }
+      }
+      else if(QPossibleValueSourceType.ENUM.equals(possibleValueSource.getType()))
+      {
+         if(possibleValueSource.getIdType() == null)
+         {
+            if(CollectionUtils.nullSafeHasContents(possibleValueSource.getEnumValues()))
+            {
+               Object id = possibleValueSource.getEnumValues().get(0).getId();
+               try
+               {
+                  possibleValueSource.setIdType(QFieldType.fromClass(id.getClass()));
+               }
+               catch(Exception e)
+               {
+                  LOG.warn("Error enriching possible value source with idType based on first enum value", e, logPair("possibleValueSource", possibleValueSource.getName()), logPair("id", id));
+               }
+            }
+         }
+      }
+      else if(QPossibleValueSourceType.CUSTOM.equals(possibleValueSource.getType()))
+      {
+         if(possibleValueSource.getIdType() == null)
+         {
+            try
+            {
+               QCustomPossibleValueProvider<?> customPossibleValueProvider = QCodeLoader.getCustomPossibleValueProvider(possibleValueSource);
+
+               Method getPossibleValueMethod = customPossibleValueProvider.getClass().getDeclaredMethod("getPossibleValue", Serializable.class);
+               Type   returnType             = getPossibleValueMethod.getGenericReturnType();
+               Type   idType                 = ((ParameterizedType) returnType).getActualTypeArguments()[0];
+
+               if(idType instanceof Class<?> c)
+               {
+                  possibleValueSource.setIdType(QFieldType.fromClass(c));
+               }
+            }
+            catch(Exception e)
+            {
+               LOG.warn("Error enriching possible value source with idType based on first custom value", e, logPair("possibleValueSource", possibleValueSource.getName()));
+            }
+         }
+      }
+
+      runPlugins(QPossibleValueSource.class, possibleValueSource, qInstance);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public static void addEnricherPlugin(QInstanceEnricherPluginInterface<?> plugin)
+   {
+      Optional<Method> enrichMethod = Arrays.stream(plugin.getClass().getDeclaredMethods())
+         .filter(m -> m.getName().equals("enrich")
+            && m.getParameterCount() == 2
+            && !m.getParameterTypes()[0].equals(Object.class)
+            && m.getParameterTypes()[1].equals(QInstance.class)
+         ).findFirst();
+
+      if(enrichMethod.isPresent())
+      {
+         Class<?> parameterType = enrichMethod.get().getParameterTypes()[0];
+         enricherPlugins.add(parameterType, plugin);
+      }
+      else
+      {
+         LOG.warn("Could not find enrich method on enricher plugin [" + plugin.getClass().getName() + "] (to infer type being enriched) - this plugin will not be used.");
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public static void removeAllEnricherPlugins()
+   {
+      enricherPlugins.clear();
+   }
+
+
+
+   /***************************************************************************
+    ** scan the classpath for classes in the specified package name which
+    ** implement the QInstanceEnricherPluginInterface - any found get added
+    ***************************************************************************/
+   public static void discoverAndAddPluginsInPackage(String packageName) throws QException
+   {
+      try
+      {
+         for(Class<?> aClass : ClassPathUtils.getClassesInPackage(packageName))
+         {
+            if(QInstanceEnricherPluginInterface.class.isAssignableFrom(aClass))
+            {
+               QInstanceEnricherPluginInterface<?> plugin = (QInstanceEnricherPluginInterface<?>) aClass.getConstructor().newInstance();
+               addEnricherPlugin(plugin);
+            }
+         }
+      }
+      catch(Exception e)
+      {
+         throw (new QException("Error discovering and adding enricher plugins in package [" + packageName + "]", e));
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private <T> void runPlugins(Class<T> c, T t, QInstance qInstance)
+   {
+      for(QInstanceEnricherPluginInterface<?> plugin : CollectionUtils.nonNullList(enricherPlugins.get(c)))
+      {
+         @SuppressWarnings("unchecked")
+         QInstanceEnricherPluginInterface<T> castedPlugin = (QInstanceEnricherPluginInterface<T>) plugin;
+         castedPlugin.enrich(t, qInstance);
       }
    }
 

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceHelpContentManager.java

@@ -30,7 +30,6 @@ import java.util.Objects;
 import java.util.Optional;
 import java.util.Set;
 import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
-import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.logging.QLogger;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput;
@@ -111,7 +110,7 @@ public class QInstanceHelpContentManager
             }
             else
             {
-               LOG.info("Discarding help content with key that does not contain name:value format", logPair("key", key), logPair("id", record.getValue("id")));
+               LOG.info("Discarding help content with key-part that does not contain name:value format", logPair("key", key), logPair("part", part), logPair("id", record.getValue("id")));
             }
          }
 
@@ -150,19 +149,19 @@ public class QInstanceHelpContentManager
          ///////////////////////////////////////////////////////////////////////////////////
          if(StringUtils.hasContent(tableName))
          {
-            processHelpContentForTable(key, tableName, sectionName, fieldName, slotName, roles, helpContent);
+            processHelpContentForTable(qInstance, key, tableName, sectionName, fieldName, slotName, roles, helpContent);
          }
          else if(StringUtils.hasContent(processName))
          {
-            processHelpContentForProcess(key, processName, fieldName, stepName, roles, helpContent);
+            processHelpContentForProcess(qInstance, key, processName, fieldName, stepName, roles, helpContent);
          }
          else if(StringUtils.hasContent(widgetName))
          {
-            processHelpContentForWidget(key, widgetName, slotName, roles, helpContent);
+            processHelpContentForWidget(qInstance, key, widgetName, slotName, roles, helpContent);
          }
          else if(nameValuePairs.containsKey("instanceLevel"))
          {
-            processHelpContentForInstance(key, slotName, roles, helpContent);
+            processHelpContentForInstance(qInstance, key, slotName, roles, helpContent);
          }
       }
       catch(Exception e)
@@ -176,9 +175,9 @@ public class QInstanceHelpContentManager
    /*******************************************************************************
     **
     *******************************************************************************/
-   private static void processHelpContentForTable(String key, String tableName, String sectionName, String fieldName, String slotName, Set<HelpRole> roles, QHelpContent helpContent)
+   private static void processHelpContentForTable(QInstance qInstance, String key, String tableName, String sectionName, String fieldName, String slotName, Set<HelpRole> roles, QHelpContent helpContent)
    {
-      QTableMetaData table = QContext.getQInstance().getTable(tableName);
+      QTableMetaData table = qInstance.getTable(tableName);
       if(table == null)
       {
          LOG.info("Unrecognized table in help content", logPair("key", key));
@@ -246,9 +245,30 @@ public class QInstanceHelpContentManager
    /*******************************************************************************
     **
     *******************************************************************************/
-   private static void processHelpContentForProcess(String key, String processName, String fieldName, String stepName, Set<HelpRole> roles, QHelpContent helpContent)
+   private static void processHelpContentForProcess(QInstance qInstance, String key, String processName, String fieldName, String stepName, Set<HelpRole> roles, QHelpContent helpContent)
    {
-      QProcessMetaData process = QContext.getQInstance().getProcess(processName);
+      if(processName.startsWith("*") && processName.length() > 1)
+      {
+         boolean anyMatched = false;
+         String subName = processName.substring(1);
+         for(QProcessMetaData process : qInstance.getProcesses().values())
+         {
+            if(process.getName().endsWith(subName))
+            {
+               anyMatched = true;
+               processHelpContentForProcess(qInstance, key, process.getName(), fieldName, stepName, roles, helpContent);
+            }
+         }
+
+         if(!anyMatched)
+         {
+            LOG.info("Wildcard process name did not match any processes in help content", logPair("key", key));
+         }
+
+         return;
+      }
+
+      QProcessMetaData process = qInstance.getProcess(processName);
       if(process == null)
       {
          LOG.info("Unrecognized process in help content", logPair("key", key));
@@ -306,9 +326,9 @@ public class QInstanceHelpContentManager
    /*******************************************************************************
     **
     *******************************************************************************/
-   private static void processHelpContentForWidget(String key, String widgetName, String slotName, Set<HelpRole> roles, QHelpContent helpContent)
+   private static void processHelpContentForWidget(QInstance qInstance, String key, String widgetName, String slotName, Set<HelpRole> roles, QHelpContent helpContent)
    {
-      QWidgetMetaDataInterface widget = QContext.getQInstance().getWidget(widgetName);
+      QWidgetMetaDataInterface widget = qInstance.getWidget(widgetName);
       if(!StringUtils.hasContent(slotName))
       {
          LOG.info("Missing slot name in help content", logPair("key", key));
@@ -335,7 +355,7 @@ public class QInstanceHelpContentManager
    /*******************************************************************************
     **
     *******************************************************************************/
-   private static void processHelpContentForInstance(String key, String slotName, Set<HelpRole> roles, QHelpContent helpContent)
+   private static void processHelpContentForInstance(QInstance qInstance, String key, String slotName, Set<HelpRole> roles, QHelpContent helpContent)
    {
       if(!StringUtils.hasContent(slotName))
       {
@@ -345,11 +365,11 @@ public class QInstanceHelpContentManager
       {
          if(helpContent != null)
          {
-            QContext.getQInstance().withHelpContent(slotName, helpContent);
+            qInstance.withHelpContent(slotName, helpContent);
          }
          else
          {
-            QContext.getQInstance().removeHelpContent(slotName, roles);
+            qInstance.removeHelpContent(slotName, roles);
          }
       }
    }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidator.java

@@ -37,12 +37,15 @@ import java.util.Objects;
 import java.util.Optional;
 import java.util.Set;
 import java.util.TimeZone;
+import java.util.function.Function;
 import java.util.function.Supplier;
+import java.util.stream.Collectors;
 import java.util.stream.Stream;
 import com.kingsrook.qqq.backend.core.actions.automation.RecordAutomationHandler;
 import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizers;
 import com.kingsrook.qqq.backend.core.actions.dashboard.widgets.AbstractWidgetRenderer;
 import com.kingsrook.qqq.backend.core.actions.metadata.JoinGraph;
+import com.kingsrook.qqq.backend.core.actions.metadata.MetaDataFilterInterface;
 import com.kingsrook.qqq.backend.core.actions.processes.BackendStep;
 import com.kingsrook.qqq.backend.core.actions.reporting.customizers.ReportCustomRecordSourceInterface;
 import com.kingsrook.qqq.backend.core.actions.scripts.TestScriptActionInterface;
@@ -69,11 +72,13 @@ import com.kingsrook.qqq.backend.core.model.metadata.fields.FieldBehavior;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.ValueTooLongBehavior;
 import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinOn;
+import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinType;
 import com.kingsrook.qqq.backend.core.model.metadata.joins.QJoinMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppChildMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppSection;
 import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSource;
+import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSourceType;
 import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.processes.QStepMetaData;
@@ -105,12 +110,16 @@ import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.Automatio
 import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.QTableAutomationDetails;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.cache.CacheOf;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.cache.CacheUseCase;
+import com.kingsrook.qqq.backend.core.model.metadata.variants.BackendVariantSetting;
+import com.kingsrook.qqq.backend.core.model.metadata.variants.BackendVariantsConfig;
 import com.kingsrook.qqq.backend.core.modules.authentication.QAuthenticationModuleCustomizerInterface;
 import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
 import com.kingsrook.qqq.backend.core.utils.ListingHash;
 import com.kingsrook.qqq.backend.core.utils.StringUtils;
 import com.kingsrook.qqq.backend.core.utils.ValueUtils;
+import com.kingsrook.qqq.backend.core.utils.lambdas.UnsafeFunction;
 import com.kingsrook.qqq.backend.core.utils.lambdas.UnsafeLambda;
+import org.apache.commons.lang.BooleanUtils;
 import org.quartz.CronExpression;
 import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
 
@@ -133,6 +142,8 @@ public class QInstanceValidator
 
    private static ListingHash<Class<?>, QInstanceValidatorPluginInterface<?>> validatorPlugins = new ListingHash<>();
 
+   private JoinGraph joinGraph = null;
+
    private List<String> errors = new ArrayList<>();
 
 
@@ -160,8 +171,7 @@ public class QInstanceValidator
       // the enricher will build a join graph (if there are any joins).  we'd like to only do that       //
       // once, during the enrichment/validation work, so, capture it, and store it back in the instance. //
       /////////////////////////////////////////////////////////////////////////////////////////////////////
-      JoinGraph joinGraph = null;
-      long      start     = System.currentTimeMillis();
+      long start = System.currentTimeMillis();
       try
       {
          /////////////////////////////////////////////////////////////////////////////////////////////////
@@ -170,7 +180,7 @@ public class QInstanceValidator
          // TODO - possible point of customization (use a different enricher, or none, or pass it options).
          QInstanceEnricher qInstanceEnricher = new QInstanceEnricher(qInstance);
          qInstanceEnricher.enrich();
-         joinGraph = qInstanceEnricher.getJoinGraph();
+         this.joinGraph = qInstanceEnricher.getJoinGraph();
       }
       catch(Exception e)
       {
@@ -185,6 +195,7 @@ public class QInstanceValidator
       //////////////////////////////////////////////////////////////////////////
       try
       {
+         validateInstanceAttributes(qInstance);
          validateBackends(qInstance);
          validateAuthentication(qInstance);
          validateAutomationProviders(qInstance);
@@ -225,6 +236,19 @@ public class QInstanceValidator
 
 
 
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   private void validateInstanceAttributes(QInstance qInstance)
+   {
+      if(qInstance.getMetaDataFilter() != null)
+      {
+         validateSimpleCodeReference("Instance metaDataFilter ", qInstance.getMetaDataFilter(), MetaDataFilterInterface.class);
+      }
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/
@@ -356,8 +380,8 @@ public class QInstanceValidator
          assertCondition(join.getType() != null, "Missing type for join: " + joinName);
          assertCondition(CollectionUtils.nullSafeHasContents(join.getJoinOns()), "Missing joinOns for join: " + joinName);
 
-         boolean leftTableExists  = assertCondition(qInstance.getTable(join.getLeftTable()) != null, "Left-table name " + join.getLeftTable() + " join " + joinName + " is not a defined table in this instance.");
-         boolean rightTableExists = assertCondition(qInstance.getTable(join.getRightTable()) != null, "Right-table name " + join.getRightTable() + " join " + joinName + " is not a defined table in this instance.");
+         boolean leftTableExists  = assertCondition(qInstance.getTable(join.getLeftTable()) != null, "Left-table name " + join.getLeftTable() + " in join " + joinName + " is not a defined table in this instance.");
+         boolean rightTableExists = assertCondition(qInstance.getTable(join.getRightTable()) != null, "Right-table name " + join.getRightTable() + " in join " + joinName + " is not a defined table in this instance.");
 
          for(JoinOn joinOn : CollectionUtils.nonNullList(join.getJoinOns()))
          {
@@ -526,6 +550,60 @@ public class QInstanceValidator
          {
             assertCondition(Objects.equals(backendName, backend.getName()), "Inconsistent naming for backend: " + backendName + "/" + backend.getName() + ".");
 
+            ///////////////////////
+            // validate variants //
+            ///////////////////////
+            BackendVariantsConfig backendVariantsConfig = backend.getBackendVariantsConfig();
+            if(BooleanUtils.isTrue(backend.getUsesVariants()))
+            {
+               if(assertCondition(backendVariantsConfig != null, "Missing backendVariantsConfig in backend [" + backendName + "] which is marked as usesVariants"))
+               {
+                  assertCondition(StringUtils.hasContent(backendVariantsConfig.getVariantTypeKey()), "Missing variantTypeKey in backendVariantsConfig in [" + backendName + "]");
+
+                  String         optionsTableName = backendVariantsConfig.getOptionsTableName();
+                  QTableMetaData optionsTable     = qInstance.getTable(optionsTableName);
+                  if(assertCondition(StringUtils.hasContent(optionsTableName), "Missing optionsTableName in backendVariantsConfig in [" + backendName + "]"))
+                  {
+                     if(assertCondition(optionsTable != null, "Unrecognized optionsTableName [" + optionsTableName + "] in backendVariantsConfig in [" + backendName + "]"))
+                     {
+                        QQueryFilter optionsFilter = backendVariantsConfig.getOptionsFilter();
+                        if(optionsFilter != null)
+                        {
+                           validateQueryFilter(qInstance, "optionsFilter in backendVariantsConfig in backend [" + backendName + "]: ", optionsTable, optionsFilter, null);
+                        }
+                     }
+                  }
+
+                  Map<BackendVariantSetting, String> backendSettingSourceFieldNameMap = backendVariantsConfig.getBackendSettingSourceFieldNameMap();
+                  if(assertCondition(CollectionUtils.nullSafeHasContents(backendSettingSourceFieldNameMap), "Missing or empty backendSettingSourceFieldNameMap in backendVariantsConfig in [" + backendName + "]"))
+                  {
+                     ///////////////////////////////////////////////////////////////////////////////////////////////////////////////
+                     // only validate field names in the backendSettingSourceFieldNameMap if there is NOT a variantRecordSupplier //
+                     // (the idea being, that the supplier might be building a record with fieldNames that aren't in the table... //
+                     ///////////////////////////////////////////////////////////////////////////////////////////////////////////////
+                     if(optionsTable != null && backendVariantsConfig.getVariantRecordLookupFunction() == null)
+                     {
+                        for(Map.Entry<BackendVariantSetting, String> entry : backendSettingSourceFieldNameMap.entrySet())
+                        {
+                           assertCondition(optionsTable.getFields().containsKey(entry.getValue()), "Unrecognized fieldName [" + entry.getValue() + "] in backendSettingSourceFieldNameMap in backendVariantsConfig in [" + backendName + "]");
+                        }
+                     }
+                  }
+
+                  if(backendVariantsConfig.getVariantRecordLookupFunction() != null)
+                  {
+                     validateSimpleCodeReference("VariantRecordSupplier in backendVariantsConfig in backend [" + backendName + "]: ", backendVariantsConfig.getVariantRecordLookupFunction(), UnsafeFunction.class, Function.class);
+                  }
+               }
+            }
+            else
+            {
+               assertCondition(backendVariantsConfig == null, "Should not have a backendVariantsConfig in backend [" + backendName + "] which is not marked as usesVariants");
+            }
+
+            ///////////////////////////////////////////
+            // let the backend do its own validation //
+            ///////////////////////////////////////////
             backend.performValidation(this);
 
             runPlugins(QBackendMetaData.class, backend, qInstance);
@@ -560,7 +638,7 @@ public class QInstanceValidator
    private void validateAuthentication(QInstance qInstance)
    {
       QAuthenticationMetaData authentication = qInstance.getAuthentication();
-      if(authentication != null)
+      if(assertCondition(authentication != null, "Authentication MetaData must be defined."))
       {
          if(authentication.getCustomizer() != null)
          {
@@ -763,15 +841,38 @@ public class QInstanceValidator
       {
          if(assertCondition(StringUtils.hasContent(association.getName()), "missing a name for an Association on table " + table.getName()))
          {
-            String messageSuffix = " for Association " + association.getName() + " on table " + table.getName();
+            String  messageSuffix   = " for Association " + association.getName() + " on table " + table.getName();
+            boolean recognizedTable = false;
             if(assertCondition(StringUtils.hasContent(association.getAssociatedTableName()), "missing associatedTableName" + messageSuffix))
             {
-               assertCondition(qInstance.getTable(association.getAssociatedTableName()) != null, "unrecognized associatedTableName " + association.getAssociatedTableName() + messageSuffix);
+               if(assertCondition(qInstance.getTable(association.getAssociatedTableName()) != null, "unrecognized associatedTableName " + association.getAssociatedTableName() + messageSuffix))
+               {
+                  recognizedTable = true;
+               }
             }
 
             if(assertCondition(StringUtils.hasContent(association.getJoinName()), "missing joinName" + messageSuffix))
             {
-               assertCondition(qInstance.getJoin(association.getJoinName()) != null, "unrecognized joinName " + association.getJoinName() + messageSuffix);
+               QJoinMetaData join = qInstance.getJoin(association.getJoinName());
+               if(assertCondition(join != null, "unrecognized joinName " + association.getJoinName() + messageSuffix))
+               {
+                  assert join != null; // covered by the assertCondition
+
+                  if(recognizedTable)
+                  {
+                     boolean isLeftToRight = join.getLeftTable().equals(table.getName()) && join.getRightTable().equals(association.getAssociatedTableName());
+                     boolean isRightToLeft = join.getRightTable().equals(table.getName()) && join.getLeftTable().equals(association.getAssociatedTableName());
+                     assertCondition(isLeftToRight || isRightToLeft, "join [" + association.getJoinName() + "] does not connect tables [" + table.getName() + "] and [" + association.getAssociatedTableName() + "]" + messageSuffix);
+                     if(isLeftToRight)
+                     {
+                        assertCondition(join.getType().equals(JoinType.ONE_TO_MANY) || join.getType().equals(JoinType.ONE_TO_ONE), "Join type does not have 'one' on this table's side side (left)" + messageSuffix);
+                     }
+                     else if(isRightToLeft)
+                     {
+                        assertCondition(join.getType().equals(JoinType.MANY_TO_ONE) || join.getType().equals(JoinType.ONE_TO_ONE), "Join type does not have 'one' on this table's side (right)" + messageSuffix);
+                     }
+                  }
+               }
             }
          }
       }
@@ -928,13 +1029,8 @@ public class QInstanceValidator
       assertCondition(Objects.equals(fieldName, field.getName()),
          "Inconsistent naming in table " + tableName + " for field " + fieldName + "/" + field.getName() + ".");
 
-      if(field.getPossibleValueSourceName() != null)
-      {
-         assertCondition(qInstance.getPossibleValueSource(field.getPossibleValueSourceName()) != null,
-            "Unrecognized possibleValueSourceName " + field.getPossibleValueSourceName() + " in table " + tableName + " for field " + fieldName + ".");
-      }
-
       String prefix = "Field " + fieldName + " in table " + tableName + " ";
+      validateFieldPossibleValueSourceAttributes(qInstance, field, prefix);
 
       ///////////////////////////////////////////////////
       // validate things we know about field behaviors //
@@ -953,7 +1049,15 @@ public class QInstanceValidator
             @SuppressWarnings("unchecked")
             Class<FieldBehavior<?>> behaviorClass = (Class<FieldBehavior<?>>) fieldBehavior.getClass();
 
-            errors.addAll(fieldBehavior.validateBehaviorConfiguration(table, field));
+            List<String> behaviorErrors = fieldBehavior.validateBehaviorConfiguration(table, field);
+            if(behaviorErrors != null)
+            {
+               String prefixMinusTrailingSpace = prefix.replaceFirst(" *$", "");
+               for(String behaviorError : behaviorErrors)
+               {
+                  errors.add(prefixMinusTrailingSpace + ": " + behaviorClass.getSimpleName() + ": " + behaviorError);
+               }
+            }
 
             if(!fieldBehavior.allowMultipleBehaviorsOfThisType())
             {
@@ -1039,6 +1143,31 @@ public class QInstanceValidator
 
 
 
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   private void validateFieldPossibleValueSourceAttributes(QInstance qInstance, QFieldMetaData field, String prefix)
+   {
+      if(field.getPossibleValueSourceName() != null)
+      {
+         assertCondition(qInstance.getPossibleValueSource(field.getPossibleValueSourceName()) != null,
+            prefix + "has an unrecognized possibleValueSourceName " + field.getPossibleValueSourceName());
+
+         assertCondition(field.getInlinePossibleValueSource() == null, prefix.trim() + " has both a possibleValueSourceName and an inlinePossibleValueSource, which is not allowed.");
+      }
+
+      if(field.getInlinePossibleValueSource() != null)
+      {
+         String name = "inlinePossibleValueSource for " + prefix.trim();
+         if(assertCondition(QPossibleValueSourceType.ENUM.equals(field.getInlinePossibleValueSource().getType()), name + " must have a type of ENUM."))
+         {
+            validatePossibleValueSource(qInstance, name, field.getInlinePossibleValueSource());
+         }
+      }
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/
@@ -1288,7 +1417,7 @@ public class QInstanceValidator
                ////////////////////////////////////////////////////////////////////////
                if(customizerInstance != null && tableCustomizer.getExpectedType() != null)
                {
-                  assertObjectCanBeCasted(prefix, tableCustomizer.getExpectedType(), customizerInstance);
+                  assertObjectCanBeCasted(prefix, customizerInstance, tableCustomizer.getExpectedType());
                }
             }
          }
@@ -1300,18 +1429,31 @@ public class QInstanceValidator
    /*******************************************************************************
     ** Make sure that a given object can be casted to an expected type.
     *******************************************************************************/
-   private <T> T assertObjectCanBeCasted(String errorPrefix, Class<T> expectedType, Object object)
+   private void assertObjectCanBeCasted(String errorPrefix, Object object, Class<?>... anyOfExpectedClasses)
    {
-      T castedObject = null;
-      try
+      for(Class<?> expectedClass : anyOfExpectedClasses)
       {
-         castedObject = expectedType.cast(object);
+         try
+         {
+            expectedClass.cast(object);
+            return;
+         }
+         catch(ClassCastException e)
+         {
+            /////////////////////////////////////
+            // try next type (if there is one) //
+            /////////////////////////////////////
+         }
       }
-      catch(ClassCastException e)
+
+      if(anyOfExpectedClasses.length == 1)
       {
-         errors.add(errorPrefix + "CodeReference is not of the expected type: " + expectedType);
+         errors.add(errorPrefix + "CodeReference is not of the expected type: " + anyOfExpectedClasses[0]);
+      }
+      else
+      {
+         errors.add(errorPrefix + "CodeReference is not any of the expected types: " + Arrays.stream(anyOfExpectedClasses).map(c -> c.getName()).collect(Collectors.joining(", ")));
       }
-      return castedObject;
    }
 
 
@@ -1546,6 +1688,16 @@ public class QInstanceValidator
                }
             }
 
+            for(QFieldMetaData field : process.getInputFields())
+            {
+               validateFieldPossibleValueSourceAttributes(qInstance, field, "Process " + processName + ", input field " + field.getName() + " ");
+            }
+
+            for(QFieldMetaData field : process.getOutputFields())
+            {
+               validateFieldPossibleValueSourceAttributes(qInstance, field, "Process " + processName + ", output field " + field.getName() + " ");
+            }
+
             if(process.getCancelStep() != null)
             {
                if(assertCondition(process.getCancelStep().getCode() != null, "Cancel step is missing a code reference, in process " + processName))
@@ -1754,7 +1906,7 @@ public class QInstanceValidator
    /*******************************************************************************
     **
     *******************************************************************************/
-   private void validateQueryFilter(QInstance qInstance, String context, QTableMetaData table, QQueryFilter queryFilter, List<QueryJoin> queryJoins)
+   public void validateQueryFilter(QInstance qInstance, String context, QTableMetaData table, QQueryFilter queryFilter, List<QueryJoin> queryJoins)
    {
       for(QFilterCriteria criterion : CollectionUtils.nonNullList(queryFilter.getCriteria()))
       {
@@ -1798,7 +1950,8 @@ public class QInstanceValidator
       {
          if(fieldName.contains("."))
          {
-            String fieldNameAfterDot = fieldName.substring(fieldName.lastIndexOf(".") + 1);
+            String fieldNameAfterDot  = fieldName.substring(fieldName.lastIndexOf(".") + 1);
+            String tableNameBeforeDot = fieldName.substring(0, fieldName.lastIndexOf("."));
 
             if(CollectionUtils.nullSafeHasContents(queryJoins))
             {
@@ -1822,11 +1975,32 @@ public class QInstanceValidator
             }
             else
             {
-               errors.add("QInstanceValidator does not yet support finding a field that looks like a join field, but isn't associated with a query.");
-               return (true);
-               // todo! for(QJoinMetaData join : CollectionUtils.nonNullMap(qInstance.getJoins()).values())
-               // {
-               // }
+               if(this.joinGraph != null)
+               {
+                  Set<JoinGraph.JoinConnectionList> joinConnections = joinGraph.getJoinConnections(table.getName());
+                  for(JoinGraph.JoinConnectionList joinConnectionList : joinConnections)
+                  {
+                     JoinGraph.JoinConnection joinConnection = joinConnectionList.list().get(joinConnectionList.list().size() - 1);
+                     if(tableNameBeforeDot.equals(joinConnection.joinTable()))
+                     {
+                        QTableMetaData joinTable = qInstance.getTable(tableNameBeforeDot);
+                        if(joinTable.getFields().containsKey(fieldNameAfterDot))
+                        {
+                           /////////////////////////
+                           // mmm, looks valid... //
+                           /////////////////////////
+                           return (true);
+                        }
+                     }
+                  }
+               }
+
+               //////////////////////////////////////////////////////////////////////////////////////
+               // todo - not sure how vulnerable we are to ongoing issues here...                  //
+               // idea:  let a filter (or any object?) be opted out of validation, some version of //
+               // a static map of objects we can check at the top of various validate methods...   //
+               //////////////////////////////////////////////////////////////////////////////////////
+               errors.add("Failed to find field named: " + fieldName);
             }
          }
       }
@@ -1930,6 +2104,11 @@ public class QInstanceValidator
                   }
                }
 
+               if(widget.getValidatorPlugin() != null)
+               {
+                  widget.getValidatorPlugin().validate(widget, qInstance, this);
+               }
+
                runPlugins(QWidgetMetaDataInterface.class, widget, qInstance);
             }
          );
@@ -1948,78 +2127,90 @@ public class QInstanceValidator
          qInstance.getPossibleValueSources().forEach((pvsName, possibleValueSource) ->
          {
             assertCondition(Objects.equals(pvsName, possibleValueSource.getName()), "Inconsistent naming for possibleValueSource: " + pvsName + "/" + possibleValueSource.getName() + ".");
-            if(assertCondition(possibleValueSource.getType() != null, "Missing type for possibleValueSource: " + pvsName))
+            validatePossibleValueSource(qInstance, pvsName, possibleValueSource);
+         });
+      }
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   private void validatePossibleValueSource(QInstance qInstance, String name, QPossibleValueSource possibleValueSource)
+   {
+      if(assertCondition(possibleValueSource.getType() != null, "Missing type for possibleValueSource: " + name))
+      {
+         ////////////////////////////////////////////////////////////////////////////////////////////////
+         // assert about fields that should and should not be set, based on possible value source type //
+         // do additional type-specific validations as well                                            //
+         ////////////////////////////////////////////////////////////////////////////////////////////////
+         switch(possibleValueSource.getType())
+         {
+            case ENUM ->
             {
-               ////////////////////////////////////////////////////////////////////////////////////////////////
-               // assert about fields that should and should not be set, based on possible value source type //
-               // do additional type-specific validations as well                                            //
-               ////////////////////////////////////////////////////////////////////////////////////////////////
-               switch(possibleValueSource.getType())
+               assertCondition(!StringUtils.hasContent(possibleValueSource.getTableName()), "enum-type possibleValueSource " + name + " should not have a tableName.");
+               assertCondition(!CollectionUtils.nullSafeHasContents(possibleValueSource.getSearchFields()), "enum-type possibleValueSource " + name + " should not have searchFields.");
+               assertCondition(!CollectionUtils.nullSafeHasContents(possibleValueSource.getOrderByFields()), "enum-type possibleValueSource " + name + " should not have orderByFields.");
+               assertCondition(possibleValueSource.getCustomCodeReference() == null, "enum-type possibleValueSource " + name + " should not have a customCodeReference.");
+
+               assertCondition(CollectionUtils.nullSafeHasContents(possibleValueSource.getEnumValues()), "enum-type possibleValueSource " + name + " is missing enum values");
+            }
+            case TABLE ->
+            {
+               assertCondition(CollectionUtils.nullSafeIsEmpty(possibleValueSource.getEnumValues()), "table-type possibleValueSource " + name + " should not have enum values.");
+               assertCondition(possibleValueSource.getCustomCodeReference() == null, "table-type possibleValueSource " + name + " should not have a customCodeReference.");
+
+               QTableMetaData tableMetaData = null;
+               if(assertCondition(StringUtils.hasContent(possibleValueSource.getTableName()), "table-type possibleValueSource " + name + " is missing a tableName."))
                {
-                  case ENUM ->
-                  {
-                     assertCondition(!StringUtils.hasContent(possibleValueSource.getTableName()), "enum-type possibleValueSource " + pvsName + " should not have a tableName.");
-                     assertCondition(!CollectionUtils.nullSafeHasContents(possibleValueSource.getSearchFields()), "enum-type possibleValueSource " + pvsName + " should not have searchFields.");
-                     assertCondition(!CollectionUtils.nullSafeHasContents(possibleValueSource.getOrderByFields()), "enum-type possibleValueSource " + pvsName + " should not have orderByFields.");
-                     assertCondition(possibleValueSource.getCustomCodeReference() == null, "enum-type possibleValueSource " + pvsName + " should not have a customCodeReference.");
-
-                     assertCondition(CollectionUtils.nullSafeHasContents(possibleValueSource.getEnumValues()), "enum-type possibleValueSource " + pvsName + " is missing enum values");
-                  }
-                  case TABLE ->
-                  {
-                     assertCondition(CollectionUtils.nullSafeIsEmpty(possibleValueSource.getEnumValues()), "table-type possibleValueSource " + pvsName + " should not have enum values.");
-                     assertCondition(possibleValueSource.getCustomCodeReference() == null, "table-type possibleValueSource " + pvsName + " should not have a customCodeReference.");
-
-                     QTableMetaData tableMetaData = null;
-                     if(assertCondition(StringUtils.hasContent(possibleValueSource.getTableName()), "table-type possibleValueSource " + pvsName + " is missing a tableName."))
-                     {
-                        tableMetaData = qInstance.getTable(possibleValueSource.getTableName());
-                        assertCondition(tableMetaData != null, "Unrecognized table " + possibleValueSource.getTableName() + " for possibleValueSource " + pvsName + ".");
-                     }
-
-                     if(assertCondition(CollectionUtils.nullSafeHasContents(possibleValueSource.getSearchFields()), "table-type possibleValueSource " + pvsName + " is missing searchFields."))
-                     {
-                        if(tableMetaData != null)
-                        {
-                           QTableMetaData finalTableMetaData = tableMetaData;
-                           for(String searchField : possibleValueSource.getSearchFields())
-                           {
-                              assertNoException(() -> finalTableMetaData.getField(searchField), "possibleValueSource " + pvsName + " has an unrecognized searchField: " + searchField);
-                           }
-                        }
-                     }
-
-                     if(assertCondition(CollectionUtils.nullSafeHasContents(possibleValueSource.getOrderByFields()), "table-type possibleValueSource " + pvsName + " is missing orderByFields."))
-                     {
-                        if(tableMetaData != null)
-                        {
-                           QTableMetaData finalTableMetaData = tableMetaData;
-
-                           for(QFilterOrderBy orderByField : possibleValueSource.getOrderByFields())
-                           {
-                              assertNoException(() -> finalTableMetaData.getField(orderByField.getFieldName()), "possibleValueSource " + pvsName + " has an unrecognized orderByField: " + orderByField.getFieldName());
-                           }
-                        }
-                     }
-                  }
-                  case CUSTOM ->
-                  {
-                     assertCondition(CollectionUtils.nullSafeIsEmpty(possibleValueSource.getEnumValues()), "custom-type possibleValueSource " + pvsName + " should not have enum values.");
-                     assertCondition(!StringUtils.hasContent(possibleValueSource.getTableName()), "custom-type possibleValueSource " + pvsName + " should not have a tableName.");
-                     assertCondition(!CollectionUtils.nullSafeHasContents(possibleValueSource.getSearchFields()), "custom-type possibleValueSource " + pvsName + " should not have searchFields.");
-                     assertCondition(!CollectionUtils.nullSafeHasContents(possibleValueSource.getOrderByFields()), "custom-type possibleValueSource " + pvsName + " should not have orderByFields.");
-
-                     if(assertCondition(possibleValueSource.getCustomCodeReference() != null, "custom-type possibleValueSource " + pvsName + " is missing a customCodeReference."))
-                     {
-                        validateSimpleCodeReference("PossibleValueSource " + pvsName + " custom code reference: ", possibleValueSource.getCustomCodeReference(), QCustomPossibleValueProvider.class);
-                     }
-                  }
-                  default -> errors.add("Unexpected possibleValueSource type: " + possibleValueSource.getType());
+                  tableMetaData = qInstance.getTable(possibleValueSource.getTableName());
+                  assertCondition(tableMetaData != null, "Unrecognized table " + possibleValueSource.getTableName() + " for possibleValueSource " + name + ".");
                }
 
-               runPlugins(QPossibleValueSource.class, possibleValueSource, qInstance);
+               if(assertCondition(CollectionUtils.nullSafeHasContents(possibleValueSource.getSearchFields()), "table-type possibleValueSource " + name + " is missing searchFields."))
+               {
+                  if(tableMetaData != null)
+                  {
+                     QTableMetaData finalTableMetaData = tableMetaData;
+                     for(String searchField : possibleValueSource.getSearchFields())
+                     {
+                        assertNoException(() -> finalTableMetaData.getField(searchField), "possibleValueSource " + name + " has an unrecognized searchField: " + searchField);
+                     }
+                  }
+               }
+
+               if(assertCondition(CollectionUtils.nullSafeHasContents(possibleValueSource.getOrderByFields()), "table-type possibleValueSource " + name + " is missing orderByFields."))
+               {
+                  if(tableMetaData != null)
+                  {
+                     QTableMetaData finalTableMetaData = tableMetaData;
+
+                     for(QFilterOrderBy orderByField : possibleValueSource.getOrderByFields())
+                     {
+                        assertNoException(() -> finalTableMetaData.getField(orderByField.getFieldName()), "possibleValueSource " + name + " has an unrecognized orderByField: " + orderByField.getFieldName());
+                     }
+                  }
+               }
             }
-         });
+            case CUSTOM ->
+            {
+               assertCondition(CollectionUtils.nullSafeIsEmpty(possibleValueSource.getEnumValues()), "custom-type possibleValueSource " + name + " should not have enum values.");
+               assertCondition(!StringUtils.hasContent(possibleValueSource.getTableName()), "custom-type possibleValueSource " + name + " should not have a tableName.");
+               assertCondition(!CollectionUtils.nullSafeHasContents(possibleValueSource.getSearchFields()), "custom-type possibleValueSource " + name + " should not have searchFields.");
+               assertCondition(!CollectionUtils.nullSafeHasContents(possibleValueSource.getOrderByFields()), "custom-type possibleValueSource " + name + " should not have orderByFields.");
+
+               if(assertCondition(possibleValueSource.getCustomCodeReference() != null, "custom-type possibleValueSource " + name + " is missing a customCodeReference."))
+               {
+                  validateSimpleCodeReference("PossibleValueSource " + name + " custom code reference: ", possibleValueSource.getCustomCodeReference(), QCustomPossibleValueProvider.class);
+               }
+            }
+            default -> errors.add("Unexpected possibleValueSource type: " + possibleValueSource.getType());
+         }
+
+         assertCondition(possibleValueSource.getIdType() != null, "possibleValueSource " + name + " is missing its idType.");
+
+         runPlugins(QPossibleValueSource.class, possibleValueSource, qInstance);
       }
    }
 
@@ -2028,7 +2219,8 @@ public class QInstanceValidator
    /*******************************************************************************
     **
     *******************************************************************************/
-   private void validateSimpleCodeReference(String prefix, QCodeReference codeReference, Class<?> expectedClass)
+   @SafeVarargs
+   private void validateSimpleCodeReference(String prefix, QCodeReference codeReference, Class<?>... anyOfExpectedClasses)
    {
       if(!preAssertionsForCodeReference(codeReference, prefix))
       {
@@ -2056,7 +2248,7 @@ public class QInstanceValidator
             ////////////////////////////////////////////////////////////////////////
             if(classInstance != null)
             {
-               assertObjectCanBeCasted(prefix, expectedClass, classInstance);
+               assertObjectCanBeCasted(prefix, classInstance, anyOfExpectedClasses);
             }
          }
       }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/enrichment/plugins/QInstanceEnricherPluginInterface.java

@@ -0,0 +1,40 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2025.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.instances.enrichment.plugins;
+
+
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+
+
+/*******************************************************************************
+ ** Interface for additional / optional enrichment to be done on q instance members.
+ ** Some may be provided by QQQ - others can be defined by applications.
+ *******************************************************************************/
+public interface QInstanceEnricherPluginInterface<T>
+{
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   void enrich(T object, QInstance qInstance);
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/audits/AuditSingleInput.java

@@ -326,6 +326,20 @@ public class AuditSingleInput implements Serializable
 
 
 
+   /*******************************************************************************
+    ** Fluent setter for details
+    *******************************************************************************/
+   public AuditSingleInput withDetailMessages(List<String> details)
+   {
+      for(String detail : details)
+      {
+         addDetail(message);
+      }
+      return (this);
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/metadata/MetaDataInput.java

@@ -31,6 +31,16 @@ import com.kingsrook.qqq.backend.core.model.actions.AbstractActionInput;
  *******************************************************************************/
 public class MetaDataInput extends AbstractActionInput
 {
+   private String frontendName;
+   private String frontendVersion;
+
+   private String middlewareName;
+   private String middlewareVersion;
+
+   private String applicationName;
+   private String applicationVersion;
+
+
 
    /*******************************************************************************
     **
@@ -39,4 +49,190 @@ public class MetaDataInput extends AbstractActionInput
    {
    }
 
+
+
+   /*******************************************************************************
+    ** Getter for frontendName
+    *******************************************************************************/
+   public String getFrontendName()
+   {
+      return (this.frontendName);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for frontendName
+    *******************************************************************************/
+   public void setFrontendName(String frontendName)
+   {
+      this.frontendName = frontendName;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for frontendName
+    *******************************************************************************/
+   public MetaDataInput withFrontendName(String frontendName)
+   {
+      this.frontendName = frontendName;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for frontendVersion
+    *******************************************************************************/
+   public String getFrontendVersion()
+   {
+      return (this.frontendVersion);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for frontendVersion
+    *******************************************************************************/
+   public void setFrontendVersion(String frontendVersion)
+   {
+      this.frontendVersion = frontendVersion;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for frontendVersion
+    *******************************************************************************/
+   public MetaDataInput withFrontendVersion(String frontendVersion)
+   {
+      this.frontendVersion = frontendVersion;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for middlewareName
+    *******************************************************************************/
+   public String getMiddlewareName()
+   {
+      return (this.middlewareName);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for middlewareName
+    *******************************************************************************/
+   public void setMiddlewareName(String middlewareName)
+   {
+      this.middlewareName = middlewareName;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for middlewareName
+    *******************************************************************************/
+   public MetaDataInput withMiddlewareName(String middlewareName)
+   {
+      this.middlewareName = middlewareName;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for middlewareVersion
+    *******************************************************************************/
+   public String getMiddlewareVersion()
+   {
+      return (this.middlewareVersion);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for middlewareVersion
+    *******************************************************************************/
+   public void setMiddlewareVersion(String middlewareVersion)
+   {
+      this.middlewareVersion = middlewareVersion;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for middlewareVersion
+    *******************************************************************************/
+   public MetaDataInput withMiddlewareVersion(String middlewareVersion)
+   {
+      this.middlewareVersion = middlewareVersion;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for applicationName
+    *******************************************************************************/
+   public String getApplicationName()
+   {
+      return (this.applicationName);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for applicationName
+    *******************************************************************************/
+   public void setApplicationName(String applicationName)
+   {
+      this.applicationName = applicationName;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for applicationName
+    *******************************************************************************/
+   public MetaDataInput withApplicationName(String applicationName)
+   {
+      this.applicationName = applicationName;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for applicationVersion
+    *******************************************************************************/
+   public String getApplicationVersion()
+   {
+      return (this.applicationVersion);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for applicationVersion
+    *******************************************************************************/
+   public void setApplicationVersion(String applicationVersion)
+   {
+      this.applicationVersion = applicationVersion;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for applicationVersion
+    *******************************************************************************/
+   public MetaDataInput withApplicationVersion(String applicationVersion)
+   {
+      this.applicationVersion = applicationVersion;
+      return (this);
+   }
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/ProcessMetaDataAdjustment.java

@@ -0,0 +1,139 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.model.actions.processes;
+
+
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import com.kingsrook.qqq.backend.core.logging.QLogger;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendStepMetaData;
+import com.kingsrook.qqq.backend.core.utils.StringUtils;
+import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
+
+
+/*******************************************************************************
+ ** Object that stores adjustments that a process wants to make, at run-time,
+ ** to its meta-data.
+ **
+ ** e.g., changing the steps; updating fields (e.g., changing an inline PVS,
+ ** or an isRequired attribute)
+ *******************************************************************************/
+public class ProcessMetaDataAdjustment
+{
+   private static final QLogger LOG = QLogger.getLogger(ProcessMetaDataAdjustment.class);
+
+   private List<QFrontendStepMetaData> updatedFrontendStepList = null;
+   private Map<String, QFieldMetaData> updatedFields           = null;
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public ProcessMetaDataAdjustment withUpdatedField(QFieldMetaData field)
+   {
+      if(updatedFields == null)
+      {
+         updatedFields = new LinkedHashMap<>();
+      }
+
+      if(!StringUtils.hasContent(field.getName()))
+      {
+         LOG.warn("Missing name on field in withUpdatedField - no update will happen.");
+      }
+      else
+      {
+         if(updatedFields.containsKey(field.getName()))
+         {
+            LOG.info("UpdatedFields map already contained a field with this name - overwriting it.", logPair("fieldName", field.getName()));
+         }
+
+         updatedFields.put(field.getName(), field);
+      }
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for updatedFrontendStepList
+    *******************************************************************************/
+   public List<QFrontendStepMetaData> getUpdatedFrontendStepList()
+   {
+      return (this.updatedFrontendStepList);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for updatedFrontendStepList
+    *******************************************************************************/
+   public void setUpdatedFrontendStepList(List<QFrontendStepMetaData> updatedFrontendStepList)
+   {
+      this.updatedFrontendStepList = updatedFrontendStepList;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for updatedFrontendStepList
+    *******************************************************************************/
+   public ProcessMetaDataAdjustment withUpdatedFrontendStepList(List<QFrontendStepMetaData> updatedFrontendStepList)
+   {
+      this.updatedFrontendStepList = updatedFrontendStepList;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for updatedFields
+    *******************************************************************************/
+   public Map<String, QFieldMetaData> getUpdatedFields()
+   {
+      return (this.updatedFields);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for updatedFields
+    *******************************************************************************/
+   public void setUpdatedFields(Map<String, QFieldMetaData> updatedFields)
+   {
+      this.updatedFields = updatedFields;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for updatedFields
+    *******************************************************************************/
+   public ProcessMetaDataAdjustment withUpdatedFields(Map<String, QFieldMetaData> updatedFields)
+   {
+      this.updatedFields = updatedFields;
+      return (this);
+   }
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/ProcessState.java

@@ -29,7 +29,6 @@ import java.util.List;
 import java.util.Map;
 import java.util.Optional;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
-import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendStepMetaData;
 
 
 /*******************************************************************************
@@ -41,11 +40,10 @@ public class ProcessState implements Serializable
    private Map<String, Serializable> values       = new HashMap<>();
    private List<String>              stepList     = new ArrayList<>();
    private Optional<String>          nextStepName = Optional.empty();
+   private Optional<String>          backStepName = Optional.empty();
+   private boolean                   isStepBack   = false;
 
-   /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-   // maybe, remove this altogether - just let the frontend compute & send if needed... but how does it know last version...? //
-   /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-   private List<QFrontendStepMetaData> updatedFrontendStepList = null;
+   private ProcessMetaDataAdjustment processMetaDataAdjustment = null;
 
 
 
@@ -126,6 +124,39 @@ public class ProcessState implements Serializable
 
 
 
+   /*******************************************************************************
+    ** Getter for backStepName
+    **
+    *******************************************************************************/
+   public Optional<String> getBackStepName()
+   {
+      return backStepName;
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for backStepName
+    **
+    *******************************************************************************/
+   public void setBackStepName(String backStepName)
+   {
+      this.backStepName = Optional.of(backStepName);
+   }
+
+
+
+   /*******************************************************************************
+    ** clear out the value of backStepName (set the Optional to empty)
+    **
+    *******************************************************************************/
+   public void clearBackStepName()
+   {
+      this.backStepName = Optional.empty();
+   }
+
+
+
    /*******************************************************************************
     ** Getter for stepList
     **
@@ -148,33 +179,67 @@ public class ProcessState implements Serializable
 
 
 
+
+
    /*******************************************************************************
-    ** Getter for updatedFrontendStepList
+    ** Getter for processMetaDataAdjustment
     *******************************************************************************/
-   public List<QFrontendStepMetaData> getUpdatedFrontendStepList()
+   public ProcessMetaDataAdjustment getProcessMetaDataAdjustment()
    {
-      return (this.updatedFrontendStepList);
+      return (this.processMetaDataAdjustment);
    }
 
 
 
    /*******************************************************************************
-    ** Setter for updatedFrontendStepList
+    ** Setter for processMetaDataAdjustment
     *******************************************************************************/
-   public void setUpdatedFrontendStepList(List<QFrontendStepMetaData> updatedFrontendStepList)
+   public void setProcessMetaDataAdjustment(ProcessMetaDataAdjustment processMetaDataAdjustment)
    {
-      this.updatedFrontendStepList = updatedFrontendStepList;
+      this.processMetaDataAdjustment = processMetaDataAdjustment;
    }
 
 
 
    /*******************************************************************************
-    ** Fluent setter for updatedFrontendStepList
+    ** Fluent setter for processMetaDataAdjustment
     *******************************************************************************/
-   public ProcessState withUpdatedFrontendStepList(List<QFrontendStepMetaData> updatedFrontendStepList)
+   public ProcessState withProcessMetaDataAdjustment(ProcessMetaDataAdjustment processMetaDataAdjustment)
    {
-      this.updatedFrontendStepList = updatedFrontendStepList;
+      this.processMetaDataAdjustment = processMetaDataAdjustment;
       return (this);
    }
 
+
+
+   /*******************************************************************************
+    ** Getter for isStepBack
+    *******************************************************************************/
+   public boolean getIsStepBack()
+   {
+      return (this.isStepBack);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for isStepBack
+    *******************************************************************************/
+   public void setIsStepBack(boolean isStepBack)
+   {
+      this.isStepBack = isStepBack;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for isStepBack
+    *******************************************************************************/
+   public ProcessState withIsStepBack(boolean isStepBack)
+   {
+      this.isStepBack = isStepBack;
+      return (this);
+   }
+
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/ProcessSummaryLine.java

@@ -53,6 +53,7 @@ public class ProcessSummaryLine implements ProcessSummaryLineInterface
    //////////////////////////////////////////////////////////////////////////
    private ArrayList<Serializable> primaryKeys;
 
+   private ArrayList<String> bulletsOfText;
 
 
    /*******************************************************************************
@@ -497,4 +498,35 @@ public class ProcessSummaryLine implements ProcessSummaryLineInterface
       return (this);
    }
 
+
+   /*******************************************************************************
+    ** Getter for bulletsOfText
+    *******************************************************************************/
+   public ArrayList<String> getBulletsOfText()
+   {
+      return (this.bulletsOfText);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for bulletsOfText
+    *******************************************************************************/
+   public void setBulletsOfText(ArrayList<String> bulletsOfText)
+   {
+      this.bulletsOfText = bulletsOfText;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for bulletsOfText
+    *******************************************************************************/
+   public ProcessSummaryLine withBulletsOfText(ArrayList<String> bulletsOfText)
+   {
+      this.bulletsOfText = bulletsOfText;
+      return (this);
+   }
+
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/RunBackendStepInput.java

@@ -25,19 +25,28 @@ package com.kingsrook.qqq.backend.core.model.actions.processes;
 import java.io.Serializable;
 import java.time.Instant;
 import java.time.LocalDate;
+import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
+import java.util.Optional;
 import java.util.UUID;
 import com.kingsrook.qqq.backend.core.actions.async.AsyncJobCallback;
 import com.kingsrook.qqq.backend.core.actions.async.AsyncJobStatus;
 import com.kingsrook.qqq.backend.core.actions.async.NonPersistedAsyncJobCallback;
 import com.kingsrook.qqq.backend.core.actions.processes.QProcessCallback;
 import com.kingsrook.qqq.backend.core.context.QContext;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.logging.QLogger;
 import com.kingsrook.qqq.backend.core.model.actions.AbstractActionInput;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.data.QRecordEntity;
+import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.processes.QStepMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+import com.kingsrook.qqq.backend.core.processes.tracing.ProcessTracerInterface;
+import com.kingsrook.qqq.backend.core.processes.tracing.ProcessTracerMessage;
 import com.kingsrook.qqq.backend.core.utils.ValueUtils;
+import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
 
 
 /*******************************************************************************
@@ -46,6 +55,8 @@ import com.kingsrook.qqq.backend.core.utils.ValueUtils;
  *******************************************************************************/
 public class RunBackendStepInput extends AbstractActionInput
 {
+   private static final QLogger LOG = QLogger.getLogger(RunBackendStepInput.class);
+
    private ProcessState                         processState;
    private String                               processName;
    private String                               tableName;
@@ -55,12 +66,13 @@ public class RunBackendStepInput extends AbstractActionInput
    private RunProcessInput.FrontendStepBehavior frontendStepBehavior;
    private Instant                              basepullLastRunTime;
 
+   private ProcessTracerInterface processTracer;
+
    ////////////////////////////////////////////////////////////////////////////
    // note - new fields should generally be added in method: cloneFieldsInto //
    ////////////////////////////////////////////////////////////////////////////
 
 
-
    /*******************************************************************************
     **
     *******************************************************************************/
@@ -96,6 +108,7 @@ public class RunBackendStepInput extends AbstractActionInput
       target.setAsyncJobCallback(getAsyncJobCallback());
       target.setFrontendStepBehavior(getFrontendStepBehavior());
       target.setValues(getValues());
+      target.setProcessTracer(getProcessTracer().orElse(null));
    }
 
 
@@ -238,6 +251,26 @@ public class RunBackendStepInput extends AbstractActionInput
 
 
 
+   /*******************************************************************************
+    ** Getter for records converted to entities of a given type.
+    **
+    *******************************************************************************/
+   public <E extends QRecordEntity> List<E> getRecordsAsEntities(Class<E> entityClass) throws QException
+   {
+      List<E> rs = new ArrayList<>();
+
+      ///////////////////////////////////////////////////////////////////////////////////
+      // note - important to call getRecords here, which is overwritten in subclasses! //
+      ///////////////////////////////////////////////////////////////////////////////////
+      for(QRecord record : getRecords())
+      {
+         rs.add(QRecordEntity.fromQRecord(entityClass, record));
+      }
+      return (rs);
+   }
+
+
+
    /*******************************************************************************
     ** Setter for records
     **
@@ -419,6 +452,17 @@ public class RunBackendStepInput extends AbstractActionInput
 
 
 
+   /*******************************************************************************
+    ** Accessor for processState's isStepBack attribute
+    **
+    *******************************************************************************/
+   public boolean getIsStepBack()
+   {
+      return processState.getIsStepBack();
+   }
+
+
+
    /*******************************************************************************
     ** Accessor for processState - protected, because we generally want to access
     ** its members through wrapper methods, we think
@@ -524,4 +568,64 @@ public class RunBackendStepInput extends AbstractActionInput
       return (this);
    }
 
+
+
+   /*******************************************************************************
+    ** Setter for processTracer
+    *******************************************************************************/
+   public void setProcessTracer(ProcessTracerInterface processTracer)
+   {
+      this.processTracer = processTracer;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for processTracer
+    *******************************************************************************/
+   public RunBackendStepInput withProcessTracer(ProcessTracerInterface processTracer)
+   {
+      this.processTracer = processTracer;
+      return (this);
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   public Optional<ProcessTracerInterface> getProcessTracer()
+   {
+      return Optional.ofNullable(processTracer);
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   public void traceMessage(ProcessTracerMessage message)
+   {
+      if(processTracer != null && message != null)
+      {
+         try
+         {
+            processTracer.handleMessage(this, message);
+         }
+         catch(Exception e)
+         {
+            LOG.warn("Error tracing message", e, logPair("message", message));
+         }
+      }
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   public QProcessMetaData getProcess()
+   {
+      return (QContext.getQInstance().getProcess(getProcessName()));
+   }
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/RunBackendStepOutput.java

@@ -33,6 +33,7 @@ import com.kingsrook.qqq.backend.core.model.actions.AbstractActionOutput;
 import com.kingsrook.qqq.backend.core.model.actions.audits.AuditInput;
 import com.kingsrook.qqq.backend.core.model.actions.audits.AuditSingleInput;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.data.QRecordEntity;
 import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendStepMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
 import com.kingsrook.qqq.backend.core.utils.ValueUtils;
@@ -258,7 +259,7 @@ public class RunBackendStepOutput extends AbstractActionOutput implements Serial
 
 
    /*******************************************************************************
-    **
+    ** add a record to the step output, e.g., for going through to the next step.
     *******************************************************************************/
    public void addRecord(QRecord record)
    {
@@ -271,6 +272,16 @@ public class RunBackendStepOutput extends AbstractActionOutput implements Serial
 
 
 
+   /***************************************************************************
+    ** add a RecordEntity to the step output, e.g., for going through to the next step.
+    ***************************************************************************/
+   public void addRecordEntity(QRecordEntity recordEntity)
+   {
+      addRecord(recordEntity.toQRecord());
+   }
+
+
+
    /*******************************************************************************
     ** Getter for auditInputList
     *******************************************************************************/
@@ -374,7 +385,13 @@ public class RunBackendStepOutput extends AbstractActionOutput implements Serial
          .map(step -> (QFrontendStepMetaData) step)
          .toList());
 
-      setUpdatedFrontendStepList(updatedFrontendStepList);
+      ProcessMetaDataAdjustment processMetaDataAdjustment = getProcessMetaDataAdjustment();
+      if(processMetaDataAdjustment == null)
+      {
+         processMetaDataAdjustment = new ProcessMetaDataAdjustment();
+      }
+      processMetaDataAdjustment.setUpdatedFrontendStepList(updatedFrontendStepList);
+      setProcessMetaDataAdjustment(processMetaDataAdjustment);
    }
 
 
@@ -411,21 +428,21 @@ public class RunBackendStepOutput extends AbstractActionOutput implements Serial
 
 
    /*******************************************************************************
-    ** Getter for updatedFrontendStepList
+    ** Getter for ProcessMetaDataAdjustment (pass-through to processState)
     *******************************************************************************/
-   public List<QFrontendStepMetaData> getUpdatedFrontendStepList()
+   public ProcessMetaDataAdjustment getProcessMetaDataAdjustment()
    {
-      return (this.processState.getUpdatedFrontendStepList());
+      return (this.processState.getProcessMetaDataAdjustment());
    }
 
 
 
    /*******************************************************************************
-    ** Setter for updatedFrontendStepList
+    ** Setter for updatedFrontendStepList (pass-through to processState)
     *******************************************************************************/
-   public void setUpdatedFrontendStepList(List<QFrontendStepMetaData> updatedFrontendStepList)
+   public void setProcessMetaDataAdjustment(ProcessMetaDataAdjustment processMetaDataAdjustment)
    {
-      this.processState.setUpdatedFrontendStepList(updatedFrontendStepList);
+      this.processState.setProcessMetaDataAdjustment(processMetaDataAdjustment);
    }
 
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/RunProcessInput.java

@@ -49,6 +49,7 @@ public class RunProcessInput extends AbstractActionInput
    private ProcessState         processState;
    private FrontendStepBehavior frontendStepBehavior = FrontendStepBehavior.BREAK;
    private String               startAfterStep;
+   private String               startAtStep;
    private String               processUUID;
    private AsyncJobCallback     asyncJobCallback;
 
@@ -451,4 +452,35 @@ public class RunProcessInput extends AbstractActionInput
    {
       return asyncJobCallback;
    }
+
+   /*******************************************************************************
+    ** Getter for startAtStep
+    *******************************************************************************/
+   public String getStartAtStep()
+   {
+      return (this.startAtStep);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for startAtStep
+    *******************************************************************************/
+   public void setStartAtStep(String startAtStep)
+   {
+      this.startAtStep = startAtStep;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for startAtStep
+    *******************************************************************************/
+   public RunProcessInput withStartAtStep(String startAtStep)
+   {
+      this.startAtStep = startAtStep;
+      return (this);
+   }
+
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/RunProcessOutput.java

@@ -33,6 +33,7 @@ import java.util.Optional;
 import com.kingsrook.qqq.backend.core.model.actions.AbstractActionOutput;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
 import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendStepMetaData;
+import com.kingsrook.qqq.backend.core.utils.ObjectUtils;
 import com.kingsrook.qqq.backend.core.utils.ValueUtils;
 
 
@@ -336,7 +337,12 @@ public class RunProcessOutput extends AbstractActionOutput implements Serializab
     *******************************************************************************/
    public void setUpdatedFrontendStepList(List<QFrontendStepMetaData> updatedFrontendStepList)
    {
-      this.processState.setUpdatedFrontendStepList(updatedFrontendStepList);
+      if(this.processState.getProcessMetaDataAdjustment() == null)
+      {
+         this.processState.setProcessMetaDataAdjustment(new ProcessMetaDataAdjustment());
+      }
+
+      this.processState.getProcessMetaDataAdjustment().setUpdatedFrontendStepList(updatedFrontendStepList);
    }
 
 
@@ -346,7 +352,27 @@ public class RunProcessOutput extends AbstractActionOutput implements Serializab
     *******************************************************************************/
    public List<QFrontendStepMetaData> getUpdatedFrontendStepList()
    {
-      return this.processState.getUpdatedFrontendStepList();
+      return ObjectUtils.tryElse(() -> this.processState.getProcessMetaDataAdjustment().getUpdatedFrontendStepList(), null);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for processMetaDataAdjustment
+    *******************************************************************************/
+   public ProcessMetaDataAdjustment getProcessMetaDataAdjustment()
+   {
+      return (this.processState.getProcessMetaDataAdjustment());
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for processMetaDataAdjustment
+    *******************************************************************************/
+   public void setProcessMetaDataAdjustment(ProcessMetaDataAdjustment processMetaDataAdjustment)
+   {
+      this.processState.setProcessMetaDataAdjustment(processMetaDataAdjustment);
    }
 
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/CriteriaOption.java

@@ -0,0 +1,31 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2025.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.model.actions.tables.query;
+
+
+/*******************************************************************************
+ **
+ *******************************************************************************/
+public enum CriteriaOption implements CriteriaOptionInterface
+{
+   CASE_INSENSITIVE;
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/CriteriaOptionInterface.java

@@ -0,0 +1,30 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2025.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.model.actions.tables.query;
+
+
+/*******************************************************************************
+ **
+ *******************************************************************************/
+public interface CriteriaOptionInterface
+{
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/JoinsContext.java

@@ -82,7 +82,7 @@ public class JoinsContext
    /////////////////////////////////////////////////////////////////////////////
    // we will get a TON of more output if this gets turned up, so be cautious //
    /////////////////////////////////////////////////////////////////////////////
-   private Level logLevel = Level.OFF;
+   private Level logLevel          = Level.OFF;
    private Level logLevelForFilter = Level.OFF;
 
 
@@ -404,6 +404,12 @@ public class JoinsContext
                chainIsInner = false;
             }
 
+            if(hasAllAccessKey(recordSecurityLock))
+            {
+               queryJoin.withType(QueryJoin.Type.LEFT);
+               chainIsInner = false;
+            }
+
             addQueryJoin(queryJoin, "forRecordSecurityLock (non-flipped)", "- ");
             addedQueryJoins.add(queryJoin);
             tmpTable = instance.getTable(join.getRightTable());
@@ -423,6 +429,12 @@ public class JoinsContext
                chainIsInner = false;
             }
 
+            if(hasAllAccessKey(recordSecurityLock))
+            {
+               queryJoin.withType(QueryJoin.Type.LEFT);
+               chainIsInner = false;
+            }
+
             addQueryJoin(queryJoin, "forRecordSecurityLock (flipped)", "- ");
             addedQueryJoins.add(queryJoin);
             tmpTable = instance.getTable(join.getLeftTable());
@@ -456,44 +468,53 @@ public class JoinsContext
 
 
 
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   private boolean hasAllAccessKey(RecordSecurityLock recordSecurityLock)
+   {
+      //////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      // check if the key type has an all-access key, and if so, if it's set to true for the current user/session //
+      //////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      QSecurityKeyType securityKeyType = instance.getSecurityKeyType(recordSecurityLock.getSecurityKeyType());
+      if(StringUtils.hasContent(securityKeyType.getAllAccessKeyName()))
+      {
+         QSession session = QContext.getQSession();
+         if(session.hasSecurityKeyValue(securityKeyType.getAllAccessKeyName(), true, QFieldType.BOOLEAN))
+         {
+            return (true);
+         }
+      }
+
+      return (false);
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/
    private void addSubFilterForRecordSecurityLock(RecordSecurityLock recordSecurityLock, QTableMetaData table, String tableNameOrAlias, boolean isOuter, QueryJoin sourceQueryJoin)
    {
-      QSession session = QContext.getQSession();
-
-      //////////////////////////////////////////////////////////////////////////////////////////////////////////////
-      // check if the key type has an all-access key, and if so, if it's set to true for the current user/session //
-      //////////////////////////////////////////////////////////////////////////////////////////////////////////////
-      QSecurityKeyType securityKeyType  = instance.getSecurityKeyType(recordSecurityLock.getSecurityKeyType());
-      boolean          haveAllAccessKey = false;
-      if(StringUtils.hasContent(securityKeyType.getAllAccessKeyName()))
+      boolean haveAllAccessKey = hasAllAccessKey(recordSecurityLock);
+      if(haveAllAccessKey)
       {
-         //////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-         // if we have all-access on this key, then we don't need a criterion for it (as long as we're in an AND filter) //
-         //////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-         if(session.hasSecurityKeyValue(securityKeyType.getAllAccessKeyName(), true, QFieldType.BOOLEAN))
+         if(sourceQueryJoin != null)
          {
-            haveAllAccessKey = true;
+            ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+            // in case the queryJoin object is re-used between queries, and its security criteria need to be different (!!), reset it //
+            // this can be exposed in tests - maybe not entirely expected in real-world, but seems safe enough                        //
+            ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+            sourceQueryJoin.withSecurityCriteria(new ArrayList<>());
+         }
 
-            if(sourceQueryJoin != null)
-            {
-               ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-               // in case the queryJoin object is re-used between queries, and its security criteria need to be different (!!), reset it //
-               // this can be exposed in tests - maybe not entirely expected in real-world, but seems safe enough                        //
-               ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-               sourceQueryJoin.withSecurityCriteria(new ArrayList<>());
-            }
-
-            ////////////////////////////////////////////////////////////////////////////////////////
-            // if we're in an AND filter, then we don't need a criteria for this lock, so return. //
-            ////////////////////////////////////////////////////////////////////////////////////////
-            boolean inAnAndFilter = securityFilterCursor.getBooleanOperator() == QQueryFilter.BooleanOperator.AND;
-            if(inAnAndFilter)
-            {
-               return;
-            }
+         ////////////////////////////////////////////////////////////////////////////////////////
+         // if we're in an AND filter, then we don't need a criteria for this lock, so return. //
+         ////////////////////////////////////////////////////////////////////////////////////////
+         boolean inAnAndFilter = securityFilterCursor.getBooleanOperator() == QQueryFilter.BooleanOperator.AND;
+         if(inAnAndFilter)
+         {
+            return;
          }
       }
 
@@ -545,7 +566,7 @@ public class JoinsContext
       }
       else
       {
-         List<Serializable> securityKeyValues = session.getSecurityKeyValues(recordSecurityLock.getSecurityKeyType(), type);
+         List<Serializable> securityKeyValues = QContext.getQSession().getSecurityKeyValues(recordSecurityLock.getSecurityKeyType(), type);
          if(CollectionUtils.nullSafeIsEmpty(securityKeyValues))
          {
             ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QFilterCriteria.java

@@ -26,8 +26,10 @@ import java.io.Serializable;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Objects;
+import java.util.Set;
 import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
 import com.kingsrook.qqq.backend.core.logging.QLogger;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.serialization.QFilterCriteriaDeserializer;
@@ -44,7 +46,7 @@ public class QFilterCriteria implements Serializable, Cloneable
 {
    private static final QLogger LOG = QLogger.getLogger(QFilterCriteria.class);
 
-   private String fieldName;
+   private String             fieldName;
    private QCriteriaOperator  operator;
    private List<Serializable> values;
 
@@ -53,6 +55,8 @@ public class QFilterCriteria implements Serializable, Cloneable
    ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
    private String otherFieldName;
 
+   private Set<CriteriaOptionInterface> options = null;
+
 
 
    /*******************************************************************************
@@ -69,6 +73,13 @@ public class QFilterCriteria implements Serializable, Cloneable
             clone.values = new ArrayList<>();
             clone.values.addAll(values);
          }
+
+         if(options != null)
+         {
+            clone.options = new HashSet<>();
+            clone.options.addAll(options);
+         }
+
          return clone;
       }
       catch(CloneNotSupportedException e)
@@ -385,4 +396,78 @@ public class QFilterCriteria implements Serializable, Cloneable
       return Objects.hash(fieldName, operator, values, otherFieldName);
    }
 
+
+
+   /*******************************************************************************
+    ** Getter for options
+    *******************************************************************************/
+   public Set<CriteriaOptionInterface> getOptions()
+   {
+      return (this.options);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for options
+    *******************************************************************************/
+   public void setOptions(Set<CriteriaOptionInterface> options)
+   {
+      this.options = options;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for options
+    *******************************************************************************/
+   public QFilterCriteria withOptions(Set<CriteriaOptionInterface> options)
+   {
+      this.options = options;
+      return (this);
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   public QFilterCriteria withOption(CriteriaOptionInterface option)
+   {
+      if(options == null)
+      {
+         options = new HashSet<>();
+      }
+      options.add(option);
+      return (this);
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   public QFilterCriteria withoutOption(CriteriaOptionInterface option)
+   {
+      if(options != null)
+      {
+         options.remove(option);
+      }
+      return (this);
+   }
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   public boolean hasOption(CriteriaOptionInterface option)
+   {
+      if(options == null)
+      {
+         return (false);
+      }
+
+      return (options.contains(option));
+   }
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QQueryFilter.java

@@ -55,6 +55,16 @@ public class QQueryFilter implements Serializable, Cloneable
    private BooleanOperator    booleanOperator = BooleanOperator.AND;
    private List<QQueryFilter> subFilters      = new ArrayList<>();
 
+   //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+   // initial intent here was - put, e.g., UNION between multiple SELECT (with the individual selects being defined in subFilters) //
+   // but, actually SQL would let us do, e.g., SELECT UNION SELECT INTERSECT SELECT                                                //
+   // so - we could see a future implementation where we:                                                                          //
+   // - used the top-level subFilterSetOperator to indicate hat we are doing a multi-query set-operation query.                    //
+   // - looked within the subFilter, to see if it specified a subFilterSetOperator - and use that operator before that query       //
+   // but - in v0, just using the one at the top-level works                                                                       //
+   //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+   private SubFilterSetOperator subFilterSetOperator = null;
+
    ////////////////////////////////////////////////////////////////////////////////////////////////////////////
    // skip & limit are meant to only apply to QueryAction (at least at the initial time they are added here) //
    // e.g., they are ignored in CountAction, AggregateAction, etc, where their meanings may be less obvious  //
@@ -75,6 +85,19 @@ public class QQueryFilter implements Serializable, Cloneable
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public enum SubFilterSetOperator
+   {
+      UNION,
+      UNION_ALL,
+      INTERSECT,
+      EXCEPT
+   }
+
+
+
    /*******************************************************************************
     ** Constructor
     **
@@ -799,4 +822,51 @@ public class QQueryFilter implements Serializable, Cloneable
       }
    }
 
+
+   /*******************************************************************************
+    ** Getter for subFilterSetOperator
+    *******************************************************************************/
+   public SubFilterSetOperator getSubFilterSetOperator()
+   {
+      return (this.subFilterSetOperator);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for subFilterSetOperator
+    *******************************************************************************/
+   public void setSubFilterSetOperator(SubFilterSetOperator subFilterSetOperator)
+   {
+      this.subFilterSetOperator = subFilterSetOperator;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for subFilterSetOperator
+    *******************************************************************************/
+   public QQueryFilter withSubFilterSetOperator(SubFilterSetOperator subFilterSetOperator)
+   {
+      this.subFilterSetOperator = subFilterSetOperator;
+      return (this);
+   }
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   public void applyCriteriaOptionToAllCriteria(CriteriaOptionInterface criteriaOption)
+   {
+      for(QFilterCriteria criteria : CollectionUtils.nonNullList(this.criteria))
+      {
+         criteria.withOption(criteriaOption);
+      }
+
+      for(QQueryFilter subFilter : CollectionUtils.nonNullList(subFilters))
+      {
+         subFilter.applyCriteriaOptionToAllCriteria(criteriaOption);
+      }
+   }
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/storage/StorageInput.java

@@ -22,15 +22,17 @@
 package com.kingsrook.qqq.backend.core.model.actions.tables.storage;
 
 
+import java.io.Serializable;
 import com.kingsrook.qqq.backend.core.model.actions.AbstractTableActionInput;
 
 
 /*******************************************************************************
  ** Input for Storage actions.
  *******************************************************************************/
-public class StorageInput extends AbstractTableActionInput
+public class StorageInput extends AbstractTableActionInput implements Serializable
 {
    private String reference;
+   private String contentType;
 
 
 
@@ -74,4 +76,35 @@ public class StorageInput extends AbstractTableActionInput
       return (this);
    }
 
+
+
+   /*******************************************************************************
+    ** Getter for contentType
+    *******************************************************************************/
+   public String getContentType()
+   {
+      return (this.contentType);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for contentType
+    *******************************************************************************/
+   public void setContentType(String contentType)
+   {
+      this.contentType = contentType;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for contentType
+    *******************************************************************************/
+   public StorageInput withContentType(String contentType)
+   {
+      this.contentType = contentType;
+      return (this);
+   }
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/values/SearchPossibleValueSourceInput.java

@@ -38,9 +38,10 @@ public class SearchPossibleValueSourceInput extends AbstractActionInput implemen
    private QQueryFilter       defaultQueryFilter;
    private String             searchTerm;
    private List<Serializable> idList;
+   private List<String>       labelList;
 
    private Integer skip  = 0;
-   private Integer limit = 100;
+   private Integer limit = 250;
 
 
 
@@ -281,4 +282,35 @@ public class SearchPossibleValueSourceInput extends AbstractActionInput implemen
       this.limit = limit;
       return (this);
    }
+
+
+   /*******************************************************************************
+    ** Getter for labelList
+    *******************************************************************************/
+   public List<String> getLabelList()
+   {
+      return (this.labelList);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for labelList
+    *******************************************************************************/
+   public void setLabelList(List<String> labelList)
+   {
+      this.labelList = labelList;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for labelList
+    *******************************************************************************/
+   public SearchPossibleValueSourceInput withLabelList(List<String> labelList)
+   {
+      this.labelList = labelList;
+      return (this);
+   }
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/values/SearchPossibleValueSourceOutput.java

@@ -35,6 +35,7 @@ public class SearchPossibleValueSourceOutput extends AbstractActionOutput
 {
    private List<QPossibleValue<?>> results = new ArrayList<>();
 
+   private String warning;
 
 
    /*******************************************************************************
@@ -88,4 +89,35 @@ public class SearchPossibleValueSourceOutput extends AbstractActionOutput
       return (this);
    }
 
+
+   /*******************************************************************************
+    ** Getter for warning
+    *******************************************************************************/
+   public String getWarning()
+   {
+      return (this.warning);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for warning
+    *******************************************************************************/
+   public void setWarning(String warning)
+   {
+      this.warning = warning;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for warning
+    *******************************************************************************/
+   public SearchPossibleValueSourceOutput withWarning(String warning)
+   {
+      this.warning = warning;
+      return (this);
+   }
+
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/dashboard/widgets/AlertData.java

@@ -22,6 +22,9 @@
 package com.kingsrook.qqq.backend.core.model.dashboard.widgets;
 
 
+import java.util.List;
+
+
 /*******************************************************************************
  ** Model containing datastructure expected by frontend alert widget
  **
@@ -40,8 +43,10 @@ public class AlertData extends QWidgetData
 
 
 
-   private String    html;
-   private AlertType alertType;
+   private String       html;
+   private AlertType    alertType;
+   private Boolean      hideWidget = false;
+   private List<String> bulletList;
 
 
 
@@ -139,4 +144,66 @@ public class AlertData extends QWidgetData
       return (this);
    }
 
+
+
+   /*******************************************************************************
+    ** Getter for hideWidget
+    *******************************************************************************/
+   public boolean getHideWidget()
+   {
+      return (this.hideWidget);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for hideWidget
+    *******************************************************************************/
+   public void setHideWidget(boolean hideWidget)
+   {
+      this.hideWidget = hideWidget;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for hideWidget
+    *******************************************************************************/
+   public AlertData withHideWidget(boolean hideWidget)
+   {
+      this.hideWidget = hideWidget;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for bulletList
+    *******************************************************************************/
+   public List<String> getBulletList()
+   {
+      return (this.bulletList);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for bulletList
+    *******************************************************************************/
+   public void setBulletList(List<String> bulletList)
+   {
+      this.bulletList = bulletList;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for bulletList
+    *******************************************************************************/
+   public AlertData withBulletList(List<String> bulletList)
+   {
+      this.bulletList = bulletList;
+      return (this);
+   }
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/dashboard/widgets/ChildRecordListData.java

@@ -39,13 +39,19 @@ public class ChildRecordListData extends QWidgetData
    private QueryOutput    queryOutput;
    private QTableMetaData childTableMetaData;
 
+   private String  tableName;
    private String  tablePath;
    private String  viewAllLink;
    private Integer totalRows;
+   private Boolean disableRowClick   = false;
+   private Boolean allowRecordEdit   = false;
+   private Boolean allowRecordDelete = false;
+   private Boolean isInProcess       = false;
 
    private boolean                   canAddChildRecord = false;
    private Map<String, Serializable> defaultValuesForNewChildRecords;
    private Set<String>               disabledFieldsForNewChildRecords;
+   private Map<String, String>       defaultValuesForNewChildRecordsFromParentFields;
 
 
 
@@ -352,4 +358,204 @@ public class ChildRecordListData extends QWidgetData
       return (this);
    }
 
+
+
+   /*******************************************************************************
+    ** Getter for tableName
+    *******************************************************************************/
+   public String getTableName()
+   {
+      return (this.tableName);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for tableName
+    *******************************************************************************/
+   public void setTableName(String tableName)
+   {
+      this.tableName = tableName;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for tableName
+    *******************************************************************************/
+   public ChildRecordListData withTableName(String tableName)
+   {
+      this.tableName = tableName;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for tablePath
+    *******************************************************************************/
+   public ChildRecordListData withTablePath(String tablePath)
+   {
+      this.tablePath = tablePath;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for disableRowClick
+    *******************************************************************************/
+   public Boolean getDisableRowClick()
+   {
+      return (this.disableRowClick);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for disableRowClick
+    *******************************************************************************/
+   public void setDisableRowClick(Boolean disableRowClick)
+   {
+      this.disableRowClick = disableRowClick;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for disableRowClick
+    *******************************************************************************/
+   public ChildRecordListData withDisableRowClick(Boolean disableRowClick)
+   {
+      this.disableRowClick = disableRowClick;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for allowRecordEdit
+    *******************************************************************************/
+   public Boolean getAllowRecordEdit()
+   {
+      return (this.allowRecordEdit);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for allowRecordEdit
+    *******************************************************************************/
+   public void setAllowRecordEdit(Boolean allowRecordEdit)
+   {
+      this.allowRecordEdit = allowRecordEdit;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for allowRecordEdit
+    *******************************************************************************/
+   public ChildRecordListData withAllowRecordEdit(Boolean allowRecordEdit)
+   {
+      this.allowRecordEdit = allowRecordEdit;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for allowRecordDelete
+    *******************************************************************************/
+   public Boolean getAllowRecordDelete()
+   {
+      return (this.allowRecordDelete);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for allowRecordDelete
+    *******************************************************************************/
+   public void setAllowRecordDelete(Boolean allowRecordDelete)
+   {
+      this.allowRecordDelete = allowRecordDelete;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for allowRecordDelete
+    *******************************************************************************/
+   public ChildRecordListData withAllowRecordDelete(Boolean allowRecordDelete)
+   {
+      this.allowRecordDelete = allowRecordDelete;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for isInProcess
+    *******************************************************************************/
+   public Boolean getIsInProcess()
+   {
+      return (this.isInProcess);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for isInProcess
+    *******************************************************************************/
+   public void setIsInProcess(Boolean isInProcess)
+   {
+      this.isInProcess = isInProcess;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for isInProcess
+    *******************************************************************************/
+   public ChildRecordListData withIsInProcess(Boolean isInProcess)
+   {
+      this.isInProcess = isInProcess;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for defaultValuesForNewChildRecordsFromParentFields
+    *******************************************************************************/
+   public Map<String, String> getDefaultValuesForNewChildRecordsFromParentFields()
+   {
+      return (this.defaultValuesForNewChildRecordsFromParentFields);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for defaultValuesForNewChildRecordsFromParentFields
+    *******************************************************************************/
+   public void setDefaultValuesForNewChildRecordsFromParentFields(Map<String, String> defaultValuesForNewChildRecordsFromParentFields)
+   {
+      this.defaultValuesForNewChildRecordsFromParentFields = defaultValuesForNewChildRecordsFromParentFields;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for defaultValuesForNewChildRecordsFromParentFields
+    *******************************************************************************/
+   public ChildRecordListData withDefaultValuesForNewChildRecordsFromParentFields(Map<String, String> defaultValuesForNewChildRecordsFromParentFields)
+   {
+      this.defaultValuesForNewChildRecordsFromParentFields = defaultValuesForNewChildRecordsFromParentFields;
+      return (this);
+   }
+
 }
+
+
+

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/dashboard/widgets/CompositeWidgetData.java

@@ -40,6 +40,20 @@ public class CompositeWidgetData extends AbstractBlockWidgetData<CompositeWidget
 {
    private List<AbstractBlockWidgetData<?, ?, ?, ?>> blocks = new ArrayList<>();
 
+   private ModalMode modalMode;
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   public enum ModalMode
+   {
+      MODAL
+   }
+
+
+
    private Layout                    layout;
    private Map<String, Serializable> styleOverrides        = new HashMap<>();
    private String                    overlayHtml;
@@ -52,12 +66,14 @@ public class CompositeWidgetData extends AbstractBlockWidgetData<CompositeWidget
     *******************************************************************************/
    public enum Layout
    {
-      /////////////////////////////////////////////////////////////
-      // note, these are used in QQQ FMD CompositeWidgetData.tsx //
-      /////////////////////////////////////////////////////////////
+      /////////////////////////////////////////////////////////
+      // note, these are used in QQQ FMD CompositeWidget.tsx //
+      // and qqq-android CompositeWidgetBlock.kt             //
+      /////////////////////////////////////////////////////////
       FLEX_COLUMN,
       FLEX_ROW_WRAPPED,
       FLEX_ROW_SPACE_BETWEEN,
+      FLEX_ROW_CENTER,
       TABLE_SUB_ROW_DETAILS,
       BADGES_WRAPPER
    }
@@ -306,4 +322,35 @@ public class CompositeWidgetData extends AbstractBlockWidgetData<CompositeWidget
       this.overlayStyleOverrides.put(key, value);
    }
 
+
+
+   /*******************************************************************************
+    ** Getter for modalMode
+    *******************************************************************************/
+   public ModalMode getModalMode()
+   {
+      return (this.modalMode);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for modalMode
+    *******************************************************************************/
+   public void setModalMode(ModalMode modalMode)
+   {
+      this.modalMode = modalMode;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for modalMode
+    *******************************************************************************/
+   public CompositeWidgetData withModalMode(ModalMode modalMode)
+   {
+      this.modalMode = modalMode;
+      return (this);
+   }
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/dashboard/widgets/FilterAndColumnsSetupData.java

@@ -34,8 +34,12 @@ public class FilterAndColumnsSetupData extends QWidgetData
    private String       tableName;
    private Boolean      allowVariables = false;
    private Boolean      hideColumns    = false;
+   private Boolean      hidePreview    = false;
    private List<String> filterDefaultFieldNames;
 
+   private String filterFieldName = "queryFilterJson";
+   private String columnFieldName = "columnsJson";
+
 
 
    /*******************************************************************************
@@ -193,4 +197,97 @@ public class FilterAndColumnsSetupData extends QWidgetData
       return (this);
    }
 
+
+
+   /*******************************************************************************
+    ** Getter for hidePreview
+    *******************************************************************************/
+   public Boolean getHidePreview()
+   {
+      return (this.hidePreview);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for hidePreview
+    *******************************************************************************/
+   public void setHidePreview(Boolean hidePreview)
+   {
+      this.hidePreview = hidePreview;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for hidePreview
+    *******************************************************************************/
+   public FilterAndColumnsSetupData withHidePreview(Boolean hidePreview)
+   {
+      this.hidePreview = hidePreview;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for filterFieldName
+    *******************************************************************************/
+   public String getFilterFieldName()
+   {
+      return (this.filterFieldName);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for filterFieldName
+    *******************************************************************************/
+   public void setFilterFieldName(String filterFieldName)
+   {
+      this.filterFieldName = filterFieldName;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for filterFieldName
+    *******************************************************************************/
+   public FilterAndColumnsSetupData withFilterFieldName(String filterFieldName)
+   {
+      this.filterFieldName = filterFieldName;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for columnFieldName
+    *******************************************************************************/
+   public String getColumnFieldName()
+   {
+      return (this.columnFieldName);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for columnFieldName
+    *******************************************************************************/
+   public void setColumnFieldName(String columnFieldName)
+   {
+      this.columnFieldName = columnFieldName;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for columnFieldName
+    *******************************************************************************/
+   public FilterAndColumnsSetupData withColumnFieldName(String columnFieldName)
+   {
+      this.columnFieldName = columnFieldName;
+      return (this);
+   }
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/dashboard/widgets/QWidgetData.java

@@ -31,7 +31,7 @@ import java.util.Map;
  ** Base class for the data returned by rendering a Widget.
  **
  *******************************************************************************/
-public abstract class QWidgetData
+public abstract class QWidgetData implements Serializable
 {
    private String       label;
    private String       sublabel;

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/dashboard/widgets/blocks/AbstractBlockWidgetData.java

@@ -52,6 +52,11 @@ public abstract class AbstractBlockWidgetData<
    private V  values;
    private SX styles;
 
+   ///////////////////////////////////////////////////////////////////////////////////
+   // optional field name to act as a 'guard' for the block - e.g., only include it //
+   // if the value for this field is true                                           //
+   ///////////////////////////////////////////////////////////////////////////////////
+   private String conditional;
 
 
    /*******************************************************************************
@@ -443,4 +448,35 @@ public abstract class AbstractBlockWidgetData<
       return (T) this;
    }
 
+
+   /*******************************************************************************
+    ** Getter for conditional
+    *******************************************************************************/
+   public String getConditional()
+   {
+      return (this.conditional);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for conditional
+    *******************************************************************************/
+   public void setConditional(String conditional)
+   {
+      this.conditional = conditional;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for conditional
+    *******************************************************************************/
+   public AbstractBlockWidgetData withConditional(String conditional)
+   {
+      this.conditional = conditional;
+      return (this);
+   }
+
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/dashboard/widgets/blocks/audio/AudioBlockData.java

@@ -0,0 +1,45 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.model.dashboard.widgets.blocks.audio;
+
+
+import com.kingsrook.qqq.backend.core.model.dashboard.widgets.blocks.AbstractBlockWidgetData;
+import com.kingsrook.qqq.backend.core.model.dashboard.widgets.blocks.base.BaseSlots;
+import com.kingsrook.qqq.backend.core.model.dashboard.widgets.blocks.base.BaseStyles;
+
+
+/*******************************************************************************
+ ** block that plays an audio file
+ *******************************************************************************/
+public class AudioBlockData extends AbstractBlockWidgetData<AudioBlockData, AudioValues, BaseSlots, BaseStyles>
+{
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public String getBlockTypeName()
+   {
+      return "AUDIO";
+   }
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/dashboard/widgets/blocks/audio/AudioValues.java

@@ -0,0 +1,130 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.model.dashboard.widgets.blocks.audio;
+
+
+import com.kingsrook.qqq.backend.core.model.dashboard.widgets.blocks.BlockValuesInterface;
+
+
+/*******************************************************************************
+ **
+ *******************************************************************************/
+public class AudioValues implements BlockValuesInterface
+{
+   private String  path;
+   private boolean showControls = false;
+   private boolean autoPlay     = true;
+
+
+
+   /*******************************************************************************
+    ** Getter for path
+    *******************************************************************************/
+   public String getPath()
+   {
+      return (this.path);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for path
+    *******************************************************************************/
+   public void setPath(String path)
+   {
+      this.path = path;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for path
+    *******************************************************************************/
+   public AudioValues withPath(String path)
+   {
+      this.path = path;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for showControls
+    *******************************************************************************/
+   public boolean getShowControls()
+   {
+      return (this.showControls);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for showControls
+    *******************************************************************************/
+   public void setShowControls(boolean showControls)
+   {
+      this.showControls = showControls;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for showControls
+    *******************************************************************************/
+   public AudioValues withShowControls(boolean showControls)
+   {
+      this.showControls = showControls;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for autoPlay
+    *******************************************************************************/
+   public boolean getAutoPlay()
+   {
+      return (this.autoPlay);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for autoPlay
+    *******************************************************************************/
+   public void setAutoPlay(boolean autoPlay)
+   {
+      this.autoPlay = autoPlay;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for autoPlay
+    *******************************************************************************/
+   public AudioValues withAutoPlay(boolean autoPlay)
+   {
+      this.autoPlay = autoPlay;
+      return (this);
+   }
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/dashboard/widgets/blocks/base/BaseStyles.java

@@ -30,4 +30,335 @@ import com.kingsrook.qqq.backend.core.model.dashboard.widgets.blocks.BlockStyles
  *******************************************************************************/
 public class BaseStyles implements BlockStylesInterface
 {
+   private Directional<String> padding;
+
+   private String backgroundColor;
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   public static class Directional<T>
+   {
+      private T top;
+      private T bottom;
+      private T left;
+      private T right;
+
+
+
+      /*******************************************************************************
+       ** Constructor
+       **
+       *******************************************************************************/
+      public Directional()
+      {
+      }
+
+
+
+      /*******************************************************************************
+       ** Constructor
+       **
+       *******************************************************************************/
+      public Directional(T top, T right, T bottom, T left)
+      {
+         this.top = top;
+         this.right = right;
+         this.bottom = bottom;
+         this.left = left;
+      }
+
+
+
+      /***************************************************************************
+       **
+       ***************************************************************************/
+      public static <T> Directional<T> of(T top, T right, T bottom, T left)
+      {
+         return (new Directional<>(top, right, bottom, left));
+      }
+
+
+
+
+      /***************************************************************************
+       **
+       ***************************************************************************/
+      public static <T> Directional<T> of(T value)
+      {
+         return (new Directional<>(value, value, value, value));
+      }
+
+
+
+      /***************************************************************************
+       **
+       ***************************************************************************/
+      public static <T> Directional<T> ofTop(T top)
+      {
+         return (new Directional<>(top, null, null, null));
+      }
+
+
+
+      /***************************************************************************
+       **
+       ***************************************************************************/
+      public static <T> Directional<T> ofRight(T right)
+      {
+         return (new Directional<>(null, right, null, null));
+      }
+
+
+
+      /***************************************************************************
+       **
+       ***************************************************************************/
+      public static <T> Directional<T> ofBottom(T bottom)
+      {
+         return (new Directional<>(null, null, bottom, null));
+      }
+
+
+
+      /***************************************************************************
+       **
+       ***************************************************************************/
+      public static <T> Directional<T> ofLeft(T left)
+      {
+         return (new Directional<>(null, null, null, left));
+      }
+
+
+
+      /***************************************************************************
+       **
+       ***************************************************************************/
+      public static <T> Directional<T> ofX(T x)
+      {
+         return (new Directional<>(null, x, null, x));
+      }
+
+
+
+      /***************************************************************************
+       **
+       ***************************************************************************/
+      public static <T> Directional<T> ofY(T y)
+      {
+         return (new Directional<>(y, null, y, null));
+      }
+
+
+
+      /***************************************************************************
+       **
+       ***************************************************************************/
+      public static <T> Directional<T> ofXY(T x, T y)
+      {
+         return (new Directional<>(y, x, y, x));
+      }
+
+
+
+      /*******************************************************************************
+       ** Getter for top
+       **
+       *******************************************************************************/
+      public T getTop()
+      {
+         return top;
+      }
+
+
+
+      /*******************************************************************************
+       ** Setter for top
+       **
+       *******************************************************************************/
+      public void setTop(T top)
+      {
+         this.top = top;
+      }
+
+
+
+      /*******************************************************************************
+       ** Fluent setter for top
+       **
+       *******************************************************************************/
+      public Directional<T> withTop(T top)
+      {
+         this.top = top;
+         return (this);
+      }
+
+
+
+      /*******************************************************************************
+       ** Getter for bottom
+       **
+       *******************************************************************************/
+      public T getBottom()
+      {
+         return bottom;
+      }
+
+
+
+      /*******************************************************************************
+       ** Setter for bottom
+       **
+       *******************************************************************************/
+      public void setBottom(T bottom)
+      {
+         this.bottom = bottom;
+      }
+
+
+
+      /*******************************************************************************
+       ** Fluent setter for bottom
+       **
+       *******************************************************************************/
+      public Directional<T> withBottom(T bottom)
+      {
+         this.bottom = bottom;
+         return (this);
+      }
+
+
+
+      /*******************************************************************************
+       ** Getter for left
+       **
+       *******************************************************************************/
+      public T getLeft()
+      {
+         return left;
+      }
+
+
+
+      /*******************************************************************************
+       ** Setter for left
+       **
+       *******************************************************************************/
+      public void setLeft(T left)
+      {
+         this.left = left;
+      }
+
+
+
+      /*******************************************************************************
+       ** Fluent setter for left
+       **
+       *******************************************************************************/
+      public Directional<T> withLeft(T left)
+      {
+         this.left = left;
+         return (this);
+      }
+
+
+
+      /*******************************************************************************
+       ** Getter for right
+       **
+       *******************************************************************************/
+      public T getRight()
+      {
+         return right;
+      }
+
+
+
+      /*******************************************************************************
+       ** Setter for right
+       **
+       *******************************************************************************/
+      public void setRight(T right)
+      {
+         this.right = right;
+      }
+
+
+
+      /*******************************************************************************
+       ** Fluent setter for right
+       **
+       *******************************************************************************/
+      public Directional<T> withRight(T right)
+      {
+         this.right = right;
+         return (this);
+      }
+
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for padding
+    *******************************************************************************/
+   public Directional<String> getPadding()
+   {
+      return (this.padding);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for padding
+    *******************************************************************************/
+   public void setPadding(Directional<String> padding)
+   {
+      this.padding = padding;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for padding
+    *******************************************************************************/
+   public BaseStyles withPadding(Directional<String> padding)
+   {
+      this.padding = padding;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for backgroundColor
+    *******************************************************************************/
+   public String getBackgroundColor()
+   {
+      return (this.backgroundColor);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for backgroundColor
+    *******************************************************************************/
+   public void setBackgroundColor(String backgroundColor)
+   {
+      this.backgroundColor = backgroundColor;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for backgroundColor
+    *******************************************************************************/
+   public BaseStyles withBackgroundColor(String backgroundColor)
+   {
+      this.backgroundColor = backgroundColor;
+      return (this);
+   }
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/dashboard/widgets/blocks/button/ButtonBlockData.java

@@ -0,0 +1,46 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.model.dashboard.widgets.blocks.button;
+
+
+import com.kingsrook.qqq.backend.core.model.dashboard.widgets.blocks.AbstractBlockWidgetData;
+import com.kingsrook.qqq.backend.core.model.dashboard.widgets.blocks.base.BaseSlots;
+
+
+/*******************************************************************************
+ ** a button (for a process - not sure yet what this could do in a standalone
+ ** widget?) to submit the process screen to run a specific action (e.g., not just
+ ** 'next'), or do other control-ish things
+ *******************************************************************************/
+public class ButtonBlockData extends AbstractBlockWidgetData<ButtonBlockData, ButtonValues, BaseSlots, ButtonStyles>
+{
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public String getBlockTypeName()
+   {
+      return "BUTTON";
+   }
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/dashboard/widgets/blocks/button/ButtonStyles.java

@@ -0,0 +1,143 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.model.dashboard.widgets.blocks.button;
+
+
+import com.kingsrook.qqq.backend.core.model.dashboard.widgets.blocks.BlockStylesInterface;
+
+
+/*******************************************************************************
+ **
+ *******************************************************************************/
+public class ButtonStyles implements BlockStylesInterface
+{
+   private String color;
+   private String format;
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   public enum StandardColor
+   {
+      SUCCESS,
+      WARNING,
+      ERROR,
+      INFO,
+      MUTED
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   public enum StandardFormat
+   {
+      OUTLINED,
+      FILLED,
+      TEXT
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for color
+    *******************************************************************************/
+   public String getColor()
+   {
+      return (this.color);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for color
+    *******************************************************************************/
+   public void setColor(String color)
+   {
+      this.color = color;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for color
+    *******************************************************************************/
+   public ButtonStyles withColor(String color)
+   {
+      this.color = color;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for format
+    *******************************************************************************/
+   public String getFormat()
+   {
+      return (this.format);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for format
+    *******************************************************************************/
+   public void setFormat(String format)
+   {
+      this.format = format;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for format
+    *******************************************************************************/
+   public ButtonStyles withFormat(String format)
+   {
+      this.format = format;
+      return (this);
+   }
+
+   /*******************************************************************************
+    ** Setter for format
+    *******************************************************************************/
+   public void setFormat(StandardFormat format)
+   {
+      this.format = (format == null ? null : format.name().toLowerCase());
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for format
+    *******************************************************************************/
+   public ButtonStyles withFormat(StandardFormat format)
+   {
+      setFormat(format);
+      return (this);
+   }
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/dashboard/widgets/blocks/button/ButtonValues.java

@@ -0,0 +1,218 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.model.dashboard.widgets.blocks.button;
+
+
+import com.kingsrook.qqq.backend.core.model.dashboard.widgets.blocks.BlockValuesInterface;
+import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon;
+
+
+/*******************************************************************************
+ **
+ *******************************************************************************/
+public class ButtonValues implements BlockValuesInterface
+{
+   private String label;
+   private String actionCode;
+   private String controlCode;
+
+   private QIcon startIcon;
+   private QIcon endIcon;
+
+
+
+   /*******************************************************************************
+    ** Constructor
+    **
+    *******************************************************************************/
+   public ButtonValues()
+   {
+   }
+
+
+
+   /*******************************************************************************
+    ** Constructor
+    **
+    *******************************************************************************/
+   public ButtonValues(String label, String actionCode)
+   {
+      setLabel(label);
+      setActionCode(actionCode);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for label
+    *******************************************************************************/
+   public String getLabel()
+   {
+      return (this.label);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for label
+    *******************************************************************************/
+   public void setLabel(String label)
+   {
+      this.label = label;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for label
+    *******************************************************************************/
+   public ButtonValues withLabel(String label)
+   {
+      this.label = label;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for actionCode
+    *******************************************************************************/
+   public String getActionCode()
+   {
+      return (this.actionCode);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for actionCode
+    *******************************************************************************/
+   public void setActionCode(String actionCode)
+   {
+      this.actionCode = actionCode;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for actionCode
+    *******************************************************************************/
+   public ButtonValues withActionCode(String actionCode)
+   {
+      this.actionCode = actionCode;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for startIcon
+    *******************************************************************************/
+   public QIcon getStartIcon()
+   {
+      return (this.startIcon);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for startIcon
+    *******************************************************************************/
+   public void setStartIcon(QIcon startIcon)
+   {
+      this.startIcon = startIcon;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for startIcon
+    *******************************************************************************/
+   public ButtonValues withStartIcon(QIcon startIcon)
+   {
+      this.startIcon = startIcon;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for endIcon
+    *******************************************************************************/
+   public QIcon getEndIcon()
+   {
+      return (this.endIcon);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for endIcon
+    *******************************************************************************/
+   public void setEndIcon(QIcon endIcon)
+   {
+      this.endIcon = endIcon;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for endIcon
+    *******************************************************************************/
+   public ButtonValues withEndIcon(QIcon endIcon)
+   {
+      this.endIcon = endIcon;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for controlCode
+    *******************************************************************************/
+   public String getControlCode()
+   {
+      return (this.controlCode);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for controlCode
+    *******************************************************************************/
+   public void setControlCode(String controlCode)
+   {
+      this.controlCode = controlCode;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for controlCode
+    *******************************************************************************/
+   public ButtonValues withControlCode(String controlCode)
+   {
+      this.controlCode = controlCode;
+      return (this);
+   }
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/dashboard/widgets/blocks/image/ImageBlockData.java

@@ -0,0 +1,44 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.model.dashboard.widgets.blocks.image;
+
+
+import com.kingsrook.qqq.backend.core.model.dashboard.widgets.blocks.AbstractBlockWidgetData;
+import com.kingsrook.qqq.backend.core.model.dashboard.widgets.blocks.base.BaseSlots;
+
+
+/*******************************************************************************
+ ** block to display an image
+ *******************************************************************************/
+public class ImageBlockData extends AbstractBlockWidgetData<ImageBlockData, ImageValues, BaseSlots, ImageStyles>
+{
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public String getBlockTypeName()
+   {
+      return "IMAGE";
+   }
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/dashboard/widgets/blocks/image/ImageStyles.java

@@ -0,0 +1,108 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.model.dashboard.widgets.blocks.image;
+
+
+import com.kingsrook.qqq.backend.core.model.dashboard.widgets.blocks.base.BaseStyles;
+
+
+/*******************************************************************************
+ **
+ *******************************************************************************/
+public class ImageStyles extends BaseStyles
+{
+   private String width;
+   private String height;
+
+
+   /*******************************************************************************
+    ** Fluent setter for padding
+    *******************************************************************************/
+   @Override
+   public ImageStyles withPadding(Directional<String> padding)
+   {
+      super.setPadding(padding);
+      return (this);
+   }
+
+
+   /*******************************************************************************
+    ** Getter for width
+    *******************************************************************************/
+   public String getWidth()
+   {
+      return (this.width);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for width
+    *******************************************************************************/
+   public void setWidth(String width)
+   {
+      this.width = width;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for width
+    *******************************************************************************/
+   public ImageStyles withWidth(String width)
+   {
+      this.width = width;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for height
+    *******************************************************************************/
+   public String getHeight()
+   {
+      return (this.height);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for height
+    *******************************************************************************/
+   public void setHeight(String height)
+   {
+      this.height = height;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for height
+    *******************************************************************************/
+   public ImageStyles withHeight(String height)
+   {
+      this.height = height;
+      return (this);
+   }
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/dashboard/widgets/blocks/image/ImageValues.java

@@ -0,0 +1,98 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.model.dashboard.widgets.blocks.image;
+
+
+import com.kingsrook.qqq.backend.core.model.dashboard.widgets.blocks.BlockValuesInterface;
+
+
+/*******************************************************************************
+ **
+ *******************************************************************************/
+public class ImageValues implements BlockValuesInterface
+{
+   private String path;
+   private String alt;
+
+
+
+   /*******************************************************************************
+    ** Getter for path
+    *******************************************************************************/
+   public String getPath()
+   {
+      return (this.path);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for path
+    *******************************************************************************/
+   public void setPath(String path)
+   {
+      this.path = path;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for path
+    *******************************************************************************/
+   public ImageValues withPath(String path)
+   {
+      this.path = path;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for alt
+    *******************************************************************************/
+   public String getAlt()
+   {
+      return (this.alt);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for alt
+    *******************************************************************************/
+   public void setAlt(String alt)
+   {
+      this.alt = alt;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for alt
+    *******************************************************************************/
+   public ImageValues withAlt(String alt)
+   {
+      this.alt = alt;
+      return (this);
+   }
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/dashboard/widgets/blocks/inputfield/InputFieldBlockData.java

@@ -0,0 +1,45 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.model.dashboard.widgets.blocks.inputfield;
+
+
+import com.kingsrook.qqq.backend.core.model.dashboard.widgets.blocks.AbstractBlockWidgetData;
+import com.kingsrook.qqq.backend.core.model.dashboard.widgets.blocks.base.BaseSlots;
+import com.kingsrook.qqq.backend.core.model.dashboard.widgets.blocks.base.BaseStyles;
+
+
+/*******************************************************************************
+ ** block to display an input field - initially targeted at widgets-in-processes
+ *******************************************************************************/
+public class InputFieldBlockData extends AbstractBlockWidgetData<InputFieldBlockData, InputFieldValues, BaseSlots, BaseStyles>
+{
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public String getBlockTypeName()
+   {
+      return "INPUT_FIELD";
+   }
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/dashboard/widgets/blocks/inputfield/InputFieldValues.java

@@ -0,0 +1,217 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.model.dashboard.widgets.blocks.inputfield;
+
+
+import com.kingsrook.qqq.backend.core.model.dashboard.widgets.blocks.BlockValuesInterface;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
+
+
+/*******************************************************************************
+ **
+ *******************************************************************************/
+public class InputFieldValues implements BlockValuesInterface
+{
+   private QFieldMetaData fieldMetaData;
+
+   private Boolean autoFocus;
+   private Boolean submitOnEnter;
+   private Boolean hideSoftKeyboard;
+   private String  placeholder;
+
+
+
+   /*******************************************************************************
+    ** Constructor
+    **
+    *******************************************************************************/
+   public InputFieldValues()
+   {
+   }
+
+
+
+   /*******************************************************************************
+    ** Constructor
+    **
+    *******************************************************************************/
+   public InputFieldValues(QFieldMetaData fieldMetaData)
+   {
+      setFieldMetaData(fieldMetaData);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for fieldMetaData
+    *******************************************************************************/
+   public QFieldMetaData getFieldMetaData()
+   {
+      return (this.fieldMetaData);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for fieldMetaData
+    *******************************************************************************/
+   public void setFieldMetaData(QFieldMetaData fieldMetaData)
+   {
+      this.fieldMetaData = fieldMetaData;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for fieldMetaData
+    *******************************************************************************/
+   public InputFieldValues withFieldMetaData(QFieldMetaData fieldMetaData)
+   {
+      this.fieldMetaData = fieldMetaData;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for autoFocus
+    *******************************************************************************/
+   public Boolean getAutoFocus()
+   {
+      return (this.autoFocus);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for autoFocus
+    *******************************************************************************/
+   public void setAutoFocus(Boolean autoFocus)
+   {
+      this.autoFocus = autoFocus;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for autoFocus
+    *******************************************************************************/
+   public InputFieldValues withAutoFocus(Boolean autoFocus)
+   {
+      this.autoFocus = autoFocus;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for submitOnEnter
+    *******************************************************************************/
+   public Boolean getSubmitOnEnter()
+   {
+      return (this.submitOnEnter);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for submitOnEnter
+    *******************************************************************************/
+   public void setSubmitOnEnter(Boolean submitOnEnter)
+   {
+      this.submitOnEnter = submitOnEnter;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for submitOnEnter
+    *******************************************************************************/
+   public InputFieldValues withSubmitOnEnter(Boolean submitOnEnter)
+   {
+      this.submitOnEnter = submitOnEnter;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for placeholder
+    *******************************************************************************/
+   public String getPlaceholder()
+   {
+      return (this.placeholder);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for placeholder
+    *******************************************************************************/
+   public void setPlaceholder(String placeholder)
+   {
+      this.placeholder = placeholder;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for placeholder
+    *******************************************************************************/
+   public InputFieldValues withPlaceholder(String placeholder)
+   {
+      this.placeholder = placeholder;
+      return (this);
+   }
+
+
+   /*******************************************************************************
+    ** Getter for hideSoftKeyboard
+    *******************************************************************************/
+   public Boolean getHideSoftKeyboard()
+   {
+      return (this.hideSoftKeyboard);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for hideSoftKeyboard
+    *******************************************************************************/
+   public void setHideSoftKeyboard(Boolean hideSoftKeyboard)
+   {
+      this.hideSoftKeyboard = hideSoftKeyboard;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for hideSoftKeyboard
+    *******************************************************************************/
+   public InputFieldValues withHideSoftKeyboard(Boolean hideSoftKeyboard)
+   {
+      this.hideSoftKeyboard = hideSoftKeyboard;
+      return (this);
+   }
+
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/dashboard/widgets/blocks/text/TextStyles.java

@@ -30,4 +30,325 @@ import com.kingsrook.qqq.backend.core.model.dashboard.widgets.blocks.BlockStyles
  *******************************************************************************/
 public class TextStyles implements BlockStylesInterface
 {
+   private String color;
+   private String format;
+   private String weight;
+   private String size;
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   public enum StandardColor
+   {
+      SUCCESS,
+      WARNING,
+      ERROR,
+      INFO,
+      MUTED
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   public enum StandardFormat
+   {
+      DEFAULT,
+      ALERT,
+      BANNER
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   public enum StandardSize
+   {
+      LARGEST,
+      HEADLINE,
+      TITLE,
+      BODY,
+      SMALLEST
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   public enum StandardWeight
+   {
+      EXTRA_LIGHT("extralight"),
+      THIN("thin"),
+      MEDIUM("medium"),
+      SEMI_BOLD("semibold"),
+      BLACK("black"),
+      BOLD("bold"),
+      EXTRA_BOLD("extrabold"),
+      W100("100"),
+      W200("200"),
+      W300("300"),
+      W400("400"),
+      W500("500"),
+      W600("600"),
+      W700("700"),
+      W800("800"),
+      W900("900");
+
+      private final String value;
+
+
+
+      /*******************************************************************************
+       ** Constructor
+       **
+       *******************************************************************************/
+      StandardWeight(String value)
+      {
+         this.value = value;
+      }
+
+
+
+      /*******************************************************************************
+       ** Getter for value
+       **
+       *******************************************************************************/
+      public String getValue()
+      {
+         return value;
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** Constructor
+    **
+    *******************************************************************************/
+   public TextStyles()
+   {
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   public TextStyles(StandardColor standardColor)
+   {
+      setColor(standardColor);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for format
+    *******************************************************************************/
+   public String getFormat()
+   {
+      return (this.format);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for format
+    *******************************************************************************/
+   public void setFormat(String format)
+   {
+      this.format = format;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for format
+    *******************************************************************************/
+   public TextStyles withFormat(String format)
+   {
+      this.format = format;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for format
+    *******************************************************************************/
+   public void setFormat(StandardFormat format)
+   {
+      this.format = format == null ? null : format.name().toLowerCase();
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for format
+    *******************************************************************************/
+   public TextStyles withFormat(StandardFormat format)
+   {
+      this.setFormat(format);
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for weight
+    *******************************************************************************/
+   public String getWeight()
+   {
+      return (this.weight);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for weight
+    *******************************************************************************/
+   public void setWeight(String weight)
+   {
+      this.weight = weight;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for weight
+    *******************************************************************************/
+   public TextStyles withWeight(String weight)
+   {
+      this.weight = weight;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for weight
+    *******************************************************************************/
+   public void setWeight(StandardWeight weight)
+   {
+      setWeight(weight == null ? null : weight.getValue());
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for weight
+    *******************************************************************************/
+   public TextStyles withWeight(StandardWeight weight)
+   {
+      setWeight(weight);
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for size
+    *******************************************************************************/
+   public String getSize()
+   {
+      return (this.size);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for size
+    *******************************************************************************/
+   public void setSize(String size)
+   {
+      this.size = size;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for size
+    *******************************************************************************/
+   public TextStyles withSize(String size)
+   {
+      this.size = size;
+      return (this);
+   }
+
+
+   /*******************************************************************************
+    ** Setter for size
+    *******************************************************************************/
+   public void setSize(StandardSize size)
+   {
+      this.size = (size == null ? null : size.name().toLowerCase());
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for size
+    *******************************************************************************/
+   public TextStyles withSize(StandardSize size)
+   {
+      setSize(size);
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for color
+    *******************************************************************************/
+   public String getColor()
+   {
+      return (this.color);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for color
+    *******************************************************************************/
+   public void setColor(String color)
+   {
+      this.color = color;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for color
+    *******************************************************************************/
+   public TextStyles withColor(String color)
+   {
+      this.color = color;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for color
+    *******************************************************************************/
+   public void setColor(StandardColor color)
+   {
+      this.color = color == null ? null : color.name();
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for color
+    *******************************************************************************/
+   public TextStyles withColor(StandardColor color)
+   {
+      setColor(color);
+      return (this);
+   }
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/dashboard/widgets/blocks/text/TextValues.java

@@ -23,6 +23,7 @@ package com.kingsrook.qqq.backend.core.model.dashboard.widgets.blocks.text;
 
 
 import com.kingsrook.qqq.backend.core.model.dashboard.widgets.blocks.BlockValuesInterface;
+import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon;
 
 
 /*******************************************************************************
@@ -32,6 +33,9 @@ public class TextValues implements BlockValuesInterface
 {
    private String text;
 
+   private QIcon startIcon;
+   private QIcon endIcon;
+
 
 
    /*******************************************************************************
@@ -84,4 +88,66 @@ public class TextValues implements BlockValuesInterface
       return (this);
    }
 
+
+   /*******************************************************************************
+    ** Getter for startIcon
+    *******************************************************************************/
+   public QIcon getStartIcon()
+   {
+      return (this.startIcon);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for startIcon
+    *******************************************************************************/
+   public void setStartIcon(QIcon startIcon)
+   {
+      this.startIcon = startIcon;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for startIcon
+    *******************************************************************************/
+   public TextValues withStartIcon(QIcon startIcon)
+   {
+      this.startIcon = startIcon;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for endIcon
+    *******************************************************************************/
+   public QIcon getEndIcon()
+   {
+      return (this.endIcon);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for endIcon
+    *******************************************************************************/
+   public void setEndIcon(QIcon endIcon)
+   {
+      this.endIcon = endIcon;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for endIcon
+    *******************************************************************************/
+   public TextValues withEndIcon(QIcon endIcon)
+   {
+      this.endIcon = endIcon;
+      return (this);
+   }
+
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/data/QField.java

@@ -89,6 +89,11 @@ public @interface QField
     *******************************************************************************/
    int maxLength() default Integer.MAX_VALUE;
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   int gridColumns() default -1;
+
    /*******************************************************************************
     **
     *******************************************************************************/

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/data/QRecord.java

@@ -154,7 +154,7 @@ public class QRecord implements Serializable
          return (null);
       }
 
-      Map<String, V> clone = new LinkedHashMap<>();
+      Map<String, V> clone = new LinkedHashMap<>(map.size());
       for(Map.Entry<String, V> entry : map.entrySet())
       {
          Serializable value = entry.getValue();
@@ -246,6 +246,24 @@ public class QRecord implements Serializable
    }
 
 
+   /***************************************************************************
+    ** copy all values from 'joinedRecord' into this record's values map,
+    ** prefixing field names with joinTableNam + "."
+    ***************************************************************************/
+   public void addJoinedRecordValues(String joinTableName, QRecord joinedRecord)
+   {
+      if(joinedRecord == null)
+      {
+         return;
+      }
+
+      for(Map.Entry<String, Serializable> entry : joinedRecord.getValues().entrySet())
+      {
+         setValue(joinTableName + "." + entry.getKey(), entry.getValue());
+      }
+   }
+
+
 
    /*******************************************************************************
     **

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/data/QRecordEntity.java

@@ -41,11 +41,14 @@ import java.util.Locale;
 import java.util.Map;
 import java.util.Objects;
 import java.util.Optional;
+import java.util.function.Function;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.exceptions.QRuntimeException;
 import com.kingsrook.qqq.backend.core.logging.QLogger;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
 import com.kingsrook.qqq.backend.core.utils.ListingHash;
+import com.kingsrook.qqq.backend.core.utils.ObjectUtils;
 import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
 
 
@@ -61,6 +64,11 @@ public abstract class QRecordEntity
 
    private Map<String, Serializable> originalRecordValues;
 
+   ////////////////////////////////////////////////////////////////////////////////
+   // map of entity class names to QTableMetaData objects that they helped build //
+   ////////////////////////////////////////////////////////////////////////////////
+   private static Map<String, QTableMetaData> tableReferences = new HashMap<>();
+
 
 
    /*******************************************************************************
@@ -95,6 +103,19 @@ public abstract class QRecordEntity
 
 
 
+   /***************************************************************************
+    ** register a mapping between an entity class and a table that it is associated with.
+    ***************************************************************************/
+   public static void registerTable(Class<? extends QRecordEntity> entityClass, QTableMetaData table)
+   {
+      if(entityClass != null && table != null)
+      {
+         tableReferences.put(entityClass.getName(), table);
+      }
+   }
+
+
+
    /*******************************************************************************
     ** Build an entity of this QRecord type from a QRecord
     **
@@ -176,7 +197,10 @@ public abstract class QRecordEntity
 
 
    /*******************************************************************************
-    ** Convert this entity to a QRecord.
+    ** Convert this entity to a QRecord.  ALL fields in the entity will be set
+    ** in the QRecord.  Note that, if you're using this for an input to the UpdateAction,
+    ** that this could cause values to be set to null, e.g., if you constructed
+    ** a entity from scratch, and didn't set all values in it!!
     **
     *******************************************************************************/
    public QRecord toQRecord() throws QRuntimeException
@@ -190,25 +214,7 @@ public abstract class QRecordEntity
             qRecord.setValue(qRecordEntityField.getFieldName(), (Serializable) qRecordEntityField.getGetter().invoke(this));
          }
 
-         for(QRecordEntityAssociation qRecordEntityAssociation : getAssociationList(this.getClass()))
-         {
-            @SuppressWarnings("unchecked")
-            List<? extends QRecordEntity> associatedEntities = (List<? extends QRecordEntity>) qRecordEntityAssociation.getGetter().invoke(this);
-            String                        associationName    = qRecordEntityAssociation.getAssociationAnnotation().name();
-
-            if(associatedEntities != null)
-            {
-               /////////////////////////////////////////////////////////////////////////////////
-               // do this so an empty list in the entity becomes an empty list in the QRecord //
-               /////////////////////////////////////////////////////////////////////////////////
-               qRecord.withAssociatedRecords(associationName, new ArrayList<>());
-            }
-
-            for(QRecordEntity associatedEntity : CollectionUtils.nonNullList(associatedEntities))
-            {
-               qRecord.withAssociatedRecord(associationName, associatedEntity.toQRecord());
-            }
-         }
+         toQRecordProcessAssociations(qRecord, (entity) -> entity.toQRecord());
 
          return (qRecord);
       }
@@ -220,15 +226,65 @@ public abstract class QRecordEntity
 
 
 
+   /***************************************************************************
+    *
+    ***************************************************************************/
+   private void toQRecordProcessAssociations(QRecord outputRecord, Function<QRecordEntity, QRecord> toRecordFunction) throws Exception
+   {
+      for(QRecordEntityAssociation qRecordEntityAssociation : getAssociationList(this.getClass()))
+      {
+         @SuppressWarnings("unchecked")
+         List<? extends QRecordEntity> associatedEntities = (List<? extends QRecordEntity>) qRecordEntityAssociation.getGetter().invoke(this);
+         String associationName = qRecordEntityAssociation.getAssociationAnnotation().name();
+
+         if(associatedEntities != null)
+         {
+            outputRecord.withAssociatedRecords(associationName, new ArrayList<>());
+            for(QRecordEntity associatedEntity : associatedEntities)
+            {
+               outputRecord.withAssociatedRecord(associationName, toRecordFunction.apply(associatedEntity));
+            }
+         }
+      }
+   }
+
+
+
    /*******************************************************************************
-    **
+    ** Overload of toQRecordOnlyChangedFields that preserves original behavior of
+    ** that method, which is, to NOT includePrimaryKey
     *******************************************************************************/
+   @Deprecated(since = "includePrimaryKey param was added")
    public QRecord toQRecordOnlyChangedFields()
+   {
+      return toQRecordOnlyChangedFields(false);
+   }
+
+
+
+   /*******************************************************************************
+    ** Useful for the use-case of:
+    ** - fetch a QRecord (e.g., QueryAction or GetAction)
+    ** - build a QRecordEntity out of it
+    ** - change a field (or two) in it
+    ** - want to pass it into an UpdateAction, and want to see only the fields that
+    **   you know you changed get passed in to UpdateAction (e.g., PATCH semantics).
+    **
+    ** But also - per the includePrimaryKey param, include the primaryKey in the
+    ** records (e.g., to tell the Update which records to update).
+    **
+    ** Also, useful for:
+    ** - construct new entity, calling setters to populate some fields
+    ** - pass that entity into
+    *******************************************************************************/
+   public QRecord toQRecordOnlyChangedFields(boolean includePrimaryKey)
    {
       try
       {
          QRecord qRecord = new QRecord();
 
+         String primaryKeyFieldName = ObjectUtils.tryElse(() -> tableReferences.get(getClass().getName()).getPrimaryKeyField(), null);
+
          for(QRecordEntityField qRecordEntityField : getFieldList(this.getClass()))
          {
             Serializable thisValue     = (Serializable) qRecordEntityField.getGetter().invoke(this);
@@ -238,31 +294,16 @@ public abstract class QRecordEntity
                originalValue = originalRecordValues.get(qRecordEntityField.getFieldName());
             }
 
-            if(!Objects.equals(thisValue, originalValue))
+            ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+            // if this value and the original value don't match - OR - this is the table's primary key field - then put the value in the record. //
+            ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+            if(!Objects.equals(thisValue, originalValue) || (includePrimaryKey && Objects.equals(primaryKeyFieldName, qRecordEntityField.getFieldName())))
             {
                qRecord.setValue(qRecordEntityField.getFieldName(), thisValue);
             }
          }
 
-         for(QRecordEntityAssociation qRecordEntityAssociation : getAssociationList(this.getClass()))
-         {
-            @SuppressWarnings("unchecked")
-            List<? extends QRecordEntity> associatedEntities = (List<? extends QRecordEntity>) qRecordEntityAssociation.getGetter().invoke(this);
-            String                        associationName    = qRecordEntityAssociation.getAssociationAnnotation().name();
-
-            if(associatedEntities != null)
-            {
-               /////////////////////////////////////////////////////////////////////////////////
-               // do this so an empty list in the entity becomes an empty list in the QRecord //
-               /////////////////////////////////////////////////////////////////////////////////
-               qRecord.withAssociatedRecords(associationName, new ArrayList<>());
-            }
-
-            for(QRecordEntity associatedEntity : CollectionUtils.nonNullList(associatedEntities))
-            {
-               qRecord.withAssociatedRecord(associationName, associatedEntity.toQRecord());
-            }
-         }
+         toQRecordProcessAssociations(qRecord, (entity) -> entity.toQRecordOnlyChangedFields(includePrimaryKey));
 
          return (qRecord);
       }
@@ -488,15 +529,16 @@ public abstract class QRecordEntity
    {
       // todo - more types!!
       return (returnType.equals(String.class)
-         || returnType.equals(Integer.class)
-         || returnType.equals(int.class)
-         || returnType.equals(Boolean.class)
-         || returnType.equals(boolean.class)
-         || returnType.equals(BigDecimal.class)
-         || returnType.equals(Instant.class)
-         || returnType.equals(LocalDate.class)
-         || returnType.equals(LocalTime.class)
-         || returnType.equals(byte[].class));
+              || returnType.equals(Integer.class)
+              || returnType.equals(Long.class)
+              || returnType.equals(int.class)
+              || returnType.equals(Boolean.class)
+              || returnType.equals(boolean.class)
+              || returnType.equals(BigDecimal.class)
+              || returnType.equals(Instant.class)
+              || returnType.equals(LocalDate.class)
+              || returnType.equals(LocalTime.class)
+              || returnType.equals(byte[].class));
       /////////////////////////////////////////////
       // note - this list has implications upon: //
       // - QFieldType.fromClass                  //

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/data/QRecordWithJoinedRecords.java

@@ -0,0 +1,201 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2025.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.model.data;
+
+
+import java.io.Serializable;
+import java.util.LinkedHashMap;
+import java.util.Map;
+import java.util.function.BiFunction;
+
+
+/*******************************************************************************
+ ** Extension on QRecord, intended to be used where you've got records from
+ ** multiple tables, and you want to combine them into a single "wide" joined
+ ** record - but to do so without copying or modifying any of the individual
+ ** records.
+ **
+ ** e.g., given:
+ ** - Order (id, orderNo, orderDate) (main table)
+ ** - LineItem (id, sku, quantity)
+ ** - Extrinsic (id, key, value)
+ **
+ ** If set up in here as:
+ ** - new QRecordWithJoinedRecords(order)
+ **      .withJoinedRecordValues(lineItem)
+ **      .withJoinedRecordValues(extrinsic)
+ **
+ ** Then we'd have the appearance of values in the object like:
+ ** - id, orderNo, orderDate, lineItem.id, lineItem.sku, lineItem.quantity, extrinsic.id, extrinsic.key, extrinsic.value
+ **
+ ** Which, by the by, is how a query that returns joined records looks, and, is
+ ** what BackendQueryFilterUtils can use to do filter.
+ **
+ ** This is done without copying or mutating any of the records (which, if you just use
+ ** QRecord.withJoinedRecordValues, then those values are copied into the main record)
+ ** - because this object is just storing references to the input records.
+ **
+ ** Note that this implies that, values changed in this record (e.g, calls to setValue)
+ ** WILL impact the underlying records!
+ *******************************************************************************/
+public class QRecordWithJoinedRecords extends QRecord
+{
+   private QRecord              mainRecord;
+   private Map<String, QRecord> components = new LinkedHashMap<>();
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   public QRecordWithJoinedRecords(QRecord mainRecord)
+   {
+      this.mainRecord = mainRecord;
+   }
+
+
+
+   /*************************************************************************
+    **
+    ***************************************************************************/
+   @Override
+   public void addJoinedRecordValues(String joinTableName, QRecord joinedRecord)
+   {
+      components.put(joinTableName, joinedRecord);
+   }
+
+
+
+   /*************************************************************************
+    **
+    ***************************************************************************/
+   public QRecordWithJoinedRecords withJoinedRecordValues(QRecord record, String joinTableName)
+   {
+      addJoinedRecordValues(joinTableName, record);
+      return (this);
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   @Override
+   public Serializable getValue(String fieldName)
+   {
+      return performFunctionOnRecordBasedOnFieldName(fieldName, ((record, f) -> record.getValue(f)));
+   }
+
+
+
+   /***************************************************************************
+    *
+    ***************************************************************************/
+   @Override
+   public void setValue(String fieldName, Object value)
+   {
+      performFunctionOnRecordBasedOnFieldName(fieldName, ((record, f) ->
+      {
+         record.setValue(f, value);
+         return (null);
+      }));
+   }
+
+
+
+   /***************************************************************************
+    *
+    ***************************************************************************/
+   @Override
+   public void setValue(String fieldName, Serializable value)
+   {
+      performFunctionOnRecordBasedOnFieldName(fieldName, ((record, f) ->
+      {
+         record.setValue(f, value);
+         return (null);
+      }));
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   @Override
+   public void removeValue(String fieldName)
+   {
+      performFunctionOnRecordBasedOnFieldName(fieldName, ((record, f) ->
+      {
+         record.removeValue(f);
+         return (null);
+      }));
+   }
+
+
+
+   /***************************************************************************
+    ** avoid having this same block in all the functions that call it...
+    ** given a fieldName, which may be a joinTable.fieldName, apply the function
+    ** to the right entity.
+    ***************************************************************************/
+   private Serializable performFunctionOnRecordBasedOnFieldName(String fieldName, BiFunction<QRecord, String, Serializable> functionToPerform)
+   {
+      if(fieldName.contains("."))
+      {
+         String[] parts     = fieldName.split("\\.");
+         QRecord  component = components.get(parts[0]);
+         if(component != null)
+         {
+            return functionToPerform.apply(component, parts[1]);
+         }
+         else
+         {
+            return null;
+         }
+      }
+      else
+      {
+         return functionToPerform.apply(mainRecord, fieldName);
+      }
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   @Override
+   public Map<String, Serializable> getValues()
+   {
+      Map<String, Serializable> rs = new LinkedHashMap<>(mainRecord.getValues());
+      for(Map.Entry<String, QRecord> componentEntry : components.entrySet())
+      {
+         String  joinTableName   = componentEntry.getKey();
+         QRecord componentRecord = componentEntry.getValue();
+         for(Map.Entry<String, Serializable> entry : componentRecord.getValues().entrySet())
+         {
+            rs.put(joinTableName + "." + entry.getKey(), entry.getValue());
+         }
+      }
+      return (rs);
+   }
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/EmptyMetaDataProducerOutput.java

@@ -0,0 +1,47 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2025.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.model.metadata;
+
+
+import com.kingsrook.qqq.backend.core.logging.QLogger;
+
+
+/*******************************************************************************
+ ** for use-cases where a metaDataProducer directly adds its objects to the
+ ** qInstance, then this empty object can be returned.
+ *******************************************************************************/
+public class EmptyMetaDataProducerOutput implements MetaDataProducerOutput
+{
+   private static final QLogger LOG = QLogger.getLogger(EmptyMetaDataProducerOutput.class);
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   @Override
+   public void addSelfToInstance(QInstance instance)
+   {
+      /////////////////////////////////
+      // noop - this output is empty //
+      /////////////////////////////////
+      LOG.trace("empty meta data producer has nothing to add.");
+   }
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/MetaDataProducer.java

@@ -22,15 +22,47 @@
 package com.kingsrook.qqq.backend.core.model.metadata;
 
 
-import com.kingsrook.qqq.backend.core.model.MetaDataProducerInterface;
-
-
 /*******************************************************************************
  ** Abstract class that knows how to produce meta data objects.  Useful with
- ** MetaDataProducerHelper, to put point at a package full of these, and populate
+ ** MetaDataProducerHelper, to point at a package full of these, and populate
  ** your whole QInstance.
  *******************************************************************************/
 public abstract class MetaDataProducer<T extends MetaDataProducerOutput> implements MetaDataProducerInterface<T>
 {
+   private Class<?> sourceClass;
+
+
+   /*******************************************************************************
+    ** Getter for sourceClass
+    **
+    *******************************************************************************/
+   @Override
+   public Class<?> getSourceClass()
+   {
+      return sourceClass;
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for sourceClass
+    **
+    *******************************************************************************/
+   @Override
+   public void setSourceClass(Class<?> sourceClass)
+   {
+      this.sourceClass = sourceClass;
+   }
+
+
+   /*******************************************************************************
+    ** Fluent setter for sourceClass
+    **
+    *******************************************************************************/
+   public MetaDataProducer<T> withSourceClass(Class<?> sourceClass)
+   {
+      this.sourceClass = sourceClass;
+      return (this);
+   }
 
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/MetaDataProducerHelper.java

@@ -22,22 +22,37 @@
 package com.kingsrook.qqq.backend.core.model.metadata;
 
 
-import java.io.IOException;
+import java.io.Serializable;
 import java.lang.reflect.Constructor;
+import java.lang.reflect.Field;
 import java.lang.reflect.Modifier;
 import java.util.ArrayList;
 import java.util.Comparator;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
-import com.google.common.collect.ImmutableSet;
-import com.google.common.reflect.ClassPath;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.logging.QLogger;
-import com.kingsrook.qqq.backend.core.model.MetaDataProducerInterface;
+import com.kingsrook.qqq.backend.core.model.data.QField;
+import com.kingsrook.qqq.backend.core.model.data.QRecordEntity;
 import com.kingsrook.qqq.backend.core.model.metadata.dashboard.QWidgetMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.joins.QJoinMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.PossibleValueEnum;
+import com.kingsrook.qqq.backend.core.model.metadata.producers.ChildJoinFromRecordEntityGenericMetaDataProducer;
+import com.kingsrook.qqq.backend.core.model.metadata.producers.ChildRecordListWidgetFromRecordEntityGenericMetaDataProducer;
+import com.kingsrook.qqq.backend.core.model.metadata.producers.MetaDataCustomizerInterface;
+import com.kingsrook.qqq.backend.core.model.metadata.producers.PossibleValueSourceOfEnumGenericMetaDataProducer;
+import com.kingsrook.qqq.backend.core.model.metadata.producers.PossibleValueSourceOfTableGenericMetaDataProducer;
+import com.kingsrook.qqq.backend.core.model.metadata.producers.RecordEntityToTableGenericMetaDataProducer;
+import com.kingsrook.qqq.backend.core.model.metadata.producers.annotations.ChildRecordListWidget;
+import com.kingsrook.qqq.backend.core.model.metadata.producers.annotations.ChildTable;
+import com.kingsrook.qqq.backend.core.model.metadata.producers.annotations.QMetaDataProducingEntity;
+import com.kingsrook.qqq.backend.core.model.metadata.producers.annotations.QMetaDataProducingPossibleValueEnum;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+import com.kingsrook.qqq.backend.core.utils.ClassPathUtils;
+import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
+import com.kingsrook.qqq.backend.core.utils.StringUtils;
 import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
 
 
@@ -51,8 +66,6 @@ public class MetaDataProducerHelper
    private static Map<Class<?>, Integer> comparatorValuesByType = new HashMap<>();
    private static Integer                defaultComparatorValue;
 
-   private static ImmutableSet<ClassPath.ClassInfo> topLevelClasses;
-
    static
    {
       ////////////////////////////////////////////////////////////////////////////////////////
@@ -73,13 +86,30 @@ public class MetaDataProducerHelper
       comparatorValuesByType.put(QAppMetaData.class, 23);
    }
 
+   private static MetaDataCustomizerInterface<QTableMetaData> tableMetaDataCustomizer = null;
+
+
+
    /*******************************************************************************
     ** Recursively find all classes in the given package, that implement MetaDataProducerInterface
-    ** run them, and add their output to the given qInstance.
+    ** run them, and add their output to the given qInstance - using the provided
+    ** tableMetaDataCustomizer to help with all RecordEntity's that
+    ** are configured to make tables.
     **
     ** Note - they'll be sorted by the sortOrder they provide.
     *******************************************************************************/
-   public static void processAllMetaDataProducersInPackage(QInstance instance, String packageName) throws QException
+   public static void processAllMetaDataProducersInPackage(QInstance instance, String packageName, MetaDataCustomizerInterface<QTableMetaData> tableMetaDataCustomizer) throws QException
+   {
+      MetaDataProducerHelper.tableMetaDataCustomizer = tableMetaDataCustomizer;
+      processAllMetaDataProducersInPackage(instance, packageName);
+      MetaDataProducerHelper.tableMetaDataCustomizer = null;
+   }
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   public static List<MetaDataProducerInterface<?>> findProducers(String packageName) throws QException
    {
       List<Class<?>> classesInPackage;
       try
@@ -87,7 +117,7 @@ public class MetaDataProducerHelper
          ////////////////////////////////////////////////////////////////////////
          // find all the meta data producer classes in (and under) the package //
          ////////////////////////////////////////////////////////////////////////
-         classesInPackage = getClassesInPackage(packageName);
+         classesInPackage = ClassPathUtils.getClassesInPackage(packageName);
       }
       catch(Exception e)
       {
@@ -95,6 +125,9 @@ public class MetaDataProducerHelper
       }
       List<MetaDataProducerInterface<?>> producers = new ArrayList<>();
 
+      ////////////////////////////////////////////////////////////////////////////////////////
+      // loop over classes, processing them based on either their type or their annotations //
+      ////////////////////////////////////////////////////////////////////////////////////////
       for(Class<?> aClass : classesInPackage)
       {
          try
@@ -104,25 +137,36 @@ public class MetaDataProducerHelper
                continue;
             }
 
+            /////////////////////////////////////////////////////////////////////
+            // handle classes which are themselves MetaDataProducerInterface's //
+            /////////////////////////////////////////////////////////////////////
             if(MetaDataProducerInterface.class.isAssignableFrom(aClass))
             {
-               boolean foundValidConstructor = false;
-               for(Constructor<?> constructor : aClass.getConstructors())
-               {
-                  if(constructor.getParameterCount() == 0)
-                  {
-                     Object o = constructor.newInstance();
-                     producers.add((MetaDataProducerInterface<?>) o);
-                     foundValidConstructor = true;
-                     break;
-                  }
-               }
+               CollectionUtils.addIfNotNull(producers, processMetaDataProducer(aClass));
+            }
 
-               if(!foundValidConstructor)
+            /////////////////////////////////////////////////////////////////////////
+            // handle classes that have the @QMetaDataProducingEntity annotation - //
+            // record entities that should produce meta-data                       //
+            /////////////////////////////////////////////////////////////////////////
+            if(aClass.isAnnotationPresent(QMetaDataProducingEntity.class))
+            {
+               producers.addAll(processMetaDataProducingEntity(aClass));
+            }
+
+            //////////////////////////////////////////////////////////////////
+            // handle classes with the @QMetaDataProducingPossibleValueEnum //
+            // enums that are PVS's                                         //
+            //////////////////////////////////////////////////////////////////
+            if(aClass.isAnnotationPresent(QMetaDataProducingPossibleValueEnum.class))
+            {
+               QMetaDataProducingPossibleValueEnum qMetaDataProducingPossibleValueEnum = aClass.getAnnotation(QMetaDataProducingPossibleValueEnum.class);
+               if(qMetaDataProducingPossibleValueEnum.producePossibleValueSource())
                {
-                  LOG.warn("Found a class which implements MetaDataProducerInterface, but it does not have a no-arg constructor, so it cannot be used.", logPair("class", aClass.getSimpleName()));
+                  CollectionUtils.addIfNotNull(producers, processMetaDataProducingPossibleValueEnum(aClass));
                }
             }
+
          }
          catch(Exception e)
          {
@@ -148,6 +192,20 @@ public class MetaDataProducerHelper
             }
          }));
 
+      return (producers);
+   }
+
+
+   /*******************************************************************************
+    ** Recursively find all classes in the given package, that implement MetaDataProducerInterface
+    ** run them, and add their output to the given qInstance.
+    **
+    ** Note - they'll be sorted by the sortOrder they provide.
+    *******************************************************************************/
+   public static void processAllMetaDataProducersInPackage(QInstance instance, String packageName) throws QException
+   {
+      List<MetaDataProducerInterface<?>> producers = findProducers(packageName);
+
       ///////////////////////////////////////////////////////////////////////////
       // execute each one (if enabled), adding their meta data to the instance //
       ///////////////////////////////////////////////////////////////////////////
@@ -173,54 +231,264 @@ public class MetaDataProducerHelper
             LOG.debug("Not using producer which is not enabled", logPair("producer", producer.getClass().getSimpleName()));
          }
       }
-
    }
 
 
 
-   /*******************************************************************************
-    ** from https://stackoverflow.com/questions/520328/can-you-find-all-classes-in-a-package-using-reflection
-    ** (since the original, from ChatGPT, didn't work in jars, despite GPT hallucinating that it would)
-    *******************************************************************************/
-   private static List<Class<?>> getClassesInPackage(String packageName) throws IOException
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   @SuppressWarnings("unchecked")
+   private static <T extends Serializable & PossibleValueEnum<T>> MetaDataProducerInterface<?> processMetaDataProducingPossibleValueEnum(Class<?> sourceClass)
    {
-      List<Class<?>> classes = new ArrayList<>();
-      ClassLoader    loader  = Thread.currentThread().getContextClassLoader();
-
-      for(ClassPath.ClassInfo info : getTopLevelClasses(loader))
+      String warningPrefix = "Found a class annotated as @" + QMetaDataProducingPossibleValueEnum.class.getSimpleName();
+      if(!PossibleValueEnum.class.isAssignableFrom(sourceClass))
       {
-         if(info.getName().startsWith(packageName))
+         LOG.warn(warningPrefix + ", but which is not a " + PossibleValueEnum.class.getSimpleName() + ", so it will not be used.", logPair("class", sourceClass.getSimpleName()));
+         return null;
+      }
+
+      PossibleValueEnum<?>[] values = (PossibleValueEnum<?>[]) sourceClass.getEnumConstants();
+      PossibleValueSourceOfEnumGenericMetaDataProducer<T> producer = new PossibleValueSourceOfEnumGenericMetaDataProducer<>(sourceClass.getSimpleName(), (PossibleValueEnum<T>[]) values);
+      producer.setSourceClass(sourceClass);
+      return producer;
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   private static List<MetaDataProducerInterface<?>> processMetaDataProducingEntity(Class<?> sourceClass) throws Exception
+   {
+      List<MetaDataProducerInterface<?>> rs = new ArrayList<>();
+
+      QMetaDataProducingEntity qMetaDataProducingEntity = sourceClass.getAnnotation(QMetaDataProducingEntity.class);
+      String                   warningPrefix            = "Found a class annotated as @" + QMetaDataProducingEntity.class.getSimpleName();
+
+      ///////////////////////////////////////////////////////////
+      // make sures class is QRecordEntity and cast it as such //
+      ///////////////////////////////////////////////////////////
+      if(!QRecordEntity.class.isAssignableFrom(sourceClass))
+      {
+         LOG.warn(warningPrefix + ", but which is not a " + QRecordEntity.class.getSimpleName() + ", so it will not be used.", logPair("class", sourceClass.getSimpleName()));
+         return (rs);
+      }
+
+      @SuppressWarnings("unchecked") // safe per the check above.
+      Class<? extends QRecordEntity> recordEntityClass = (Class<? extends QRecordEntity>) sourceClass;
+
+      ////////////////////////////////////////////////
+      // get TABLE_NAME static field from the class //
+      ////////////////////////////////////////////////
+      Field tableNameField = recordEntityClass.getDeclaredField("TABLE_NAME");
+      if(!tableNameField.getType().equals(String.class))
+      {
+         LOG.warn(warningPrefix + ", but whose TABLE_NAME field is not a String, so it will not be used.", logPair("class", recordEntityClass.getSimpleName()));
+         return (rs);
+      }
+
+      String tableNameValue = (String) tableNameField.get(null);
+
+      //////////////////////////////////////////
+      // add table producer, if so configured //
+      //////////////////////////////////////////
+      if(qMetaDataProducingEntity.produceTableMetaData())
+      {
+         try
          {
-            classes.add(info.load());
+            Class<? extends MetaDataCustomizerInterface<?>>              genericMetaProductionCustomizer   = (Class<? extends MetaDataCustomizerInterface<?>>) qMetaDataProducingEntity.tableMetaDataCustomizer();
+            Class<? extends MetaDataCustomizerInterface<QTableMetaData>> tableMetaDataProductionCustomizer = null;
+            if(!genericMetaProductionCustomizer.equals(MetaDataCustomizerInterface.NoopMetaDataCustomizer.class))
+            {
+               tableMetaDataProductionCustomizer = (Class<? extends MetaDataCustomizerInterface<QTableMetaData>>) genericMetaProductionCustomizer;
+            }
+
+            RecordEntityToTableGenericMetaDataProducer producer = new RecordEntityToTableGenericMetaDataProducer(tableNameValue, recordEntityClass, tableMetaDataProductionCustomizer);
+            producer.setSourceClass(recordEntityClass);
+
+            if(tableMetaDataCustomizer != null)
+            {
+               producer.addRecordEntityTableMetaDataProductionCustomizer(tableMetaDataCustomizer);
+            }
+
+            rs.add(producer);
+         }
+         catch(Exception e)
+         {
+            throw new QException("Error processing table meta data producer for entity class: " + recordEntityClass.getName(), e);
          }
       }
 
-      return (classes);
-   }
-
-
-
-   /*******************************************************************************
-    **
-    *******************************************************************************/
-   private static ImmutableSet<ClassPath.ClassInfo> getTopLevelClasses(ClassLoader loader) throws IOException
-   {
-      if(topLevelClasses == null)
+      ////////////////////////////////////////
+      // add PVS producer, if so configured //
+      ////////////////////////////////////////
+      if(qMetaDataProducingEntity.producePossibleValueSource())
       {
-         topLevelClasses = ClassPath.from(loader).getTopLevelClasses();
+         PossibleValueSourceOfTableGenericMetaDataProducer producer = new PossibleValueSourceOfTableGenericMetaDataProducer(tableNameValue);
+         producer.setSourceClass(recordEntityClass);
+         rs.add(producer);
       }
 
-      return (topLevelClasses);
+      //////////////////////////
+      // process child tables //
+      //////////////////////////
+      for(ChildTable childTable : qMetaDataProducingEntity.childTables())
+      {
+         Class<? extends QRecordEntity> childEntityClass = childTable.childTableEntityClass();
+         if(childTable.childJoin().enabled())
+         {
+            CollectionUtils.addIfNotNull(rs, processChildJoin(recordEntityClass, childTable));
+
+            if(childTable.childRecordListWidget().enabled())
+            {
+               CollectionUtils.addIfNotNull(rs, processChildRecordListWidget(recordEntityClass, childTable));
+            }
+         }
+         else
+         {
+            if(childTable.childRecordListWidget().enabled())
+            {
+               //////////////////////////////////////////////////////////////////////////
+               // if not doing the join, can't do the child-widget, so warn about that //
+               //////////////////////////////////////////////////////////////////////////
+               LOG.warn(warningPrefix + " requested to produce a ChildRecordListWidget, but not produce a Join - which is not allowed (must do join to do widget). ", logPair("class", recordEntityClass.getSimpleName()), logPair("childEntityClass", childEntityClass.getSimpleName()));
+            }
+         }
+      }
+
+      return (rs);
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   private static MetaDataProducerInterface<?> processChildRecordListWidget(Class<? extends QRecordEntity> sourceClass, ChildTable childTable) throws Exception
+   {
+      Class<? extends QRecordEntity> childEntityClass = childTable.childTableEntityClass();
+      String                         parentTableName  = getTableNameStaticFieldValue(sourceClass);
+      String                         childTableName   = getTableNameStaticFieldValue(childEntityClass);
+
+      ChildRecordListWidget childRecordListWidget = childTable.childRecordListWidget();
+      ChildRecordListWidgetFromRecordEntityGenericMetaDataProducer producer = new ChildRecordListWidgetFromRecordEntityGenericMetaDataProducer(childTableName, parentTableName, childRecordListWidget);
+      producer.setSourceClass(sourceClass);
+      return producer;
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   private static String findPossibleValueField(Class<? extends QRecordEntity> entityClass, String possibleValueSourceName)
+   {
+      for(Field field : entityClass.getDeclaredFields())
+      {
+         if(field.isAnnotationPresent(QField.class))
+         {
+            QField qField = field.getAnnotation(QField.class);
+            if(qField.possibleValueSourceName().equals(possibleValueSourceName))
+            {
+               return field.getName();
+            }
+         }
+      }
+
+      return (null);
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   private static MetaDataProducerInterface<?> processChildJoin(Class<? extends QRecordEntity> entityClass, ChildTable childTable) throws Exception
+   {
+      Class<? extends QRecordEntity> childEntityClass = childTable.childTableEntityClass();
+
+      String parentTableName        = getTableNameStaticFieldValue(entityClass);
+      String childTableName         = getTableNameStaticFieldValue(childEntityClass);
+      String possibleValueFieldName = findPossibleValueField(childEntityClass, parentTableName);
+      if(!StringUtils.hasContent(possibleValueFieldName))
+      {
+         LOG.warn("Could not find field in [" + childEntityClass.getSimpleName() + "] with possibleValueSource referencing table [" + entityClass.getSimpleName() + "]");
+         return (null);
+      }
+
+      ChildJoinFromRecordEntityGenericMetaDataProducer producer = new ChildJoinFromRecordEntityGenericMetaDataProducer(childTableName, parentTableName, possibleValueFieldName);
+      producer.setSourceClass(entityClass);
+      return producer;
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   private static MetaDataProducerInterface<?> processMetaDataProducer(Class<?> sourceCClass) throws Exception
+   {
+      for(Constructor<?> constructor : sourceCClass.getConstructors())
+      {
+         if(constructor.getParameterCount() == 0)
+         {
+            Object o = constructor.newInstance();
+            MetaDataProducerInterface<?> producer = (MetaDataProducerInterface<?>) o;
+            producer.setSourceClass(sourceCClass);
+            return producer;
+         }
+      }
+
+      LOG.warn("Found a class which implements MetaDataProducerInterface, but it does not have a no-arg constructor, so it cannot be used.", logPair("class", sourceCClass.getSimpleName()));
+      return null;
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   private static String getTableNameStaticFieldValue(Class<?> aClass) throws NoSuchFieldException, IllegalAccessException
+   {
+      Field tableNameField = aClass.getDeclaredField("TABLE_NAME");
+      if(!tableNameField.getType().equals(String.class))
+      {
+         return (null);
+      }
+
+      String tableNameValue = (String) tableNameField.get(null);
+      return (tableNameValue);
    }
 
 
 
    /*******************************************************************************
-    **
+    ** Getter for tableMetaDataCustomizer
     *******************************************************************************/
-   public static void clearTopLevelClassCache()
+   public MetaDataCustomizerInterface<QTableMetaData> getTableMetaDataCustomizer()
    {
-      topLevelClasses = null;
+      return (MetaDataProducerHelper.tableMetaDataCustomizer);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for tableMetaDataCustomizer
+    *******************************************************************************/
+   public void setTableMetaDataCustomizer(MetaDataCustomizerInterface<QTableMetaData> tableMetaDataCustomizer)
+   {
+      MetaDataProducerHelper.tableMetaDataCustomizer = tableMetaDataCustomizer;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for tableMetaDataCustomizer
+    *******************************************************************************/
+   public void withTableMetaDataCustomizer(MetaDataCustomizerInterface<QTableMetaData> tableMetaDataCustomizer)
+   {
+      MetaDataProducerHelper.tableMetaDataCustomizer = tableMetaDataCustomizer;
    }
 
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/MetaDataProducerInterface.java

@@ -1,6 +1,6 @@
 /*
  * QQQ - Low-code Application Framework for Engineers.
- * Copyright (C) 2021-2023.  Kingsrook, LLC
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
  * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
  * contact@kingsrook.com
  * https://github.com/Kingsrook/
@@ -19,22 +19,20 @@
  * along with this program.  If not, see <https://www.gnu.org/licenses/>.
  */
 
-package com.kingsrook.qqq.backend.core.model;
+package com.kingsrook.qqq.backend.core.model.metadata;
 
 
 import com.kingsrook.qqq.backend.core.exceptions.QException;
-import com.kingsrook.qqq.backend.core.model.metadata.MetaDataProducerOutput;
-import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
 
 
 /*******************************************************************************
  ** Interface for classes that know how to produce meta data objects.  Useful with
- ** MetaDataProducerHelper, to put point at a package full of these, and populate
+ ** MetaDataProducerHelper, to point at a package full of these, and populate
  ** your whole QInstance.
  **
  ** See also MetaDataProducer - an implementer of this interface, which actually
  ** came first, and is fine to extend if producing a meta-data class is all your
- ** clas means to do (nice and "Single-responsibility principle").
+ ** class means to do (nice and "Single-responsibility principle").
  **
  ** But, in some applications you may want to, for example, have one class that
  ** defines a process step, and also produces the meta-data for that process, so
@@ -75,4 +73,23 @@ public interface MetaDataProducerInterface<T extends MetaDataProducerOutput>
       return (true);
    }
 
+
+   /***************************************************************************
+    *
+    ***************************************************************************/
+   default void setSourceClass(Class<?> sourceClass)
+   {
+      //////////
+      // noop //
+      //////////
+   }
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   default Class<?> getSourceClass()
+   {
+      return null;
+   }
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/MetaDataProducerMultiOutput.java

@@ -24,6 +24,7 @@ package com.kingsrook.qqq.backend.core.model.metadata;
 
 import java.util.ArrayList;
 import java.util.List;
+import com.kingsrook.qqq.backend.core.model.metadata.qbits.SourceQBitAware;
 import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
 
 
@@ -31,10 +32,12 @@ import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
  ** Output object for a MetaDataProducer, which contains multiple meta-data
  ** objects.
  *******************************************************************************/
-public class MetaDataProducerMultiOutput implements MetaDataProducerOutput
+public class MetaDataProducerMultiOutput implements MetaDataProducerOutput, SourceQBitAware
 {
    private List<MetaDataProducerOutput> contents;
 
+   private String sourceQBitName;
+
 
 
    /*******************************************************************************
@@ -98,4 +101,48 @@ public class MetaDataProducerMultiOutput implements MetaDataProducerOutput
       return (rs);
    }
 
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   @Override
+   public String getSourceQBitName()
+   {
+      return (this.sourceQBitName);
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   @Override
+   public void setSourceQBitName(String sourceQBitName)
+   {
+      this.sourceQBitName = sourceQBitName;
+
+      /////////////////////////////////////////////
+      // propagate the name down to the children //
+      /////////////////////////////////////////////
+      for(MetaDataProducerOutput content : contents)
+      {
+         if(content instanceof SourceQBitAware aware)
+         {
+            aware.setSourceQBitName(sourceQBitName);
+         }
+      }
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   @Override
+   public MetaDataProducerMultiOutput withSourceQBitName(String sourceQBitName)
+   {
+      setSourceQBitName(sourceQBitName);
+      return this;
+   }
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/QBackendMetaData.java

@@ -26,8 +26,13 @@ import java.util.HashSet;
 import java.util.Set;
 import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
 import com.kingsrook.qqq.backend.core.instances.QInstanceValidator;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
 import com.kingsrook.qqq.backend.core.model.metadata.serialization.QBackendMetaDataDeserializer;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.Capability;
+import com.kingsrook.qqq.backend.core.model.metadata.variants.BackendVariantsConfig;
+import com.kingsrook.qqq.backend.core.model.metadata.variants.LegacyBackendVariantSetting;
 import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleInterface;
 
 
@@ -45,21 +50,18 @@ public class QBackendMetaData implements TopLevelMetaDataInterface
    private Set<Capability> enabledCapabilities  = new HashSet<>();
    private Set<Capability> disabledCapabilities = new HashSet<>();
 
-   private Boolean usesVariants = false;
-   private String  variantOptionsTableIdField;
-   private String  variantOptionsTableNameField;
-   private String  variantOptionsTableTypeField;
-   private String  variantOptionsTableTypeValue;
-   private String  variantOptionsTableUsernameField;
-   private String  variantOptionsTablePasswordField;
-   private String  variantOptionsTableApiKeyField;
-   private String  variantOptionsTableClientIdField;
-   private String  variantOptionsTableClientSecretField;
-   private String  variantOptionsTableName;
+   private Boolean               usesVariants = false;
+   private BackendVariantsConfig backendVariantsConfig;
 
    // todo - at some point, we may want to apply this to secret properties on subclasses?
    // @JsonFilter("secretsFilter")
 
+   @Deprecated(since = "Replaced by filter in backendVariantsConfig - but leaving as field to pair with ...TypeValue for building filter")
+   private String variantOptionsTableTypeField; // a field on which to filter the variant-options table, to limit which records in it are available as variants
+
+   @Deprecated(since = "Replaced by variantTypeKey and value in filter in backendVariantsConfig - but leaving as field to pair with ...TypeField for building filter")
+   private String variantOptionsTableTypeValue; // value for the type-field, to limit which records in it are available as variants; but also, the key in the session.backendVariants map!
+
 
 
    /*******************************************************************************
@@ -394,22 +396,15 @@ public class QBackendMetaData implements TopLevelMetaDataInterface
 
 
 
-   /*******************************************************************************
-    ** Getter for variantOptionsTableIdField
-    *******************************************************************************/
-   public String getVariantOptionsTableIdField()
-   {
-      return (this.variantOptionsTableIdField);
-   }
-
-
-
    /*******************************************************************************
     ** Setter for variantOptionsTableIdField
     *******************************************************************************/
+   @Deprecated(since = "backendVariantsConfig will infer this from the variant options table's primary key")
    public void setVariantOptionsTableIdField(String variantOptionsTableIdField)
    {
-      this.variantOptionsTableIdField = variantOptionsTableIdField;
+      /////////////////////////////////////////////////
+      // noop as we migrate to backendVariantsConfig //
+      /////////////////////////////////////////////////
    }
 
 
@@ -417,30 +412,24 @@ public class QBackendMetaData implements TopLevelMetaDataInterface
    /*******************************************************************************
     ** Fluent setter for variantOptionsTableIdField
     *******************************************************************************/
+   @Deprecated(since = "backendVariantsConfig will infer this from the variant options table's primary key")
    public QBackendMetaData withVariantOptionsTableIdField(String variantOptionsTableIdField)
    {
-      this.variantOptionsTableIdField = variantOptionsTableIdField;
+      this.setVariantOptionsTableIdField(variantOptionsTableIdField);
       return (this);
    }
 
 
 
-   /*******************************************************************************
-    ** Getter for variantOptionsTableNameField
-    *******************************************************************************/
-   public String getVariantOptionsTableNameField()
-   {
-      return (this.variantOptionsTableNameField);
-   }
-
-
-
    /*******************************************************************************
     ** Setter for variantOptionsTableNameField
     *******************************************************************************/
+   @Deprecated(since = "backendVariantsConfig will infer this from the variant options table's recordLabel")
    public void setVariantOptionsTableNameField(String variantOptionsTableNameField)
    {
-      this.variantOptionsTableNameField = variantOptionsTableNameField;
+      /////////////////////////////////////////////////
+      // noop as we migrate to backendVariantsConfig //
+      /////////////////////////////////////////////////
    }
 
 
@@ -448,30 +437,26 @@ public class QBackendMetaData implements TopLevelMetaDataInterface
    /*******************************************************************************
     ** Fluent setter for variantOptionsTableNameField
     *******************************************************************************/
+   @Deprecated(since = "backendVariantsConfig will infer this from the variant options table's recordLabel")
    public QBackendMetaData withVariantOptionsTableNameField(String variantOptionsTableNameField)
    {
-      this.variantOptionsTableNameField = variantOptionsTableNameField;
+      this.setVariantOptionsTableNameField(variantOptionsTableNameField);
       return (this);
    }
 
 
 
-   /*******************************************************************************
-    ** Getter for variantOptionsTableTypeField
-    *******************************************************************************/
-   public String getVariantOptionsTableTypeField()
-   {
-      return (this.variantOptionsTableTypeField);
-   }
-
-
-
    /*******************************************************************************
     ** Setter for variantOptionsTableTypeField
     *******************************************************************************/
+   @Deprecated(since = "Replaced by fieldName in filter in backendVariantsConfig - but leaving as field to pair with ...TypeValue for building filter")
    public void setVariantOptionsTableTypeField(String variantOptionsTableTypeField)
    {
       this.variantOptionsTableTypeField = variantOptionsTableTypeField;
+      if(this.variantOptionsTableTypeValue != null)
+      {
+         this.getOrWithNewBackendVariantsConfig().setOptionsFilter(new QQueryFilter(new QFilterCriteria(variantOptionsTableTypeField, QCriteriaOperator.EQUALS, variantOptionsTableTypeValue)));
+      }
    }
 
 
@@ -479,30 +464,28 @@ public class QBackendMetaData implements TopLevelMetaDataInterface
    /*******************************************************************************
     ** Fluent setter for variantOptionsTableTypeField
     *******************************************************************************/
+   @Deprecated(since = "Replaced by fieldName in filter in backendVariantsConfig - but leaving as field to pair with ...TypeValue for building filter")
    public QBackendMetaData withVariantOptionsTableTypeField(String variantOptionsTableTypeField)
    {
-      this.variantOptionsTableTypeField = variantOptionsTableTypeField;
+      this.setVariantOptionsTableTypeField(variantOptionsTableTypeField);
       return (this);
    }
 
 
 
-   /*******************************************************************************
-    ** Getter for variantOptionsTableTypeValue
-    *******************************************************************************/
-   public String getVariantOptionsTableTypeValue()
-   {
-      return (this.variantOptionsTableTypeValue);
-   }
-
-
-
    /*******************************************************************************
     ** Setter for variantOptionsTableTypeValue
     *******************************************************************************/
+   @Deprecated(since = "Replaced by variantTypeKey and value in filter in backendVariantsConfig - but leaving as field to pair with ...TypeField for building filter")
    public void setVariantOptionsTableTypeValue(String variantOptionsTableTypeValue)
    {
+      this.getOrWithNewBackendVariantsConfig().setVariantTypeKey(variantOptionsTableTypeValue);
+
       this.variantOptionsTableTypeValue = variantOptionsTableTypeValue;
+      if(this.variantOptionsTableTypeField != null)
+      {
+         this.getOrWithNewBackendVariantsConfig().setOptionsFilter(new QQueryFilter(new QFilterCriteria(variantOptionsTableTypeField, QCriteriaOperator.EQUALS, variantOptionsTableTypeValue)));
+      }
    }
 
 
@@ -510,30 +493,22 @@ public class QBackendMetaData implements TopLevelMetaDataInterface
    /*******************************************************************************
     ** Fluent setter for variantOptionsTableTypeValue
     *******************************************************************************/
+   @Deprecated(since = "Replaced by variantTypeKey and value in filter in backendVariantsConfig - but leaving as field to pair with ...TypeField for building filter")
    public QBackendMetaData withVariantOptionsTableTypeValue(String variantOptionsTableTypeValue)
    {
-      this.variantOptionsTableTypeValue = variantOptionsTableTypeValue;
+      this.setVariantOptionsTableTypeValue(variantOptionsTableTypeValue);
       return (this);
    }
 
 
 
-   /*******************************************************************************
-    ** Getter for variantOptionsTableUsernameField
-    *******************************************************************************/
-   public String getVariantOptionsTableUsernameField()
-   {
-      return (this.variantOptionsTableUsernameField);
-   }
-
-
-
    /*******************************************************************************
     ** Setter for variantOptionsTableUsernameField
     *******************************************************************************/
+   @Deprecated(since = "Replaced by backendVariantsConfig.backendSettingSourceFieldNameMap")
    public void setVariantOptionsTableUsernameField(String variantOptionsTableUsernameField)
    {
-      this.variantOptionsTableUsernameField = variantOptionsTableUsernameField;
+      this.getOrWithNewBackendVariantsConfig().withBackendSettingSourceFieldName(LegacyBackendVariantSetting.USERNAME, variantOptionsTableUsernameField);
    }
 
 
@@ -541,30 +516,22 @@ public class QBackendMetaData implements TopLevelMetaDataInterface
    /*******************************************************************************
     ** Fluent setter for variantOptionsTableUsernameField
     *******************************************************************************/
+   @Deprecated(since = "Replaced by backendVariantsConfig.backendSettingSourceFieldNameMap")
    public QBackendMetaData withVariantOptionsTableUsernameField(String variantOptionsTableUsernameField)
    {
-      this.variantOptionsTableUsernameField = variantOptionsTableUsernameField;
+      this.setVariantOptionsTableUsernameField(variantOptionsTableUsernameField);
       return (this);
    }
 
 
 
-   /*******************************************************************************
-    ** Getter for variantOptionsTablePasswordField
-    *******************************************************************************/
-   public String getVariantOptionsTablePasswordField()
-   {
-      return (this.variantOptionsTablePasswordField);
-   }
-
-
-
    /*******************************************************************************
     ** Setter for variantOptionsTablePasswordField
     *******************************************************************************/
+   @Deprecated(since = "Replaced by backendVariantsConfig.backendSettingSourceFieldNameMap")
    public void setVariantOptionsTablePasswordField(String variantOptionsTablePasswordField)
    {
-      this.variantOptionsTablePasswordField = variantOptionsTablePasswordField;
+      this.getOrWithNewBackendVariantsConfig().withBackendSettingSourceFieldName(LegacyBackendVariantSetting.PASSWORD, variantOptionsTablePasswordField);
    }
 
 
@@ -572,30 +539,22 @@ public class QBackendMetaData implements TopLevelMetaDataInterface
    /*******************************************************************************
     ** Fluent setter for variantOptionsTablePasswordField
     *******************************************************************************/
+   @Deprecated(since = "Replaced by backendVariantsConfig.backendSettingSourceFieldNameMap")
    public QBackendMetaData withVariantOptionsTablePasswordField(String variantOptionsTablePasswordField)
    {
-      this.variantOptionsTablePasswordField = variantOptionsTablePasswordField;
+      this.setVariantOptionsTablePasswordField(variantOptionsTablePasswordField);
       return (this);
    }
 
 
 
-   /*******************************************************************************
-    ** Getter for variantOptionsTableApiKeyField
-    *******************************************************************************/
-   public String getVariantOptionsTableApiKeyField()
-   {
-      return (this.variantOptionsTableApiKeyField);
-   }
-
-
-
    /*******************************************************************************
     ** Setter for variantOptionsTableApiKeyField
     *******************************************************************************/
+   @Deprecated(since = "Replaced by backendVariantsConfig.backendSettingSourceFieldNameMap")
    public void setVariantOptionsTableApiKeyField(String variantOptionsTableApiKeyField)
    {
-      this.variantOptionsTableApiKeyField = variantOptionsTableApiKeyField;
+      this.getOrWithNewBackendVariantsConfig().withBackendSettingSourceFieldName(LegacyBackendVariantSetting.API_KEY, variantOptionsTableApiKeyField);
    }
 
 
@@ -603,30 +562,22 @@ public class QBackendMetaData implements TopLevelMetaDataInterface
    /*******************************************************************************
     ** Fluent setter for variantOptionsTableApiKeyField
     *******************************************************************************/
+   @Deprecated(since = "Replaced by backendVariantsConfig.backendSettingSourceFieldNameMap")
    public QBackendMetaData withVariantOptionsTableApiKeyField(String variantOptionsTableApiKeyField)
    {
-      this.variantOptionsTableApiKeyField = variantOptionsTableApiKeyField;
+      this.setVariantOptionsTableApiKeyField(variantOptionsTableApiKeyField);
       return (this);
    }
 
 
 
-   /*******************************************************************************
-    ** Getter for variantOptionsTableName
-    *******************************************************************************/
-   public String getVariantOptionsTableName()
-   {
-      return (this.variantOptionsTableName);
-   }
-
-
-
    /*******************************************************************************
     ** Setter for variantOptionsTableName
     *******************************************************************************/
+   @Deprecated(since = "Replaced by backendVariantsConfig.tableName")
    public void setVariantOptionsTableName(String variantOptionsTableName)
    {
-      this.variantOptionsTableName = variantOptionsTableName;
+      this.getOrWithNewBackendVariantsConfig().withOptionsTableName(variantOptionsTableName);
    }
 
 
@@ -634,9 +585,10 @@ public class QBackendMetaData implements TopLevelMetaDataInterface
    /*******************************************************************************
     ** Fluent setter for variantOptionsTableName
     *******************************************************************************/
+   @Deprecated(since = "Replaced by backendVariantsConfig.tableName")
    public QBackendMetaData withVariantOptionsTableName(String variantOptionsTableName)
    {
-      this.variantOptionsTableName = variantOptionsTableName;
+      this.setVariantOptionsTableName(variantOptionsTableName);
       return (this);
    }
 
@@ -651,22 +603,15 @@ public class QBackendMetaData implements TopLevelMetaDataInterface
       qInstance.addBackend(this);
    }
 
-   /*******************************************************************************
-    ** Getter for variantOptionsTableClientIdField
-    *******************************************************************************/
-   public String getVariantOptionsTableClientIdField()
-   {
-      return (this.variantOptionsTableClientIdField);
-   }
-
 
 
    /*******************************************************************************
     ** Setter for variantOptionsTableClientIdField
     *******************************************************************************/
+   @Deprecated(since = "Replaced by backendVariantsConfig.backendSettingSourceFieldNameMap")
    public void setVariantOptionsTableClientIdField(String variantOptionsTableClientIdField)
    {
-      this.variantOptionsTableClientIdField = variantOptionsTableClientIdField;
+      this.getOrWithNewBackendVariantsConfig().withBackendSettingSourceFieldName(LegacyBackendVariantSetting.CLIENT_ID, variantOptionsTableClientIdField);
    }
 
 
@@ -674,30 +619,22 @@ public class QBackendMetaData implements TopLevelMetaDataInterface
    /*******************************************************************************
     ** Fluent setter for variantOptionsTableClientIdField
     *******************************************************************************/
+   @Deprecated(since = "Replaced by backendVariantsConfig.backendSettingSourceFieldNameMap")
    public QBackendMetaData withVariantOptionsTableClientIdField(String variantOptionsTableClientIdField)
    {
-      this.variantOptionsTableClientIdField = variantOptionsTableClientIdField;
+      this.setVariantOptionsTableClientIdField(variantOptionsTableClientIdField);
       return (this);
    }
 
 
 
-   /*******************************************************************************
-    ** Getter for variantOptionsTableClientSecretField
-    *******************************************************************************/
-   public String getVariantOptionsTableClientSecretField()
-   {
-      return (this.variantOptionsTableClientSecretField);
-   }
-
-
-
    /*******************************************************************************
     ** Setter for variantOptionsTableClientSecretField
     *******************************************************************************/
+   @Deprecated(since = "Replaced by backendVariantsConfig.backendSettingSourceFieldNameMap")
    public void setVariantOptionsTableClientSecretField(String variantOptionsTableClientSecretField)
    {
-      this.variantOptionsTableClientSecretField = variantOptionsTableClientSecretField;
+      this.getOrWithNewBackendVariantsConfig().withBackendSettingSourceFieldName(LegacyBackendVariantSetting.CLIENT_SECRET, variantOptionsTableClientSecretField);
    }
 
 
@@ -705,11 +642,55 @@ public class QBackendMetaData implements TopLevelMetaDataInterface
    /*******************************************************************************
     ** Fluent setter for variantOptionsTableClientSecretField
     *******************************************************************************/
+   @Deprecated(since = "Replaced by backendVariantsConfig.backendSettingSourceFieldNameMap")
    public QBackendMetaData withVariantOptionsTableClientSecretField(String variantOptionsTableClientSecretField)
    {
-      this.variantOptionsTableClientSecretField = variantOptionsTableClientSecretField;
+      this.setVariantOptionsTableClientSecretField(variantOptionsTableClientSecretField);
       return (this);
    }
 
 
+
+   /*******************************************************************************
+    ** Getter for backendVariantsConfig
+    *******************************************************************************/
+   public BackendVariantsConfig getBackendVariantsConfig()
+   {
+      return (this.backendVariantsConfig);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for backendVariantsConfig
+    *******************************************************************************/
+   public void setBackendVariantsConfig(BackendVariantsConfig backendVariantsConfig)
+   {
+      this.backendVariantsConfig = backendVariantsConfig;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for backendVariantsConfig
+    *******************************************************************************/
+   public QBackendMetaData withBackendVariantsConfig(BackendVariantsConfig backendVariantsConfig)
+   {
+      this.backendVariantsConfig = backendVariantsConfig;
+      return (this);
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   private BackendVariantsConfig getOrWithNewBackendVariantsConfig()
+   {
+      if(backendVariantsConfig == null)
+      {
+         setBackendVariantsConfig(new BackendVariantsConfig());
+      }
+      return backendVariantsConfig;
+   }
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/QInstance.java

@@ -43,6 +43,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.audits.QAuditRules;
 import com.kingsrook.qqq.backend.core.model.metadata.authentication.QAuthenticationMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.automation.QAutomationProviderMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.branding.QBrandingMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
 import com.kingsrook.qqq.backend.core.model.metadata.dashboard.QWidgetMetaDataInterface;
 import com.kingsrook.qqq.backend.core.model.metadata.frontend.AppTreeNode;
 import com.kingsrook.qqq.backend.core.model.metadata.frontend.AppTreeNodeType;
@@ -55,6 +56,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.permissions.QPermissionRule
 import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSource;
 import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.processes.QStepMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.qbits.QBitMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.queues.QQueueMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.queues.QQueueProviderMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportMetaData;
@@ -88,6 +90,7 @@ public class QInstance
    ////////////////////////////////////////////////////////////////////////////////////////////
    // Important to use LinkedHashmap here, to preserve the order in which entries are added. //
    ////////////////////////////////////////////////////////////////////////////////////////////
+   private Map<String, QBitMetaData>             qBits                = new LinkedHashMap<>();
    private Map<String, QTableMetaData>           tables               = new LinkedHashMap<>();
    private Map<String, QJoinMetaData>            joins                = new LinkedHashMap<>();
    private Map<String, QPossibleValueSource>     possibleValueSources = new LinkedHashMap<>();
@@ -113,6 +116,8 @@ public class QInstance
    private QPermissionRules defaultPermissionRules = QPermissionRules.defaultInstance();
    private QAuditRules      defaultAuditRules      = QAuditRules.defaultInstanceLevelNone();
 
+   private QCodeReference metaDataFilter = null;
+
    //////////////////////////////////////////////////////////////////////////////////////
    // todo - lock down the object (no more changes allowed) after it's been validated? //
    //  if doing so, may need to copy all of the collections into read-only versions... //
@@ -1485,4 +1490,100 @@ public class QInstance
       QInstanceHelpContentManager.removeHelpContentByRoleSetFromList(roles, listForSlot);
    }
 
+
+
+   /*******************************************************************************
+    ** Getter for metaDataFilter
+    *******************************************************************************/
+   public QCodeReference getMetaDataFilter()
+   {
+      return (this.metaDataFilter);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for metaDataFilter
+    *******************************************************************************/
+   public void setMetaDataFilter(QCodeReference metaDataFilter)
+   {
+      this.metaDataFilter = metaDataFilter;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for metaDataFilter
+    *******************************************************************************/
+   public QInstance withMetaDataFilter(QCodeReference metaDataFilter)
+   {
+      this.metaDataFilter = metaDataFilter;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public void addQBit(QBitMetaData qBitMetaData)
+   {
+      List<String> missingParts = new ArrayList<>();
+      if(!StringUtils.hasContent(qBitMetaData.getGroupId()))
+      {
+         missingParts.add("groupId");
+      }
+      if(!StringUtils.hasContent(qBitMetaData.getArtifactId()))
+      {
+         missingParts.add("artifactId");
+      }
+      if(!StringUtils.hasContent(qBitMetaData.getVersion()))
+      {
+         missingParts.add("version");
+
+      }
+      if(!missingParts.isEmpty())
+      {
+         throw (new IllegalArgumentException("Attempted to add a qBit without a " + StringUtils.joinWithCommasAndAnd(missingParts)));
+      }
+
+      String name = qBitMetaData.getName();
+      if(this.qBits.containsKey(name))
+      {
+         throw (new IllegalArgumentException("Attempted to add a second qBit with name (formed from 'groupId:artifactId:version[:namespace]'): " + name));
+      }
+      this.qBits.put(name, qBitMetaData);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for qBits
+    *******************************************************************************/
+   public Map<String, QBitMetaData> getQBits()
+   {
+      return (this.qBits);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for qBits
+    *******************************************************************************/
+   public void setQBits(Map<String, QBitMetaData> qBits)
+   {
+      this.qBits = qBits;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for qBits
+    *******************************************************************************/
+   public QInstance withQBits(Map<String, QBitMetaData> qBits)
+   {
+      this.qBits = qBits;
+      return (this);
+   }
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/audits/AuditLevel.java

@@ -30,4 +30,5 @@ public enum AuditLevel
    NONE,
    RECORD,
    FIELD
+   // idea:  only audit changes to fields, e.g., on edit.  though, is that a different dimension than this?
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/authentication/QAuthenticationMetaData.java

@@ -177,7 +177,7 @@ public class QAuthenticationMetaData implements TopLevelMetaDataInterface
    /*******************************************************************************
     **
     *******************************************************************************/
-   public QAuthenticationMetaData withVales(Map<String, String> values)
+   public QAuthenticationMetaData withValues(Map<String, String> values)
    {
       this.values = values;
       return (this);

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/code/QCodeReferenceLambda.java

@@ -0,0 +1,58 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.model.metadata.code;
+
+
+/*******************************************************************************
+ ** Specialized type of QCodeReference that takes a lambda function object.
+ **
+ ** Originally intended for more concise setup of backend steps in tests - but,
+ ** may be generally useful.
+ *******************************************************************************/
+public class QCodeReferenceLambda<T> extends QCodeReference
+{
+   private final T lambda;
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   public QCodeReferenceLambda(T lambda)
+   {
+      this.lambda = lambda;
+      this.setCodeType(QCodeType.JAVA);
+      this.setName("[Lambda:" + lambda.toString() + "]");
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for lambda
+    **
+    *******************************************************************************/
+   public T getLambda()
+   {
+      return lambda;
+   }
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/dashboard/QWidgetMetaData.java

@@ -30,6 +30,7 @@ import java.util.List;
 import java.util.Map;
 import java.util.Set;
 import com.kingsrook.qqq.backend.core.instances.QInstanceHelpContentManager;
+import com.kingsrook.qqq.backend.core.instances.validation.plugins.QInstanceValidatorPluginInterface;
 import com.kingsrook.qqq.backend.core.model.dashboard.widgets.WidgetType;
 import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
 import com.kingsrook.qqq.backend.core.model.metadata.help.HelpRole;
@@ -69,6 +70,7 @@ public class QWidgetMetaData implements QWidgetMetaDataInterface
 
    protected Map<String, Serializable> defaultValues = new LinkedHashMap<>();
 
+   protected QInstanceValidatorPluginInterface<QWidgetMetaDataInterface> validatorPlugin;
 
 
    /*******************************************************************************
@@ -764,4 +766,35 @@ public class QWidgetMetaData implements QWidgetMetaDataInterface
       QInstanceHelpContentManager.removeHelpContentByRoleSetFromList(roles, listForSlot);
    }
 
+
+   /*******************************************************************************
+    ** Getter for validatorPlugin
+    *******************************************************************************/
+   public QInstanceValidatorPluginInterface<QWidgetMetaDataInterface> getValidatorPlugin()
+   {
+      return (this.validatorPlugin);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for validatorPlugin
+    *******************************************************************************/
+   public void setValidatorPlugin(QInstanceValidatorPluginInterface<QWidgetMetaDataInterface> validatorPlugin)
+   {
+      this.validatorPlugin = validatorPlugin;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for validatorPlugin
+    *******************************************************************************/
+   public QWidgetMetaData withValidatorPlugin(QInstanceValidatorPluginInterface<QWidgetMetaDataInterface> validatorPlugin)
+   {
+      this.validatorPlugin = validatorPlugin;
+      return (this);
+   }
+
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/dashboard/QWidgetMetaDataInterface.java

@@ -26,6 +26,7 @@ import java.io.Serializable;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
+import com.kingsrook.qqq.backend.core.instances.validation.plugins.QInstanceValidatorPluginInterface;
 import com.kingsrook.qqq.backend.core.logging.QLogger;
 import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
 import com.kingsrook.qqq.backend.core.model.metadata.TopLevelMetaDataInterface;
@@ -277,5 +278,13 @@ public interface QWidgetMetaDataInterface extends MetaDataWithPermissionRules, T
       qInstance.addWidget(this);
    }
 
+
+   /***************************************************************************
+    ** let the widget include an instance validator plugin
+    ***************************************************************************/
+   default QInstanceValidatorPluginInterface<QWidgetMetaDataInterface> getValidatorPlugin()
+   {
+      return (null);
+   }
 }
 

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/AdornmentType.java

@@ -23,8 +23,12 @@ package com.kingsrook.qqq.backend.core.model.metadata.fields;
 
 
 import java.io.Serializable;
+import java.net.URLEncoder;
+import java.nio.charset.StandardCharsets;
+import java.util.Objects;
 import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.PossibleValueEnum;
 import com.kingsrook.qqq.backend.core.utils.Pair;
+import com.kingsrook.qqq.backend.core.utils.ValueUtils;
 import com.kingsrook.qqq.backend.core.utils.collections.MapBuilder;
 
 
@@ -41,20 +45,22 @@ public enum AdornmentType
    RENDER_HTML,
    REVEAL,
    FILE_DOWNLOAD,
+   FILE_UPLOAD,
+   TOOLTIP,
    ERROR;
    //////////////////////////////////////////////////////////////////////////
    // keep these values in sync with AdornmentType.ts in qqq-frontend-core //
    //////////////////////////////////////////////////////////////////////////
 
 
-
    /*******************************************************************************
     **
     *******************************************************************************/
    public interface LinkValues
    {
-      String TARGET               = "target";
-      String TO_RECORD_FROM_TABLE = "toRecordFromTable";
+      String TARGET                       = "target";
+      String TO_RECORD_FROM_TABLE         = "toRecordFromTable";
+      String TO_RECORD_FROM_TABLE_DYNAMIC = "toRecordFromTableDynamic";
    }
 
 
@@ -68,6 +74,11 @@ public enum AdornmentType
       String DEFAULT_EXTENSION = "defaultExtension";
       String DEFAULT_MIME_TYPE = "defaultMimeType";
 
+      String SUPPLEMENTAL_PROCESS_NAME   = "supplementalProcessName";
+      String SUPPLEMENTAL_CODE_REFERENCE = "supplementalCodeReference";
+
+      String DOWNLOAD_URL_DYNAMIC = "downloadUrlDynamic";
+
       ////////////////////////////////////////////////////
       // use these two together, as in:                 //
       // FILE_NAME_FORMAT = "Order %s Packing Slip.pdf" //
@@ -75,6 +86,17 @@ public enum AdornmentType
       ////////////////////////////////////////////////////
       String FILE_NAME_FORMAT        = "fileNameFormat";
       String FILE_NAME_FORMAT_FIELDS = "fileNameFormatFields";
+
+      /***************************************************************************
+       **
+       ***************************************************************************/
+      static String makeFieldDownloadUrl(String tableName, Serializable primaryKey, String fieldName, String fileName)
+      {
+         return ("/data/" + tableName + "/"
+            + URLEncoder.encode(Objects.requireNonNullElse(ValueUtils.getValueAsString(primaryKey), ""), StandardCharsets.UTF_8).replace("+", "%20") + "/"
+            + fieldName + "/"
+            + URLEncoder.encode(Objects.requireNonNullElse(fileName, ""), StandardCharsets.UTF_8).replace("+", "%20"));
+      }
    }
 
 
@@ -164,4 +186,76 @@ public enum AdornmentType
       }
    }
 
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public static class FileUploadAdornment
+   {
+      public static String FORMAT = "format";
+      public static String WIDTH  = "width";
+
+
+
+      /***************************************************************************
+       **
+       ***************************************************************************/
+      public static FieldAdornment newFieldAdornment()
+      {
+         return (new FieldAdornment(AdornmentType.FILE_UPLOAD));
+      }
+
+
+
+      /***************************************************************************
+       **
+       ***************************************************************************/
+      public static Pair<String, String> formatDragAndDrop()
+      {
+         return (Pair.of(FORMAT, "dragAndDrop"));
+      }
+
+
+
+      /***************************************************************************
+       **
+       ***************************************************************************/
+      public static Pair<String, String> formatButton()
+      {
+         return (Pair.of(FORMAT, "button"));
+      }
+
+
+
+      /***************************************************************************
+       **
+       ***************************************************************************/
+      public static Pair<String, String> widthFull()
+      {
+         return (Pair.of(WIDTH, "full"));
+      }
+
+
+
+      /***************************************************************************
+       **
+       ***************************************************************************/
+      public static Pair<String, String> widthHalf()
+      {
+         return (Pair.of(WIDTH, "half"));
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public interface TooltipValues
+   {
+      String STATIC_TEXT = "staticText";
+      String TOOLTIP_DYNAMIC = "tooltipDynamic";
+   }
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/FieldAdornment.java

@@ -177,7 +177,7 @@ public class FieldAdornment
     ** Fluent setter for values
     **
     *******************************************************************************/
-   public FieldAdornment withValue(Pair<String, Serializable> value)
+   public FieldAdornment withValue(Pair<String, ? extends Serializable> value)
    {
       return (withValue(value.getA(), value.getB()));
    }