CE-1328 Add job status messages

- remove MagicNumber
- add MissingJavadocType
- remove rules about contents of javadocs

.circleci/config.yml

@@ -98,6 +98,31 @@ commands:
             - ~/.m2
           key: v1-dependencies-{{ checksum "pom.xml" }}
 
+  install_asciidoctor:
+    steps:
+      - checkout
+      - run:
+          name: Install asciidoctor
+          command: |
+            sudo apt-get update
+            sudo apt install -y asciidoctor
+
+  run_asciidoctor:
+    steps:
+      - run:
+          name: Run asciidoctor
+          command: |
+            cd docs
+            asciidoctor -a docinfo=shared index.adoc
+
+  upload_docs_site:
+    steps:
+      - run:
+          name: scp html to justinsgotskinnylegs.com
+          command: |
+            cd docs
+            scp index.html dkelkhoff@45.79.44.221:/mnt/first-volume/dkelkhoff/nginx/html/justinsgotskinnylegs.com/qqq-docs.html
+
 jobs:
   mvn_test:
     executor: localstack/default
@@ -114,6 +139,13 @@ jobs:
       - mvn_verify
       - mvn_jar_deploy
 
+  publish_asciidoc:
+    executor: localstack/default
+    steps:
+      - install_asciidoctor
+      - run_asciidoctor
+      - upload_docs_site
+
 workflows:
   test_only:
     jobs:
@@ -121,7 +153,7 @@ workflows:
           context: [ qqq-maven-registry-credentials, build-qqq-sample-app ]
           filters:
             branches:
-              ignore: /dev/
+              ignore: /(dev|integration.*)/
             tags:
               ignore: /(version|snapshot)-.*/
 
@@ -131,7 +163,10 @@ workflows:
           context: [ qqq-maven-registry-credentials, build-qqq-sample-app ]
           filters:
             branches:
-              only: /dev/
+              only: /(dev|integration.*)/
             tags:
               only: /(version|snapshot)-.*/
-
+      - publish_asciidoc:
+          filters:
+            branches:
+              only: /dev/

.github/actions/install_asciidoctor/action.yml

@@ -0,0 +1,10 @@
+name: install_asciidoctor
+runs:
+  using: composite
+  steps:
+  - uses: actions/checkout@v4.1.0
+  - name: Install asciidoctor
+    run: |-
+      sudo apt-get update
+      sudo apt install -y asciidoctor
+    shell: bash

.github/actions/install_java17/action.yml

@@ -0,0 +1,16 @@
+name: install_java17
+runs:
+  using: composite
+  steps:
+  - name: Install Java 17
+    run: |-
+      sudo apt-get update
+      sudo apt install -y openjdk-17-jdk
+      sudo rm /etc/alternatives/java
+      sudo ln -s /usr/lib/jvm/java-17-openjdk-amd64/bin/java /etc/alternatives/java
+    shell: bash
+  - name: Install html2text
+    run: |-
+      sudo apt-get update
+      sudo apt-get install -y html2text
+    shell: bash

.github/actions/mvn_jar_deploy/action.yml

@@ -0,0 +1,22 @@
+name: mvn_jar_deploy
+runs:
+  using: composite
+  steps:
+  - uses: actions/checkout@v4.1.0
+  - name: Adjust pom version
+    run: ".circleci/adjust-pom-version.sh"
+    shell: bash
+  - name: restore_cache
+    uses: actions/cache@v3.3.2
+    with:
+      key: v1-dependencies-{{ checksum "pom.xml" }}
+      path: UPDATE_ME
+      restore-keys: v1-dependencies-{{ checksum "pom.xml" }}
+  - name: Run Maven Jar Deploy
+    run: mvn -s .circleci/mvn-settings.xml -T4 flatten:flatten jar:jar deploy:deploy
+    shell: bash
+  - name: save_cache
+    uses: actions/cache@v3.3.2
+    with:
+      path: "~/.m2"
+      key: v1-dependencies-{{ checksum "pom.xml" }}

.github/actions/mvn_verify/action.yml

@@ -0,0 +1,61 @@
+name: mvn_verify
+runs:
+  using: composite
+  steps:
+  - uses: actions/checkout@v4.1.0
+  - name: restore_cache
+    uses: actions/cache@v3.3.2
+    with:
+      key: v1-dependencies-{{ checksum "pom.xml" }}
+      path: UPDATE_ME
+      restore-keys: v1-dependencies-{{ checksum "pom.xml" }}
+  - name: Write .env
+    run: echo "RDBMS_PASSWORD=$RDBMS_PASSWORD" >> qqq-sample-project/.env
+    shell: bash
+  - name: Run Maven Verify
+    run: mvn -s .circleci/mvn-settings.xml -T4 verify
+    shell: bash
+  - uses: "./.github/actions/store_jacoco_site"
+    with:
+      module: qqq-backend-core
+  - uses: "./.github/actions/store_jacoco_site"
+    with:
+      module: qqq-backend-module-filesystem
+  - uses: "./.github/actions/store_jacoco_site"
+    with:
+      module: qqq-backend-module-rdbms
+  - uses: "./.github/actions/store_jacoco_site"
+    with:
+      module: qqq-backend-module-api
+  - uses: "./.github/actions/store_jacoco_site"
+    with:
+      module: qqq-middleware-api
+  - uses: "./.github/actions/store_jacoco_site"
+    with:
+      module: qqq-middleware-javalin
+  - uses: "./.github/actions/store_jacoco_site"
+    with:
+      module: qqq-middleware-picocli
+  - uses: "./.github/actions/store_jacoco_site"
+    with:
+      module: qqq-middleware-slack
+  - uses: "./.github/actions/store_jacoco_site"
+    with:
+      module: qqq-language-support-javascript
+  - uses: "./.github/actions/store_jacoco_site"
+    with:
+      module: qqq-sample-project
+  - name: Save test results
+    run: |-
+      mkdir -p ~/test-results/junit/
+      find . -type f -regex ".*/target/surefire-reports/.*xml" -exec cp {} ~/test-results/junit/ \;
+    if: always()
+    shell: bash
+  - uses: actions/upload-artifact@v4.1.0
+    with:
+      path: "~/test-results"
+  - name: save_cache
+    uses: actions/cache@v3.3.2
+    with:
+      path: "~/.m2"
+      key: v1-dependencies-{{ checksum "pom.xml" }}

.github/actions/run_asciidoctor/action.yml

@@ -0,0 +1,9 @@
+name: run_asciidoctor
+runs:
+  using: composite
+  steps:
+  - name: Run asciidoctor
+    run: |-
+      cd docs
+      asciidoctor -a docinfo=shared index.adoc
+    shell: bash

.github/actions/store_jacoco_site/action.yml

@@ -0,0 +1,13 @@
+name: store_jacoco_site
+inputs:
+  module:
+    required: false
+runs:
+  using: composite
+  steps:
+  - uses: actions/upload-artifact@v4.1.0
+    with:
+      path: "${{ inputs.module }}/target/site/jacoco/index.html"
+  - uses: actions/upload-artifact@v4.1.0
+    with:
+      path: "${{ inputs.module }}/target/site/jacoco/jacoco-resources"

.github/actions/upload_docs_site/action.yml

@@ -0,0 +1,9 @@
+name: upload_docs_site
+runs:
+  using: composite
+  steps:
+  - name: scp html to justinsgotskinnylegs.com
+    run: |-
+      cd docs
+      scp index.html dkelkhoff@45.79.44.221:/mnt/first-volume/dkelkhoff/nginx/html/justinsgotskinnylegs.com/qqq-docs.html
+    shell: bash

.github/workflows/codacy.yml

@@ -0,0 +1,61 @@
+# This workflow uses actions that are not certified by GitHub.
+# They are provided by a third-party and are governed by
+# separate terms of service, privacy policy, and support
+# documentation.
+
+# This workflow checks out code, performs a Codacy security scan
+# and integrates the results with the
+# GitHub Advanced Security code scanning feature.  For more information on
+# the Codacy security scan action usage and parameters, see
+# https://github.com/codacy/codacy-analysis-cli-action.
+# For more information on Codacy Analysis CLI in general, see
+# https://github.com/codacy/codacy-analysis-cli.
+
+name: Codacy Security Scan
+
+on:
+  push:
+    branches: [ "security" ]
+  pull_request:
+    # The branches below must be a subset of the branches above
+    branches: [ "security" ]
+  schedule:
+    - cron: '26 5 * * 4'
+
+permissions:
+  contents: read
+
+jobs:
+  codacy-security-scan:
+    permissions:
+      contents: read # for actions/checkout to fetch code
+      security-events: write # for github/codeql-action/upload-sarif to upload SARIF results
+      actions: read # only required for a private repository by github/codeql-action/upload-sarif to get the Action run status
+    name: Codacy Security Scan
+    runs-on: ubuntu-latest
+    steps:
+      # Checkout the repository to the GitHub Actions runner
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      # Execute Codacy Analysis CLI and generate a SARIF output with the security issues identified during the analysis
+      - name: Run Codacy Analysis CLI
+        uses: codacy/codacy-analysis-cli-action@d840f886c4bd4edc059706d09c6a1586111c540b
+        with:
+          # Check https://github.com/codacy/codacy-analysis-cli#project-token to get your project token from your Codacy repository
+          # You can also omit the token and run the tools that support default configurations
+          project-token: ${{ secrets.CODACY_PROJECT_TOKEN }}
+          verbose: true
+          output: results.sarif
+          format: sarif
+          # Adjust severity of non-security issues
+          gh-code-scanning-compat: true
+          # Force 0 exit code to allow SARIF file generation
+          # This will handover control about PR rejection to the GitHub side
+          max-allowed-issues: 2147483647
+
+      # Upload the SARIF file generated in the previous step
+      - name: Upload SARIF results file
+        uses: github/codeql-action/upload-sarif@v2
+        with:
+          sarif_file: results.sarif

.github/workflows/codeql.yml

@@ -0,0 +1,93 @@
+# For most projects, this workflow file will not need changing; you simply need
+# to commit it to your repository.
+#
+# You may wish to alter this file to override the set of languages analyzed,
+# or to provide custom queries or build logic.
+#
+# ******** NOTE ********
+# We have attempted to detect the languages in your repository. Please check
+# the `language` matrix defined below to confirm you have the correct set of
+# supported CodeQL languages.
+#
+name: "CodeQL"
+
+on:
+  push:
+    branches: [ "security" ]
+  pull_request:
+    branches: [ "security" ]
+  schedule:
+    - cron: '31 10 * * 3'
+
+jobs:
+  analyze:
+    name: Analyze (${{ matrix.language }})
+    # Runner size impacts CodeQL analysis time. To learn more, please see:
+    #   - https://gh.io/recommended-hardware-resources-for-running-codeql
+    #   - https://gh.io/supported-runners-and-hardware-resources
+    #   - https://gh.io/using-larger-runners (GitHub.com only)
+    # Consider using larger runners or machines with greater resources for possible analysis time improvements.
+    runs-on: ${{ (matrix.language == 'swift' && 'macos-latest') || 'ubuntu-latest' }}
+    timeout-minutes: ${{ (matrix.language == 'swift' && 120) || 360 }}
+    permissions:
+      # required for all workflows
+      security-events: write
+
+      # required to fetch internal or private CodeQL packs
+      packages: read
+
+      # only required for workflows in private repositories
+      actions: read
+      contents: read
+
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+        - language: java-kotlin
+          build-mode: none # This mode only analyzes Java. Set this to 'autobuild' or 'manual' to analyze Kotlin too.
+        # CodeQL supports the following values keywords for 'language': 'c-cpp', 'csharp', 'go', 'java-kotlin', 'javascript-typescript', 'python', 'ruby', 'swift'
+        # Use `c-cpp` to analyze code written in C, C++ or both
+        # Use 'java-kotlin' to analyze code written in Java, Kotlin or both
+        # Use 'javascript-typescript' to analyze code written in JavaScript, TypeScript or both
+        # To learn more about changing the languages that are analyzed or customizing the build mode for your analysis,
+        # see https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/customizing-your-advanced-setup-for-code-scanning.
+        # If you are analyzing a compiled language, you can modify the 'build-mode' for that language to customize how
+        # your codebase is analyzed, see https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/codeql-code-scanning-for-compiled-languages
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v4
+
+    # Initializes the CodeQL tools for scanning.
+    - name: Initialize CodeQL
+      uses: github/codeql-action/init@v3
+      with:
+        languages: ${{ matrix.language }}
+        build-mode: ${{ matrix.build-mode }}
+        # If you wish to specify custom queries, you can do so here or in a config file.
+        # By default, queries listed here will override any specified in a config file.
+        # Prefix the list here with "+" to use these queries and those in the config file.
+
+        # For more details on CodeQL's query packs, refer to: https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs
+        # queries: security-extended,security-and-quality
+
+    # If the analyze step fails for one of the languages you are analyzing with
+    # "We were unable to automatically build your code", modify the matrix above
+    # to set the build mode to "manual" for that language. Then modify this step
+    # to build your code.
+    # ℹ️ Command-line programs to run using the OS shell.
+    # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun
+    - if: matrix.build-mode == 'manual'
+      shell: bash
+      run: |
+        echo 'If you are using a "manual" build mode for one or more of the' \
+          'languages you are analyzing, replace this with the commands to build' \
+          'your code, for example:'
+        echo '  make bootstrap'
+        echo '  make release'
+        exit 1
+
+    - name: Perform CodeQL Analysis
+      uses: github/codeql-action/analyze@v3
+      with:
+        category: "/language:${{matrix.language}}"

.github/workflows/deploy.yml

@@ -0,0 +1,20 @@
+name: Kingsrook/qqq/deploy
+on:
+  push:
+    branches:
+    - release
+jobs:
+  mvn_deploy:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4.1.0
+    - uses: "./.github/actions/install_java17"
+    - uses: "./.github/actions/mvn_verify"
+    - uses: "./.github/actions/mvn_jar_deploy"
+  publish_asciidoc:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4.1.0
+    - uses: "./.github/actions/install_asciidoctor"
+    - uses: "./.github/actions/run_asciidoctor"
+    - uses: "./.github/actions/upload_docs_site"

.github/workflows/test_only.yml

@@ -0,0 +1,12 @@
+name: Kingsrook/qqq/test_only
+on:
+  push:
+    branches:
+    - release
+jobs:
+  mvn_test:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4.1.0
+    - uses: "./.github/actions/install_java17"
+    - uses: "./.github/actions/mvn_verify"

.gitignore

@@ -35,3 +35,5 @@ hs_err_pid*
 *.swp
 .flattened-pom.xml
 dependency-reduced-pom.xml
+/.env.local
+/.cache/

CONTRIBUTING.md

@@ -0,0 +1,144 @@
+# Contributing to QQQ
+
+First off, thanks for taking the time to contribute! ❤️
+
+All types of contributions are encouraged and valued. See the [Table of Contents](#table-of-contents) for different ways to help and details about how this project handles them. Please make sure to read the relevant section before making your contribution. It will make it a lot easier for us maintainers and smooth out the experience for all involved. The community looks forward to your contributions. 🎉
+
+> And if you like the project, but just don't have time to contribute, that's fine. There are other easy ways to support the project and show your appreciation, which we would also be very happy about:
+> - Star the project
+> - Tweet about it
+> - Refer this project in your project's readme
+> - Mention the project at local meetups and tell your friends/colleagues
+
+
+## Table of Contents
+
+- [Code of Conduct](#code-of-conduct)
+- [I Have a Question](#i-have-a-question)
+- [I Want To Contribute](#i-want-to-contribute)
+- [Reporting Bugs](#reporting-bugs)
+- [Suggesting Enhancements](#suggesting-enhancements)
+- [Your First Code Contribution](#your-first-code-contribution)
+- [Improving The Documentation](#improving-the-documentation)
+- [Styleguides](#styleguides)
+- [Commit Messages](#commit-messages)
+- [Join The Project Team](#join-the-project-team)
+
+
+## Code of Conduct
+
+We do not have a published Code of Conduct at this time.  
+Lacking that, please just follow the [Golden Rule](https://en.wikipedia.org/wiki/Golden_Rule)
+
+
+## I Have a Question
+
+> If you want to ask a question, we assume that you have read the available [Documentation](https://justinsgotskinnylegs.com/qqq-docs.html).
+
+Before you ask a question, it is best to search for existing [Issues](/issues) that might help you. In case you have found a suitable issue and still need clarification, you can write your question in this issue. It is also advisable to search the internet for answers first.
+
+If you then still feel the need to ask a question and need clarification, we recommend the following:
+
+- Open an [Issue](/issues/new).
+- Provide as much context as you can about what you're running into.
+- Provide project and platform versions, depending on what seems relevant.
+
+We will then take care of the issue as soon as possible.
+
+
+
+## I Want To Contribute
+
+> ### Legal Notice 
+> When contributing to this project, you must agree that you have authored 100% of the content, that you have the necessary rights to the content and that the content you contribute may be provided under the project license.
+
+### Reporting Bugs
+
+
+#### Before Submitting a Bug Report
+
+A good bug report shouldn't leave others needing to chase you up for more information. Therefore, we ask you to investigate carefully, collect information and describe the issue in detail in your report. Please complete the following steps in advance to help us fix any potential bug as fast as possible.
+
+- Make sure that you are using the latest version.
+- Determine if your bug is really a bug and not an error on your side e.g. using incompatible environment components/versions (Make sure that you have read the [documentation](https://justinsgotskinnylegs.com/qqq-docs.html). If you are looking for support, you might want to check [this section](#i-have-a-question)).
+- To see if other users have experienced (and potentially already solved) the same issue you are having, check if there is not already a bug report existing for your bug or error in the [bug tracker](issues?q=label%3Abug).
+- Also make sure to search the internet (including Stack Overflow) to see if users outside of the GitHub community have discussed the issue.
+- Collect information about the bug:
+  - Stack trace
+  - OS, Platform and Version (Windows, Linux, macOS, x86, ARM)
+  - Version of the interpreter, compiler, SDK, runtime environment, package manager, depending on what seems relevant.
+  - Possibly your input and the output
+  - Can you reliably reproduce the issue? And can you also reproduce it with older versions?
+
+
+#### How Do I Submit a Good Bug Report?
+
+> You must never report security related issues, vulnerabilities or bugs including sensitive information to the issue tracker, or elsewhere in public. Instead sensitive bugs must be sent by email to <contact@kingsrook.com>.
+
+
+We use GitHub issues to track bugs and errors. If you run into an issue with the project:
+
+- Open an [Issue](/issues/new). (Since we can't be sure at this point whether it is a bug or not, we ask you not to talk about a bug yet and not to label the issue.)
+- Explain the behavior you would expect and the actual behavior.
+- Please provide as much context as possible and describe the *reproduction steps* that someone else can follow to recreate the issue on their own. This usually includes your code. For good bug reports you should isolate the problem and create a reduced test case.
+- Provide the information you collected in the previous section.
+
+Once it's filed:
+
+- The project team will label the issue accordingly.
+- A team member will try to reproduce the issue with your provided steps. If there are no reproduction steps or no obvious way to reproduce the issue, the team will ask you for those steps and mark the issue as `needs-repro`. Bugs with the `needs-repro` tag will not be addressed until they are reproduced.
+- If the team is able to reproduce the issue, it will be marked `needs-fix`, as well as possibly other tags (such as `critical`), and the issue will be left to be [implemented by someone](#your-first-code-contribution).
+
+
+
+
+### Suggesting Enhancements
+
+This section guides you through submitting an enhancement suggestion for QQQ, **including completely new features and minor improvements to existing functionality**. Following these guidelines will help maintainers and the community to understand your suggestion and find related suggestions.
+
+
+#### Before Submitting an Enhancement
+
+- Make sure that you are using the latest version.
+- Read the [documentation](https://justinsgotskinnylegs.com/qqq-docs.html) carefully and find out if the functionality is already covered, maybe by an individual configuration.
+- Perform a [search](/issues) to see if the enhancement has already been suggested. If it has, add a comment to the existing issue instead of opening a new one.
+- Find out whether your idea fits with the scope and aims of the project. It's up to you to make a strong case to convince the project's developers of the merits of this feature. Keep in mind that we want features that will be useful to the majority of our users and not just a small subset. If you're just targeting a minority of users, consider writing an add-on/plugin library.
+
+
+#### How Do I Submit a Good Enhancement Suggestion?
+
+Enhancement suggestions are tracked as [GitHub issues](/issues).
+
+- Use a **clear and descriptive title** for the issue to identify the suggestion.
+- Provide a **step-by-step description of the suggested enhancement** in as many details as possible.
+- **Describe the current behavior** and **explain which behavior you expected to see instead** and why. At this point you can also tell which alternatives do not work for you.
+- You may want to **include screenshots and animated GIFs** which help you demonstrate the steps or point out the part which the suggestion is related to. You can use [this tool](https://www.cockos.com/licecap/) to record GIFs on macOS and Windows, and [this tool](https://github.com/colinkeenan/silentcast) or [this tool](https://github.com/GNOME/byzanz) on Linux. 
+- **Explain why this enhancement would be useful** to most QQQ users. You may also want to point out the other projects that solved it better and which could serve as inspiration.
+
+
+
+### Your First Code Contribution
+
+Unfortunately, we are not prepared at this time for accepting external code contributions in any meaningful way.  We don't have process in place for reviewing pull requests.  So, the best we can do right now would be to accept suggestions via issues, as referenced above.
+
+### Improving The Documentation
+
+Unfortunately, we are not prepared at this time for accepting external documentation contributions in any meaningful way.  We don't have process in place for reviewing pull requests.  So, the best we can do right now would be to accept suggestions via issues, as referenced above.
+
+
+## Styleguides
+
+To be documented at a point when we become ready to accept code contributions.
+
+### Commit Messages
+
+To be documented at a point when we become ready to accept code contributions.
+
+## Join The Project Team
+
+To be documented at a point when we become ready to accept more team members.
+
+
+## Attribution
+This guide is based on the **contributing.md**. [Make your own](https://contributing.md/)!
+

LICENSE.txt

@@ -0,0 +1,619 @@
+                    GNU AFFERO GENERAL PUBLIC LICENSE
+                       Version 3, 19 November 2007
+
+ Copyright (C) 2007 Free Software Foundation, Inc. <https://fsf.org/>
+ Everyone is permitted to copy and distribute verbatim copies
+ of this license document, but changing it is not allowed.
+
+                            Preamble
+
+  The GNU Affero General Public License is a free, copyleft license for
+software and other kinds of works, specifically designed to ensure
+cooperation with the community in the case of network server software.
+
+  The licenses for most software and other practical works are designed
+to take away your freedom to share and change the works.  By contrast,
+our General Public Licenses are intended to guarantee your freedom to
+share and change all versions of a program--to make sure it remains free
+software for all its users.
+
+  When we speak of free software, we are referring to freedom, not
+price.  Our General Public Licenses are designed to make sure that you
+have the freedom to distribute copies of free software (and charge for
+them if you wish), that you receive source code or can get it if you
+want it, that you can change the software or use pieces of it in new
+free programs, and that you know you can do these things.
+
+  Developers that use our General Public Licenses protect your rights
+with two steps: (1) assert copyright on the software, and (2) offer
+you this License which gives you legal permission to copy, distribute
+and/or modify the software.
+
+  A secondary benefit of defending all users' freedom is that
+improvements made in alternate versions of the program, if they
+receive widespread use, become available for other developers to
+incorporate.  Many developers of free software are heartened and
+encouraged by the resulting cooperation.  However, in the case of
+software used on network servers, this result may fail to come about.
+The GNU General Public License permits making a modified version and
+letting the public access it on a server without ever releasing its
+source code to the public.
+
+  The GNU Affero General Public License is designed specifically to
+ensure that, in such cases, the modified source code becomes available
+to the community.  It requires the operator of a network server to
+provide the source code of the modified version running there to the
+users of that server.  Therefore, public use of a modified version, on
+a publicly accessible server, gives the public access to the source
+code of the modified version.
+
+  An older license, called the Affero General Public License and
+published by Affero, was designed to accomplish similar goals.  This is
+a different license, not a version of the Affero GPL, but Affero has
+released a new version of the Affero GPL which permits relicensing under
+this license.
+
+  The precise terms and conditions for copying, distribution and
+modification follow.
+
+                       TERMS AND CONDITIONS
+
+  0. Definitions.
+
+  "This License" refers to version 3 of the GNU Affero General Public License.
+
+  "Copyright" also means copyright-like laws that apply to other kinds of
+works, such as semiconductor masks.
+
+  "The Program" refers to any copyrightable work licensed under this
+License.  Each licensee is addressed as "you".  "Licensees" and
+"recipients" may be individuals or organizations.
+
+  To "modify" a work means to copy from or adapt all or part of the work
+in a fashion requiring copyright permission, other than the making of an
+exact copy.  The resulting work is called a "modified version" of the
+earlier work or a work "based on" the earlier work.
+
+  A "covered work" means either the unmodified Program or a work based
+on the Program.
+
+  To "propagate" a work means to do anything with it that, without
+permission, would make you directly or secondarily liable for
+infringement under applicable copyright law, except executing it on a
+computer or modifying a private copy.  Propagation includes copying,
+distribution (with or without modification), making available to the
+public, and in some countries other activities as well.
+
+  To "convey" a work means any kind of propagation that enables other
+parties to make or receive copies.  Mere interaction with a user through
+a computer network, with no transfer of a copy, is not conveying.
+
+  An interactive user interface displays "Appropriate Legal Notices"
+to the extent that it includes a convenient and prominently visible
+feature that (1) displays an appropriate copyright notice, and (2)
+tells the user that there is no warranty for the work (except to the
+extent that warranties are provided), that licensees may convey the
+work under this License, and how to view a copy of this License.  If
+the interface presents a list of user commands or options, such as a
+menu, a prominent item in the list meets this criterion.
+
+  1. Source Code.
+
+  The "source code" for a work means the preferred form of the work
+for making modifications to it.  "Object code" means any non-source
+form of a work.
+
+  A "Standard Interface" means an interface that either is an official
+standard defined by a recognized standards body, or, in the case of
+interfaces specified for a particular programming language, one that
+is widely used among developers working in that language.
+
+  The "System Libraries" of an executable work include anything, other
+than the work as a whole, that (a) is included in the normal form of
+packaging a Major Component, but which is not part of that Major
+Component, and (b) serves only to enable use of the work with that
+Major Component, or to implement a Standard Interface for which an
+implementation is available to the public in source code form.  A
+"Major Component", in this context, means a major essential component
+(kernel, window system, and so on) of the specific operating system
+(if any) on which the executable work runs, or a compiler used to
+produce the work, or an object code interpreter used to run it.
+
+  The "Corresponding Source" for a work in object code form means all
+the source code needed to generate, install, and (for an executable
+work) run the object code and to modify the work, including scripts to
+control those activities.  However, it does not include the work's
+System Libraries, or general-purpose tools or generally available free
+programs which are used unmodified in performing those activities but
+which are not part of the work.  For example, Corresponding Source
+includes interface definition files associated with source files for
+the work, and the source code for shared libraries and dynamically
+linked subprograms that the work is specifically designed to require,
+such as by intimate data communication or control flow between those
+subprograms and other parts of the work.
+
+  The Corresponding Source need not include anything that users
+can regenerate automatically from other parts of the Corresponding
+Source.
+
+  The Corresponding Source for a work in source code form is that
+same work.
+
+  2. Basic Permissions.
+
+  All rights granted under this License are granted for the term of
+copyright on the Program, and are irrevocable provided the stated
+conditions are met.  This License explicitly affirms your unlimited
+permission to run the unmodified Program.  The output from running a
+covered work is covered by this License only if the output, given its
+content, constitutes a covered work.  This License acknowledges your
+rights of fair use or other equivalent, as provided by copyright law.
+
+  You may make, run and propagate covered works that you do not
+convey, without conditions so long as your license otherwise remains
+in force.  You may convey covered works to others for the sole purpose
+of having them make modifications exclusively for you, or provide you
+with facilities for running those works, provided that you comply with
+the terms of this License in conveying all material for which you do
+not control copyright.  Those thus making or running the covered works
+for you must do so exclusively on your behalf, under your direction
+and control, on terms that prohibit them from making any copies of
+your copyrighted material outside their relationship with you.
+
+  Conveying under any other circumstances is permitted solely under
+the conditions stated below.  Sublicensing is not allowed; section 10
+makes it unnecessary.
+
+  3. Protecting Users' Legal Rights From Anti-Circumvention Law.
+
+  No covered work shall be deemed part of an effective technological
+measure under any applicable law fulfilling obligations under article
+11 of the WIPO copyright treaty adopted on 20 December 1996, or
+similar laws prohibiting or restricting circumvention of such
+measures.
+
+  When you convey a covered work, you waive any legal power to forbid
+circumvention of technological measures to the extent such circumvention
+is effected by exercising rights under this License with respect to
+the covered work, and you disclaim any intention to limit operation or
+modification of the work as a means of enforcing, against the work's
+users, your or third parties' legal rights to forbid circumvention of
+technological measures.
+
+  4. Conveying Verbatim Copies.
+
+  You may convey verbatim copies of the Program's source code as you
+receive it, in any medium, provided that you conspicuously and
+appropriately publish on each copy an appropriate copyright notice;
+keep intact all notices stating that this License and any
+non-permissive terms added in accord with section 7 apply to the code;
+keep intact all notices of the absence of any warranty; and give all
+recipients a copy of this License along with the Program.
+
+  You may charge any price or no price for each copy that you convey,
+and you may offer support or warranty protection for a fee.
+
+  5. Conveying Modified Source Versions.
+
+  You may convey a work based on the Program, or the modifications to
+produce it from the Program, in the form of source code under the
+terms of section 4, provided that you also meet all of these conditions:
+
+    a) The work must carry prominent notices stating that you modified
+    it, and giving a relevant date.
+
+    b) The work must carry prominent notices stating that it is
+    released under this License and any conditions added under section
+    7.  This requirement modifies the requirement in section 4 to
+    "keep intact all notices".
+
+    c) You must license the entire work, as a whole, under this
+    License to anyone who comes into possession of a copy.  This
+    License will therefore apply, along with any applicable section 7
+    additional terms, to the whole of the work, and all its parts,
+    regardless of how they are packaged.  This License gives no
+    permission to license the work in any other way, but it does not
+    invalidate such permission if you have separately received it.
+
+    d) If the work has interactive user interfaces, each must display
+    Appropriate Legal Notices; however, if the Program has interactive
+    interfaces that do not display Appropriate Legal Notices, your
+    work need not make them do so.
+
+  A compilation of a covered work with other separate and independent
+works, which are not by their nature extensions of the covered work,
+and which are not combined with it such as to form a larger program,
+in or on a volume of a storage or distribution medium, is called an
+"aggregate" if the compilation and its resulting copyright are not
+used to limit the access or legal rights of the compilation's users
+beyond what the individual works permit.  Inclusion of a covered work
+in an aggregate does not cause this License to apply to the other
+parts of the aggregate.
+
+  6. Conveying Non-Source Forms.
+
+  You may convey a covered work in object code form under the terms
+of sections 4 and 5, provided that you also convey the
+machine-readable Corresponding Source under the terms of this License,
+in one of these ways:
+
+    a) Convey the object code in, or embodied in, a physical product
+    (including a physical distribution medium), accompanied by the
+    Corresponding Source fixed on a durable physical medium
+    customarily used for software interchange.
+
+    b) Convey the object code in, or embodied in, a physical product
+    (including a physical distribution medium), accompanied by a
+    written offer, valid for at least three years and valid for as
+    long as you offer spare parts or customer support for that product
+    model, to give anyone who possesses the object code either (1) a
+    copy of the Corresponding Source for all the software in the
+    product that is covered by this License, on a durable physical
+    medium customarily used for software interchange, for a price no
+    more than your reasonable cost of physically performing this
+    conveying of source, or (2) access to copy the
+    Corresponding Source from a network server at no charge.
+
+    c) Convey individual copies of the object code with a copy of the
+    written offer to provide the Corresponding Source.  This
+    alternative is allowed only occasionally and noncommercially, and
+    only if you received the object code with such an offer, in accord
+    with subsection 6b.
+
+    d) Convey the object code by offering access from a designated
+    place (gratis or for a charge), and offer equivalent access to the
+    Corresponding Source in the same way through the same place at no
+    further charge.  You need not require recipients to copy the
+    Corresponding Source along with the object code.  If the place to
+    copy the object code is a network server, the Corresponding Source
+    may be on a different server (operated by you or a third party)
+    that supports equivalent copying facilities, provided you maintain
+    clear directions next to the object code saying where to find the
+    Corresponding Source.  Regardless of what server hosts the
+    Corresponding Source, you remain obligated to ensure that it is
+    available for as long as needed to satisfy these requirements.
+
+    e) Convey the object code using peer-to-peer transmission, provided
+    you inform other peers where the object code and Corresponding
+    Source of the work are being offered to the general public at no
+    charge under subsection 6d.
+
+  A separable portion of the object code, whose source code is excluded
+from the Corresponding Source as a System Library, need not be
+included in conveying the object code work.
+
+  A "User Product" is either (1) a "consumer product", which means any
+tangible personal property which is normally used for personal, family,
+or household purposes, or (2) anything designed or sold for incorporation
+into a dwelling.  In determining whether a product is a consumer product,
+doubtful cases shall be resolved in favor of coverage.  For a particular
+product received by a particular user, "normally used" refers to a
+typical or common use of that class of product, regardless of the status
+of the particular user or of the way in which the particular user
+actually uses, or expects or is expected to use, the product.  A product
+is a consumer product regardless of whether the product has substantial
+commercial, industrial or non-consumer uses, unless such uses represent
+the only significant mode of use of the product.
+
+  "Installation Information" for a User Product means any methods,
+procedures, authorization keys, or other information required to install
+and execute modified versions of a covered work in that User Product from
+a modified version of its Corresponding Source.  The information must
+suffice to ensure that the continued functioning of the modified object
+code is in no case prevented or interfered with solely because
+modification has been made.
+
+  If you convey an object code work under this section in, or with, or
+specifically for use in, a User Product, and the conveying occurs as
+part of a transaction in which the right of possession and use of the
+User Product is transferred to the recipient in perpetuity or for a
+fixed term (regardless of how the transaction is characterized), the
+Corresponding Source conveyed under this section must be accompanied
+by the Installation Information.  But this requirement does not apply
+if neither you nor any third party retains the ability to install
+modified object code on the User Product (for example, the work has
+been installed in ROM).
+
+  The requirement to provide Installation Information does not include a
+requirement to continue to provide support service, warranty, or updates
+for a work that has been modified or installed by the recipient, or for
+the User Product in which it has been modified or installed.  Access to a
+network may be denied when the modification itself materially and
+adversely affects the operation of the network or violates the rules and
+protocols for communication across the network.
+
+  Corresponding Source conveyed, and Installation Information provided,
+in accord with this section must be in a format that is publicly
+documented (and with an implementation available to the public in
+source code form), and must require no special password or key for
+unpacking, reading or copying.
+
+  7. Additional Terms.
+
+  "Additional permissions" are terms that supplement the terms of this
+License by making exceptions from one or more of its conditions.
+Additional permissions that are applicable to the entire Program shall
+be treated as though they were included in this License, to the extent
+that they are valid under applicable law.  If additional permissions
+apply only to part of the Program, that part may be used separately
+under those permissions, but the entire Program remains governed by
+this License without regard to the additional permissions.
+
+  When you convey a copy of a covered work, you may at your option
+remove any additional permissions from that copy, or from any part of
+it.  (Additional permissions may be written to require their own
+removal in certain cases when you modify the work.)  You may place
+additional permissions on material, added by you to a covered work,
+for which you have or can give appropriate copyright permission.
+
+  Notwithstanding any other provision of this License, for material you
+add to a covered work, you may (if authorized by the copyright holders of
+that material) supplement the terms of this License with terms:
+
+    a) Disclaiming warranty or limiting liability differently from the
+    terms of sections 15 and 16 of this License; or
+
+    b) Requiring preservation of specified reasonable legal notices or
+    author attributions in that material or in the Appropriate Legal
+    Notices displayed by works containing it; or
+
+    c) Prohibiting misrepresentation of the origin of that material, or
+    requiring that modified versions of such material be marked in
+    reasonable ways as different from the original version; or
+
+    d) Limiting the use for publicity purposes of names of licensors or
+    authors of the material; or
+
+    e) Declining to grant rights under trademark law for use of some
+    trade names, trademarks, or service marks; or
+
+    f) Requiring indemnification of licensors and authors of that
+    material by anyone who conveys the material (or modified versions of
+    it) with contractual assumptions of liability to the recipient, for
+    any liability that these contractual assumptions directly impose on
+    those licensors and authors.
+
+  All other non-permissive additional terms are considered "further
+restrictions" within the meaning of section 10.  If the Program as you
+received it, or any part of it, contains a notice stating that it is
+governed by this License along with a term that is a further
+restriction, you may remove that term.  If a license document contains
+a further restriction but permits relicensing or conveying under this
+License, you may add to a covered work material governed by the terms
+of that license document, provided that the further restriction does
+not survive such relicensing or conveying.
+
+  If you add terms to a covered work in accord with this section, you
+must place, in the relevant source files, a statement of the
+additional terms that apply to those files, or a notice indicating
+where to find the applicable terms.
+
+  Additional terms, permissive or non-permissive, may be stated in the
+form of a separately written license, or stated as exceptions;
+the above requirements apply either way.
+
+  8. Termination.
+
+  You may not propagate or modify a covered work except as expressly
+provided under this License.  Any attempt otherwise to propagate or
+modify it is void, and will automatically terminate your rights under
+this License (including any patent licenses granted under the third
+paragraph of section 11).
+
+  However, if you cease all violation of this License, then your
+license from a particular copyright holder is reinstated (a)
+provisionally, unless and until the copyright holder explicitly and
+finally terminates your license, and (b) permanently, if the copyright
+holder fails to notify you of the violation by some reasonable means
+prior to 60 days after the cessation.
+
+  Moreover, your license from a particular copyright holder is
+reinstated permanently if the copyright holder notifies you of the
+violation by some reasonable means, this is the first time you have
+received notice of violation of this License (for any work) from that
+copyright holder, and you cure the violation prior to 30 days after
+your receipt of the notice.
+
+  Termination of your rights under this section does not terminate the
+licenses of parties who have received copies or rights from you under
+this License.  If your rights have been terminated and not permanently
+reinstated, you do not qualify to receive new licenses for the same
+material under section 10.
+
+  9. Acceptance Not Required for Having Copies.
+
+  You are not required to accept this License in order to receive or
+run a copy of the Program.  Ancillary propagation of a covered work
+occurring solely as a consequence of using peer-to-peer transmission
+to receive a copy likewise does not require acceptance.  However,
+nothing other than this License grants you permission to propagate or
+modify any covered work.  These actions infringe copyright if you do
+not accept this License.  Therefore, by modifying or propagating a
+covered work, you indicate your acceptance of this License to do so.
+
+  10. Automatic Licensing of Downstream Recipients.
+
+  Each time you convey a covered work, the recipient automatically
+receives a license from the original licensors, to run, modify and
+propagate that work, subject to this License.  You are not responsible
+for enforcing compliance by third parties with this License.
+
+  An "entity transaction" is a transaction transferring control of an
+organization, or substantially all assets of one, or subdividing an
+organization, or merging organizations.  If propagation of a covered
+work results from an entity transaction, each party to that
+transaction who receives a copy of the work also receives whatever
+licenses to the work the party's predecessor in interest had or could
+give under the previous paragraph, plus a right to possession of the
+Corresponding Source of the work from the predecessor in interest, if
+the predecessor has it or can get it with reasonable efforts.
+
+  You may not impose any further restrictions on the exercise of the
+rights granted or affirmed under this License.  For example, you may
+not impose a license fee, royalty, or other charge for exercise of
+rights granted under this License, and you may not initiate litigation
+(including a cross-claim or counterclaim in a lawsuit) alleging that
+any patent claim is infringed by making, using, selling, offering for
+sale, or importing the Program or any portion of it.
+
+  11. Patents.
+
+  A "contributor" is a copyright holder who authorizes use under this
+License of the Program or a work on which the Program is based.  The
+work thus licensed is called the contributor's "contributor version".
+
+  A contributor's "essential patent claims" are all patent claims
+owned or controlled by the contributor, whether already acquired or
+hereafter acquired, that would be infringed by some manner, permitted
+by this License, of making, using, or selling its contributor version,
+but do not include claims that would be infringed only as a
+consequence of further modification of the contributor version.  For
+purposes of this definition, "control" includes the right to grant
+patent sublicenses in a manner consistent with the requirements of
+this License.
+
+  Each contributor grants you a non-exclusive, worldwide, royalty-free
+patent license under the contributor's essential patent claims, to
+make, use, sell, offer for sale, import and otherwise run, modify and
+propagate the contents of its contributor version.
+
+  In the following three paragraphs, a "patent license" is any express
+agreement or commitment, however denominated, not to enforce a patent
+(such as an express permission to practice a patent or covenant not to
+sue for patent infringement).  To "grant" such a patent license to a
+party means to make such an agreement or commitment not to enforce a
+patent against the party.
+
+  If you convey a covered work, knowingly relying on a patent license,
+and the Corresponding Source of the work is not available for anyone
+to copy, free of charge and under the terms of this License, through a
+publicly available network server or other readily accessible means,
+then you must either (1) cause the Corresponding Source to be so
+available, or (2) arrange to deprive yourself of the benefit of the
+patent license for this particular work, or (3) arrange, in a manner
+consistent with the requirements of this License, to extend the patent
+license to downstream recipients.  "Knowingly relying" means you have
+actual knowledge that, but for the patent license, your conveying the
+covered work in a country, or your recipient's use of the covered work
+in a country, would infringe one or more identifiable patents in that
+country that you have reason to believe are valid.
+
+  If, pursuant to or in connection with a single transaction or
+arrangement, you convey, or propagate by procuring conveyance of, a
+covered work, and grant a patent license to some of the parties
+receiving the covered work authorizing them to use, propagate, modify
+or convey a specific copy of the covered work, then the patent license
+you grant is automatically extended to all recipients of the covered
+work and works based on it.
+
+  A patent license is "discriminatory" if it does not include within
+the scope of its coverage, prohibits the exercise of, or is
+conditioned on the non-exercise of one or more of the rights that are
+specifically granted under this License.  You may not convey a covered
+work if you are a party to an arrangement with a third party that is
+in the business of distributing software, under which you make payment
+to the third party based on the extent of your activity of conveying
+the work, and under which the third party grants, to any of the
+parties who would receive the covered work from you, a discriminatory
+patent license (a) in connection with copies of the covered work
+conveyed by you (or copies made from those copies), or (b) primarily
+for and in connection with specific products or compilations that
+contain the covered work, unless you entered into that arrangement,
+or that patent license was granted, prior to 28 March 2007.
+
+  Nothing in this License shall be construed as excluding or limiting
+any implied license or other defenses to infringement that may
+otherwise be available to you under applicable patent law.
+
+  12. No Surrender of Others' Freedom.
+
+  If conditions are imposed on you (whether by court order, agreement or
+otherwise) that contradict the conditions of this License, they do not
+excuse you from the conditions of this License.  If you cannot convey a
+covered work so as to satisfy simultaneously your obligations under this
+License and any other pertinent obligations, then as a consequence you may
+not convey it at all.  For example, if you agree to terms that obligate you
+to collect a royalty for further conveying from those to whom you convey
+the Program, the only way you could satisfy both those terms and this
+License would be to refrain entirely from conveying the Program.
+
+  13. Remote Network Interaction; Use with the GNU General Public License.
+
+  Notwithstanding any other provision of this License, if you modify the
+Program, your modified version must prominently offer all users
+interacting with it remotely through a computer network (if your version
+supports such interaction) an opportunity to receive the Corresponding
+Source of your version by providing access to the Corresponding Source
+from a network server at no charge, through some standard or customary
+means of facilitating copying of software.  This Corresponding Source
+shall include the Corresponding Source for any work covered by version 3
+of the GNU General Public License that is incorporated pursuant to the
+following paragraph.
+
+  Notwithstanding any other provision of this License, you have
+permission to link or combine any covered work with a work licensed
+under version 3 of the GNU General Public License into a single
+combined work, and to convey the resulting work.  The terms of this
+License will continue to apply to the part which is the covered work,
+but the work with which it is combined will remain governed by version
+3 of the GNU General Public License.
+
+  14. Revised Versions of this License.
+
+  The Free Software Foundation may publish revised and/or new versions of
+the GNU Affero General Public License from time to time.  Such new versions
+will be similar in spirit to the present version, but may differ in detail to
+address new problems or concerns.
+
+  Each version is given a distinguishing version number.  If the
+Program specifies that a certain numbered version of the GNU Affero General
+Public License "or any later version" applies to it, you have the
+option of following the terms and conditions either of that numbered
+version or of any later version published by the Free Software
+Foundation.  If the Program does not specify a version number of the
+GNU Affero General Public License, you may choose any version ever published
+by the Free Software Foundation.
+
+  If the Program specifies that a proxy can decide which future
+versions of the GNU Affero General Public License can be used, that proxy's
+public statement of acceptance of a version permanently authorizes you
+to choose that version for the Program.
+
+  Later license versions may give you additional or different
+permissions.  However, no additional obligations are imposed on any
+author or copyright holder as a result of your choosing to follow a
+later version.
+
+  15. Disclaimer of Warranty.
+
+  THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY
+APPLICABLE LAW.  EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT
+HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY
+OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO,
+THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+PURPOSE.  THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM
+IS WITH YOU.  SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF
+ALL NECESSARY SERVICING, REPAIR OR CORRECTION.
+
+  16. Limitation of Liability.
+
+  IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
+WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS
+THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY
+GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE
+USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF
+DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD
+PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS),
+EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF
+SUCH DAMAGES.
+
+  17. Interpretation of Sections 15 and 16.
+
+  If the disclaimer of warranty and limitation of liability provided
+above cannot be given local legal effect according to their terms,
+reviewing courts shall apply local law that most closely approximates
+an absolute waiver of all civil liability in connection with the
+Program, unless a warranty or assumption of liability accompanies a
+copy of the Program in return for a fee.
+
+                     END OF TERMS AND CONDITIONS

README.md

@@ -5,21 +5,34 @@ This is the top-level/parent project of qqq.
 QQQ is a Low-code Application Framework for Engineers.
 
 ## Artifacts
-QQQ can be used with a single bundle or smaller fine grained jars.
-The bundle contains all of the sub-jars.  It is named:
+*Note, this information - well, I'd say it's out of date, but honestly, I don't
+think this was ever accurate, lol.  Either way, it needs re-written, please.
+Should refrence the bom-pom, and there is no "bundle" concept at present.*
 
-```qqq-${version}.jar```
+> QQQ can be used with a single bundle or smaller fine grained jars.
+> The bundle contains all of the sub-jars.  It is named:
+> 
+> ```qqq-${version}.jar```
+> 
+> You can also use fine-grained jars:
+> - `qqq-backend-core`: The core module.  Useful if you're developing other modules.
+> - `qqq-backend-module-rdbms`: Backend module for working with Relational Databases.
+> - `qqq-backend-module-filesystem`: Backend module for working with Filesystems (including AWS S3).
+> - `qqq-middleware-javalin`: Middleware http server.  Procivdes REST API, and/or backing for a web frotnend.
+> - `qqq-middleware-picocli`: Middleware (actually, a front-end, innint?) Command Line interface.
 
-You can also use fine-grained jars:
-- `qqq-backend-core`: The core module.  Useful if you're developing other modules.
-- `qqq-backend-module-rdbms`: Backend module for working with Relational Databases.
-- `qqq-backend-module-filesystem`: Backend module for working with Filesystems (including AWS S3).
-- `qqq-middleware-javalin`: Middleware http server.  Procivdes REST API, and/or backing for a web frotnend.
-- `qqq-middleware-picocli`: Middleware (actually, a front-end, innint?) Command Line interface.
+## Framework Developer Tools/Resources
+### IntelliJ
+There are a few useful IntelliJ settings files, under `qqq-dev-tools/intellij`:
+- Kingsrook_Code_Style.xml
+- Kingsrook_Copyright_Profile.xml
+
+One will likely also want the [Kingsrook Commentator
+Plugin](https://plugins.jetbrains.com/plugin/19325-kingsrook-commentator).
 
 ## License
 QQQ - Low-code Application Framework for Engineers. \
-Copyright (C) 2022.  Kingsrook, LLC \
+Copyright (C) 2020-2024.  Kingsrook, LLC \
 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States \
 contact@kingsrook.com | https://github.com/Kingsrook/
 

SECURITY.md

@@ -0,0 +1,21 @@
+# Security Policy
+
+## Supported Versions
+
+Use this section to tell people about which versions of your project are
+currently being supported with security updates.
+
+| Version | Supported          |
+| ------- | ------------------ |
+| 5.1.x   | :white_check_mark: |
+| 5.0.x   | :x:                |
+| 4.0.x   | :white_check_mark: |
+| < 4.0   | :x:                |
+
+## Reporting a Vulnerability
+
+Use this section to tell people how to report a vulnerability.
+
+Tell them where to go, how often they can expect to get an update on a
+reported vulnerability, what to expect if the vulnerability is accepted or
+declined, etc.

checkstyle/config.xml

@@ -213,18 +213,6 @@
          <property name="tokens" value="VARIABLE_DEF"/>
          <property name="allowSamelineMultipleAnnotations" value="true"/>
       </module>
-      <module name="NonEmptyAtclauseDescription"/>
-      <!-- <module name="JavadocTagContinuationIndentation"/> -->
-      <!--
-      <module name="SummaryJavadoc">
-         <property name="forbiddenSummaryFragments" value="^@return the *|^This method returns |^A [{]@code [a-zA-Z0-9]+[}]( is a )"/>
-      </module>
-      -->
-      <!-- <module name="JavadocParagraph"/> -->
-      <module name="AtclauseOrder">
-         <property name="tagOrder" value="@param, @return, @throws, @deprecated"/>
-         <property name="target" value="CLASS_DEF, INTERFACE_DEF, ENUM_DEF, METHOD_DEF, CTOR_DEF, VARIABLE_DEF"/>
-      </module>
       <module name="JavadocMethod">
          <property name="allowMissingParamTags" value="true"/>
          <property name="allowMissingReturnTag" value="true"/>
@@ -233,23 +221,14 @@
       <module name="MissingJavadocMethod">
          <property name="scope" value="private"/>
       </module>
+      <module name="MissingJavadocType">
+         <property name="scope" value="private"/>
+      </module>
       <module name="MethodName">
          <property name="format" value="^[a-z][a-zA-Z0-9_]*$"/>
          <message key="name.invalidPattern"
           value="Method name ''{0}'' must match pattern ''{1}''."/>
       </module>
-      <module name="SingleLineJavadoc">
-         <property name="ignoreInlineTags" value="false"/>
-      </module>
-
-      <module name="MagicNumber">
-         <property name="severity" value="info"/>
-         <property name="tokens" value="NUM_DOUBLE, NUM_FLOAT, NUM_INT"/>
-         <property name="ignoreNumbers" value="0, 1, 2, 3, 4, 5, 6, 7, 8"/>
-         <property name="ignoreFieldDeclaration" value="true"/>
-         <property name="ignoreAnnotation" value="true"/>
-      </module>
-
       <module name="EmptyCatchBlock">
          <property name="exceptionVariableName" value="expected"/>
       </module>

docs/index.adoc

@@ -26,6 +26,16 @@ include::metaData/Reports.adoc[leveloffset=+1]
 include::metaData/Icons.adoc[leveloffset=+1]
 include::metaData/PermissionRules.adoc[leveloffset=+1]
 
+== Services
+
+include::misc/ScheduledJobs.adoc[leveloffset=+1]
+
+=== Web server (Javalin)
+#todo#
+
+=== API server (OpenAPI)
+#todo#
+
 == Custom Application Code
 include::misc/QContext.adoc[leveloffset=+1]
 include::misc/QRecords.adoc[leveloffset=+1]
@@ -63,4 +73,4 @@ include::implementations/TableSync.adoc[leveloffset=+1]
 // later... include::actions/RenderTemplateAction.adoc[leveloffset=+1]
 
 == QQQ Utility Classes
-include::utilities/RecordLookupHelper.adoc[leveloffset=+1]
+include::utilities/RecordLookupHelper.adoc[leveloffset=+1]

docs/metaData/Fields.adoc

@@ -21,5 +21,115 @@ Used to set values in the `displayValues` map within a `QRecord`.
 * `possibleValueSourceName` - *String* - Reference to a {link-pvs} to be used for this field.
 Values in this field should correspond to ids from the referenced Possible Value Source.
 * `maxLength` - *Integer* - Maximum length (number of characters) allowed for values in this field.
-Only applicable for fields with `type=STRING`.
-* `
+Only applicable for fields with `type=STRING`.  Needs to be used with a `FieldBehavior` of type `ValueTooLongBehavior`.
+
+==== Field Behaviors
+Additional behaviors can be attached to fields through the use of the `behaviors` attribute,
+which is a `Set` of 0 or more instances of implementations of the `FieldBehavior` interface.
+Note that in some cases, these instances may be `enum` constants,
+but other times may be regular Objects.
+
+QQQ provides a set of common field behaviors.
+Applications can also define their own field behaviors by implementing the `FieldBehavior` interface,
+and attaching instances of their custom behavior classes to fields.
+
+===== ValueTooLongBehavior
+Used on String fields.  Requires the field to have a `maxLength` set.
+Depending on the chosen instance of this enum, before a record is Inserted or Updated,
+if the value in the field is longer than the `maxLength`, then one of the following actions can occur:
+
+* `TRUNCATE` - The value will be simply truncated to the `maxLength`.
+* `TRUNCATE_ELLIPSIS` - The value will be truncated to 3 characters less than the `maxLength`, and three periods (an ellipsis) will be placed at the end.
+* `ERROR` - An error will be reported, and the record will not be inserted or updated.
+* `PASS_THROUGH` - Nothing will happen.  This is the same as not having a `ValueTooLongBehavior` on the field.
+
+[source,java]
+.Examples of using ValueTooLongBehavior
+----
+   new QFieldMetaData("sku", QFieldType.STRING)
+      .withMaxLength(40),
+      .withBehavior(ValueTooLongBehavior.ERROR),
+
+   new QFieldMetaData("reason", QFieldType.STRING)
+      .withMaxLength(250),
+      .withBehavior(ValueTooLongBehavior.TRUNCATE_ELLIPSIS),
+
+----
+
+===== DynamicDefaultValueBehavior
+Used to set a dynamic default value to a field when it is being inserted or updated.
+For example, instead of having a hard-coded `defaultValue` specified in the field meta-data,
+and instead of having to add, for example, a pre-insert custom action.
+
+* `CREATE_DATE` - On inserts, sets the field's value to the current time.
+* `MODIFY_DATE` - On inserts and updates, sets the field's value to the current time.
+* `USER_ID` - On inserts and updates, sets the field's value to the current user's id (but only if the value is currently null).
+
+_Note that the `QInstanceEnricher` will, by default, add the `CREATE_DATE` and `MODIFY_DATE` `DynamicDefaultValueBehavior`
+options to any fields named `"createDate"` or `"modifyDate"`.
+This behavior can be disabled by setting the `configAddDynamicDefaultValuesToFieldsNamedCreateDateAndModifyDate` property
+on the `QInstanceEnricher` instance used by the application to `false`._
+
+[source,java]
+.Examples of using DynamicDefaultValueBehavior
+----
+   new QFieldMetaData("createDate", QFieldType.DATE_TIME)
+      .withBehavior(DynamicDefaultValueBehavior.CREATE_DATE),
+
+   new QFieldMetaData("modifyDate", QFieldType.DATE_TIME)
+      .withBehavior(DynamicDefaultValueBehavior.MODIFY_DATE),
+
+   new QFieldMetaData("createdByUserId", QFieldType.STRING)
+      .withBehavior(DynamicDefaultValueBehavior.USER_ID),
+----
+
+===== DateTimeDisplayValueBehavior
+By default, in QQQ, fields of type `DATE_TIME` are stored in UTC,
+and their values in a QRecord is a java `Instant` instance, which is always UTC.
+However, frontends will prefer to display date-time values in the user's local Time Zone whenever possible.
+
+Using `DateTimeDisplayValueBehavior` allows a `DATE_TIME` field to be displayed in a different Time Zone.
+An example use-case for this would be displaying airplane flight times,
+where you would want a flight from California to New York to display Pacific Time for its departure time,
+and Eastern Time for its arrival.
+
+An instance of `DateTimeDisplayValueBehavior` can be configured to either use a hard-coded time `ZoneId`
+(for example, to always show users UTC, or a business's home-office time zone).
+Or, it can be set up to get the time zone to use from another field in the table.
+
+[source,java]
+.Examples of using DateTimeDisplayValueBehavior
+----
+new QTableMetaData().withName("flights").withFields(List.of(
+   ...
+   new QFieldMetaData("departureTimeZoneId", QFieldType.STRING),
+   new QFieldMetaData("arrivaTimeZoneId", QFieldType.STRING),
+
+   new QFieldMetaData("departureTime", QFieldType.DATE_TIME)
+      .withBehavior(new DateTimeDisplayValueBehavior()
+         .withZoneIdFromFieldName("departureTimeZoneId")),
+
+   new QFieldMetaData("arrivalTime", QFieldType.DATE_TIME)
+      .withBehavior(new DateTimeDisplayValueBehavior()
+         .withZoneIdFromFieldName("arrivalTimeZoneId"))
+
+   new QFieldMetaData("ticketSaleStartDateTime", QFieldType.DATE_TIME)
+      .withBehavior(new DateTimeDisplayValueBehavior()
+         .withDefaultZoneId("UTC"))
+----
+
+===== CaseChangeBehavior
+A field can be made to always go through a toUpperCase or toLowerCase transformation, both before it is stored in a backend,
+and after it is read from a backend, by adding a CaseChangeBehavior to it:
+
+[source,java]
+.Examples of using CaseChangeBehavior
+----
+new QTableMetaData().withName("item").withFields(List.of(
+
+   new QFieldMetaData("sku", QFieldType.STRING)
+      .withBehavior(CaseChangeBehavior.TO_UPPER_CASE)),
+
+   new QFieldMetaData("username", QFieldType.STRING)
+      .withBehavior(CaseChangeBehavior.TO_LOWER_CASE)),
+----

docs/metaData/Joins.adoc

@@ -2,16 +2,57 @@
 == Joins
 include::../variables.adoc[]
 
-#TODO#
+A `QJoinMetaData` is a meta-data object that tells QQQ, essentially “it is possible for these 2 tables to join, here’s how to do it”.
+
+Joins can be used then, in an application, in a number of possible ways:
+
+* In a {link-table}, we can specify joins to be “exposed”, e.g., made available to users on a query screen
+* Also in a Table, as part of an “Association”, which sets up one table as a “parent” of another,
+such that you can store (and fetch) the child-records at the same time as the parent
+** A common use-case here may be an order & lineItem table -
+such that QQQ can generate an API uses to allow you to post an order and its lines in a single request, and they get stored all together
+* In defining the security field (record lock) on a table,
+sometimes, it isn’t a field directly on the table, but instead comes from a joined table (possibly even more than just 1 table away).
+** For example, maybe a lineItem table, doesn't have a clientId, but needs secured by that field
+- so its recordLock can specify a “joinNameChain” that describes how to get from lineItem to order.clientId
+* The `QueryAction` can take (through its QueryInput object) zero or more QueryJoin objects,
+which must make a reference (implicitly or explicitly) to a QJoinMetaData.
+See the section on <<QueryJoin,QueryJoins>> for more details.
 
 === QJoinMetaData
 Joins are defined in a QQQ Instance in a `*QJoinMetaData*` object.
 
-#TODO#
+In this object, we have the concept of a "leftTable" and a "rightTable".
+There isn't generally anything special about which table is on the "left" and which is on the "right".
+But the remaining pieces of the QJoinMetaData do all need to line-up with these sides.
+
+For example:
+
+* The Type (one-to-one, one-to-many, many-to-one) - where the leftTable comes first, and rightTable comes second
+(e.g., a one-to-many means 1-row in leftTable has many-rows in rightTable associated with it)
+* In a JoinOn object, the 1st field name given is from the leftTable;
+the second fieldName from the rightTable.
 
 *QJoinMetaData Properties:*
 
-* `name` - *String, Required* - Unique name for the join within the QQQ Instance. #todo infererences or conventions?#
-
-#TODO#
+* `name` - *String, Required* - Unique name for the join within the QQQ Instance.
+** One convention is to name joins based on (leftTable + "Join" + rightTable).
+** If you do not wish to define join names yourself, the method `withInferredName()`
+can be called (which defers to
+`public static String makeInferredJoinName(String leftTable, String rightTable)`),
+to create a name for the join following the (leftTable + "Join" + rightTable) convention.
+* `leftTable` - *String, Required* - Name of a {link-table} in the {link-instance}.
+* `rightTable` - *String, Required* - Name of a {link-table} in the {link-instance}.
+* `type` - *enum, Required* - cardinality between the two tables in the join.
+** e.g., `ONE_TO_ONE`, `ONE_TO_MANY` (indicating 1 record in the left table may join
+to many records in the right table), or `MANY_TO_ONE` (vice-versa).
+** Note that there is no MANY_TO_MANY option, as a many-to-many is built as multiple QJoinMetaData's
+going through the intermediary (intersection) table.
+* `joinOns` - *List<JoinOn>, Required* - fields used to join the tables.
+Note: In the 2-arg JoinOn constructor, the leftTable's field comes first.
+Alternatively, the no-arg constructor can be used along with `.withLeftField().withRightField()`
+* `orderBys` - *List<QFilterOrderBy>* - Optional list of order-by objects,
+used in some framework-generated queries using the join.
+The field names are assumed to come from the rightTable.
 
+#TODO# what else do we need here?

docs/metaData/Processes.adoc

@@ -16,6 +16,7 @@ Processes are defined in a QQQ Instance in a `*QProcessMetaData*` object.
 In addition to directly building a `QProcessMetaData` object setting its properties directly, there are a few common process patterns that provide *Builder* objects for ease-of-use.
 See StreamedETLWithFrontendProcess below for a common example
 
+[#_QProcessMetaData_Properties]
 *QProcessMetaData Properties:*
 
 * `name` - *String, Required* - Unique name for the process within the QQQ Instance.
@@ -30,12 +31,13 @@ See below for details.
 * `permissionRules` - *QPermissionRules object* - define the permission/access rules for the process.
 See {link-permissionRules} for details.
 * `steps` and `stepList` - *Map of String → <<QStepMetaData>>* and *List of QStepMetaData* - Defines the <<QFrontendStepMetaData,screens>> and <<QBackendStepMetaData,backend code>> that makes up the process.
-** `stepList` is the list of steps in the order that they will by default be executed.
-** `steps` is a map, including all steps from `stepList`, but which may also include steps which can used by the process if its backend steps make the decision to do so, at run-time.
+** `stepList` is the list of steps in the order that they will be executed
+(that is to say - this is the _default_ order of execution - but it can be customized - see <<_custom_process_flow>> for details).
+** `steps` is a map, including all steps from `stepList`, but which may also include steps which can used by the process if its backend steps make the decision to do so, at run-time (e.g., using <<_custom_process_flow>>).
 ** A process's steps are normally defined in one of two was:
 *** 1) by a single call to `.withStepList(List<QStepMetaData>)`, which internally adds each step into the `steps` map.
 *** 2) by multiple calls to `.addStep(QStepMetaData)`, which adds a step to both the `stepList` and `steps` map.
-** If a process also needs optional steps, they should be added by a call to `.addOptionalStep(QStepMetaData)`, which only places them in the `steps` map.
+** If a process also needs optional steps (for a <<_custom_process_flow>>), they should be added by a call to `.addOptionalStep(QStepMetaData)`, which only places them in the `steps` map.
 * `schedule` - *<<QScheduleMetaData>>* - set up the process to run automatically on the specified schedule.
 See below for details.
 * `minInputRecords` - *Integer* - #not used...#
@@ -188,8 +190,8 @@ To improve this programmer-interface, an inner `Builder` class exists within `St
 
 *StreamedETLWithFrontendProcess.Builder methods:*
 
-* `withName(String name) - Set the name for the process.
-* `withLabel(String label) - Set the label for the process.
+* `withName(String name)` - Set the name for the process.
+* `withLabel(String label)` - Set the label for the process.
 * `withIcon(QIcon icon)` - Set an {link-icon} to be display with the process in the UI.
 * `withExtractStepClass(Class<? extends AbstractExtractStep>)` - Define the Extract step for the process.
 If no special extraction logic is needed, `ExtractViaQuery.class` is often a reasonable default.
@@ -214,3 +216,112 @@ But for some cases, doing page-level transactions can reduce long-transactions a
 * `withFields(List<QFieldMetaData> fieldList)` - Adds additional input fields to the preview step of the process.
 * `withBasepullConfiguration(BasepullConfiguration basepullConfiguration)` - Add a <<BasepullConfiguration>> to the process.
 * `withSchedule(QScheduleMetaData schedule)` - Add a <<QScheduleMetaData>> to the process.
+
+[#_custom_process_flow]
+==== Custom Process Flow
+As referenced in the definition of the <<_QProcessMetaData_Properties,QProcessMetaData Properties>>, by default, a process
+will execute each of its steps in-order, as defined in the `stepList` property.
+However, a Backend Step can customize this flow #todo - write more clearly here...
+
+There are generally 2 method to call (in a `BackendStep`) to do a dynamic flow:
+
+* `RunBackendStepOutput.setOverrideLastStepName(String stepName)`
+** QQQ's `RunProcessAction` keeps track of which step it "last" ran, e.g., to tell it which one to run next.
+However, if a step sets the `OverrideLastStepName` property in its output object,
+then the step named in that property becomes the effective "last" step,
+thus determining which step comes next.
+
+* `RunBackendStepOutput.updateStepList(List<String> stepNameList)`
+** Calling this method changes the process's runtime definition of steps to be executed.
+Thus allowing a completely custom flow.
+It should be noted, that the "last" step name (as tracked by QQQ within `RunProcessAction`)
+does need to be found in the new `stepNameList` - otherwise, the framework will not know where you were,
+for figuring out where to go next.
+
+[source,java]
+.Example of a defining process that can use a flexible flow:
+----
+// for a case like this, it would be recommended to define all step names in constants:
+public final static String STEP_START = "start";
+public final static String STEP_A     = "a";
+public final static String STEP_B     = "b";
+public final static String STEP_C     = "c";
+public final static String STEP_1     = "1";
+public final static String STEP_2     = "2";
+public final static String STEP_3     = "3";
+public final static String STEP_END   = "end";
+
+// also, to define the possible flows (lists of steps) in constants as well:
+public final static List<String> LETTERS_STEP_LIST = List.of(
+  STEP_START, STEP_A, STEP_B, STEP_C, STEP_END);
+
+public final static List<String> NUMBERS_STEP_LIST = List.of(
+  STEP_START, STEP_1, STEP_2, STEP_3, STEP_END);
+
+// when we define the process's meta-data, we only give a "skeleton" stepList -
+// we must at least have our starting step, and we may want at least one frontend step
+// for the UI to show some placeholder(s):
+QProcessMetaData process = new QProcessMetaData()
+   .withName(PROCESS_NAME)
+   .withStepList(List.of(
+      new QBackendStepMetaData().withName(STEP_START)
+         .withCode(new QCodeReference(/*...*/)),
+      new QFrontendStepMetaData()
+         .withName(STEP_END)
+   ));
+
+// the additional steps get added via `addOptionalStep`, which only puts them in
+// the process's stepMap, not its stepList!
+process.addOptionalStep(new QFrontendStepMetaData().withName(STEP_A));
+process.addOptionalStep(new QBackendStepMetaData().withName(STEP_B)
+   .withCode(new QCodeReference(/*...*/)));
+process.addOptionalStep(new QFrontendStepMetaData().withName(STEP_C));
+
+process.addOptionalStep(new QBackendStepMetaData().withName(STEP_1)
+   .withCode(new QCodeReference(/*...*/)));
+process.addOptionalStep(new QFrontendStepMetaData().withName(STEP_2));
+process.addOptionalStep(new QBackendStepMetaData().withName(STEP_3)
+   .withCode(new QCodeReference(/*...*/)));
+
+----
+
+[source,java]
+.Example of a process backend step adjusting the process's runtime flow:
+----
+/***************************************************************************
+** look at the value named "which". if it's "letters", then make the process
+** go through the stepList consisting of letters; else, update the step list
+** to be the "numbers" steps.
+**
+** Also - if the "skipSomeSteps" value is give as true, then set the
+** overrideLastStepName to skip again (in the letters case, skip past A, B
+** and C; in the numbers case, skip past 1 and 2).
+***************************************************************************/
+public static class StartStep implements BackendStep
+{
+  @Override
+  public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException
+  {
+     Boolean skipSomeSteps = runBackendStepInput.getValueBoolean("skipSomeSteps");
+
+     if(runBackendStepInput.getValueString("which").equals("letters"))
+     {
+        runBackendStepOutput.updateStepList(LETTERS_STEP_LIST);
+        if(BooleanUtils.isTrue(skipSomeSteps))
+        {
+           runBackendStepOutput.setOverrideLastStepName(STEP_C);
+        }
+     }
+     else
+     {
+        runBackendStepOutput.updateStepList(NUMBERS_STEP_LIST);
+        if(BooleanUtils.isTrue(skipSomeSteps))
+        {
+           runBackendStepOutput.setOverrideLastStepName(STEP_2);
+        }
+     }
+  }
+}
+----
+
+

docs/misc/ProcessBackendSteps.adoc

@@ -128,15 +128,15 @@ These steps are:
 ** The Extract step is called before the Preview, Validate, and Result screens, though for the Preview screen, it is set to only extract a small number of records (10).
 * *Transform* - a subclass of `AbstractTransformStep` - is responsible for applying the majority of the business logic of the process.
 In ETL terminology, this is the "Transform" action - which means applying some type of logical transformation an input record (found by the Extract step) to generate an output record (stored by the Load step).
-** A Transform step's `run` method will be called, potentially, multiple times, each time with a page of records in the `runBackendStepInput` parameter.
+** A Transform step's `runOnePage` method will be called, potentially, multiple times, each time with a page of records in the `runBackendStepInput` parameter.
 ** This method is responsible for adding records to the `runBackendStepOutput`, which will then be passed to the *Load* step.
 ** This class is also responsible for implementing the method `getProcessSummary`, which provides the data to the *Validate* screen.
-** The run method will generally update ProcessSummaryLine objects to facilitate this functionality.
+** The `runOnePage` method will generally update ProcessSummaryLine objects to facilitate this functionality.
 ** The Transform step is called before the Preview, Validate, and Result screens, consuming all records selected by the Extract step.
 * *Load* - a subclass of `AbstractLoadStep` - is responsible for the Load function of the ETL job.
 _A quick word on terminology - this step is actually doing what we are more likely to think of as storing data - which feels like the opposite of “loading” - but we use the name Load to keep in line with the ETL naming convention…_
 ** The Load step is ONLY called before the Result screen is presented (possibly after Preview, if the user chose to skip validation, otherwise, after validation).
-** Similar to the Transform step, the Load step's `run` method will be called potentially multiple times, with pages of records in its input.
+** Similar to the Transform step, the Load step's `runOnePage` method will be called potentially multiple times, with pages of records in its input.
 ** As such, the Load step is generally the only step where data writes should occur.
 *** e.g., a Transform step should not do any writes, as it will be called when the user is going to the Preview & Validate screens - e.g., before the user confirmed that they want to execute the action!
 ** A common pattern is that the Load step just needs to insert or update the list of records output by the Transform step, in which case the QQQ-provided `LoadViaInsertStep` or `LoadViaUpdateStep` can be used, but custom use-cases can be built as well.

docs/misc/ScheduledJobs.adoc

@@ -0,0 +1,141 @@
+== Schedulers and Scheduled Jobs
+include::../variables.adoc[]
+
+QQQ has the ability to automatically run various types of jobs on schedules,
+either defined in your instance's meta-data,
+or optionally via data in your application, in a `scheduledJob` table.
+
+=== Schedulers and QSchedulerMetaData
+2 types of schedulers are included in QQQ by default (though an application can define its own schedulers):
+
+* `SimpleScheduler` - is (as its name suggests) a simple class which uses java's `ScheduledExecutorService`
+to run jobs on repeating intervals.
+** Cannot run cron schedules - only repeating intervals.
+** If multiple servers are running, each will potentially run the same job concurrently
+** Has no configurations, e.g., to limit the number of threads.
+
+* `QuartzScheduler` - uses the 3rd party https://www.quartz-scheduler.org/[Quartz Scheduler] library to provide
+a much more capable, though admittedly more complex, scheduling solution.
+** Can run both cron schedules and repeating intervals.
+** By default, will not allow concurrent executions of the same job.
+** Supports multiple configurations, e.g., to limit the number of threads.
+
+An application can define its own scheduler by providing a class which implements the `QSchedulerInterface`.
+
+A `QInstance` can work with 0 or more schedulers, as defined by adding `QSchedulerMetaData` objects
+to the instance.
+
+This meta-data class is `abstract`, and is extended by the 2 built-in schedulers
+(e.g., `SimpleSchedulerMetaData` and `QuartzSchedulerMetaData`).  As such,
+these concrete subclasses are what you need to instantiate and add to your instance.
+
+To configure a QuartzScheduler, you can add a `Properties` object to the `QuartzSchedulerMetaData` object.
+See https://www.quartz-scheduler.org/documentation/[Quartz's documentation] for available configuration properties.
+
+[source,java]
+.Defining SchedulerMetaData
+----
+qInstance.addScheduler(new SimpleSchedulerMetaData().withName("mySimpleScheduler"));
+
+qInstance.addScheduler(new QuartzSchedulerMetaData()
+   .withName("myQuartzScheduler")
+   .withProperties(myQuartzProperties);
+----
+
+=== SchedulableTypes
+The types of jobs which can be scheduled in a QQQ application are defined in the `QInstance` by
+instances of the `SchedulableType` meta-data class.
+These objects contain a name, along with a `QCodeReference` to the `runner`,
+which must be a class that implements the `SchedulableRunner` interface.
+
+By default, (in the `QInstanceEnricher`), QQQ will make 3 `SchedulableType` options available:
+
+* `PROCESS` - Any Process defined in the `QInstance` can be scheduled.
+* `QUEUE_PROCESSOR` - A Queue defined in the `QInstance`, which requires polling (e.g., SQS), can be scheduled.
+* `TABLE_AUTOMATIONS` - A Table in the `QInstance`, with `AutomationDetails` referring to an
+AutomationProvider which requires polling, can be scheduled.
+
+If an application only wants to use a subset of these `SchedulableType` options,
+or to add custom `SchedulableType` options,
+the `QInstance` will need to have 1 or more `SchedulableType` objects in it before the `QInstanceEnricher` runs.
+
+=== User-defined Scheduled Jobs
+To allow users to schedule jobs (rather than using programmer-defined schedules (in meta-data)),
+you can add a set of tables to your `QInstance`, using the `ScheduledJobsMetaDataProvider` class:
+
+[source,java]
+.Adding the ScheduledJob tables and related meta-data to a QInstance
+----
+new ScheduledJobsMetaDataProvider().defineAll(
+   qInstance, backendName, table -> tableEnricher(table));
+----
+
+This meta-data provider adds a "scheduledJob" and "scheduledJobParameter" table, along with
+some PossibleValueSources.
+These tables include post-action customizers, which manage (re-, un-) scheduling jobs based on
+changes made to records in this these tables.
+
+Also, when `QScheduleManager` is started, it will query these tables,and will schedule jobs as defined therein.
+
+_You can use a mix of user-defined and meta-data defined scheduled jobs in your instance.
+However, if a ScheduledJob record references a process, queue, or table automation with a
+meta-data defined schedule, the ScheduledJob will NOT be started by ScheduleManager --
+rather, the meta-data definition will "win"._
+
+[source,sql]
+.Example of inserting scheduled jobs records directly into an SQL backend
+----
+-- A process:
+INSERT INTO scheduled_job (label, scheduler_name, cron_expression, cron_time_zone_id, repeat_seconds, type, is_active) VALUES
+   ('myProcess', 'QuartzScheduler', null, null, 300, 'PROCESS', 1);
+INSERT INTO scheduled_job_parameter (scheduled_job_id, `key`, value) VALUES
+   ((SELECT id FROM scheduled_job WHERE label = 'myProcess'), 'processName', 'myProcess');
+
+-- A table's insert & update automations:
+INSERT INTO scheduled_job (label, scheduler_name, cron_expression, cron_time_zone_id, repeat_seconds, type, is_active) VALUES
+   ('myTable.PENDING_INSERT_AUTOMATIONS', 'QuartzScheduler', null, null, 15, 'TABLE_AUTOMATIONS', 1),
+   ('myTable.PENDING_UPDATE_AUTOMATIONS', 'QuartzScheduler', null, null, 15, 'TABLE_AUTOMATIONS', 1);
+INSERT INTO scheduled_job_parameter (scheduled_job_id, `key`, value) VALUES
+   ((SELECT id FROM scheduled_job WHERE label = 'myTable.PENDING_INSERT_AUTOMATIONS'), 'tableName', 'myTable'),
+   ((SELECT id FROM scheduled_job WHERE label = 'myTable.PENDING_INSERT_AUTOMATIONS'), 'automationStatus', 'PENDING_INSERT_AUTOMATIONS'),
+   ((SELECT id FROM scheduled_job WHERE label = 'myTable.PENDING_UPDATE_AUTOMATIONS'), 'tableName', 'myTable'),
+   ((SELECT id FROM scheduled_job WHERE label = 'myTable.PENDING_UPDATE_AUTOMATIONS'), 'automationStatus', 'PENDING_UPDATE_AUTOMATIONS');
+
+-- A queue processor:
+INSERT INTO scheduled_job (label, scheduler_name, cron_expression, cron_time_zone_id, repeat_seconds, type, is_active) VALUES
+   ('mySqsQueue', 'QuartzScheduler', null, null, 60, 'QUEUE_PROCESSOR', 1);
+INSERT INTO scheduled_job_parameter (scheduled_job_id, `key`, value) VALUES
+   ((SELECT id FROM scheduled_job WHERE label = 'mySqsQueue'), 'queueName', 'mySqsQueue');
+----
+
+=== Running Scheduled Jobs
+In a server running QQQ, if you wish to start running scheduled jobs, you need to initialize
+the `QScheduleManger` singleton class, then call its `start()` method.
+
+Note that internally, this class will check for a system property of `qqq.scheduleManager.enabled`
+or an environment variable of `QQQ_SCHEDULE_MANAGER_ENABLED`, and if the first value found is
+`"false"`, then the scheduler will not actually run its jobs (but, in the case of the `QuartzSchdeuler`,
+it will be available for managing scheduled jobs).
+
+The method `QScheduleManager.initInstance` requires 2 parameters:  Your `QInstance`, and a
+`Supplier<QSession>` lambda, which returns the session that will be used for scheduled jobs when they
+are executed.
+
+[source,java]
+.Starting the Schedule Manager service
+----
+QScheduleManager.initInstance(qInstance, () -> systemUserSession).start();
+----
+
+=== Examples
+[source,java]
+.Attach a schedule in meta-data to a Process
+----
+QProcessMetaData myProcess = new QProcessMetaData()
+   // ...
+   .withSchedule(new QScheduleMetaData()
+      .withSchedulerName("myScheduler")
+      .withDescription("Run myProcess every five minutes")
+      .withRepeatSeconds(300))
+----
+

pom.xml

@@ -29,6 +29,7 @@
    <packaging>pom</packaging>
 
    <modules>
+      <module>qqq-bom</module>
       <module>qqq-backend-core</module>
       <module>qqq-backend-module-api</module>
       <module>qqq-backend-module-filesystem</module>
@@ -45,12 +46,11 @@
    </modules>
 
    <properties>
-      <revision>0.20.0-SNAPSHOT</revision>
+      <revision>0.21.0-SNAPSHOT</revision>
 
       <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
       <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
-      <maven.compiler.source>17</maven.compiler.source>
-      <maven.compiler.target>17</maven.compiler.target>
+      <maven.compiler.release>17</maven.compiler.release>
       <maven.compiler.showDeprecation>true</maven.compiler.showDeprecation>
       <maven.compiler.showWarnings>true</maven.compiler.showWarnings>
       <coverage.haltOnFailure>true</coverage.haltOnFailure>
@@ -149,7 +149,7 @@
                <dependency>
                   <groupId>com.puppycrawl.tools</groupId>
                   <artifactId>checkstyle</artifactId>
-                  <version>9.0</version>
+                  <version>10.16.0</version>
                </dependency>
             </dependencies>
             <executions>
@@ -167,6 +167,7 @@
                      <violationSeverity>warning</violationSeverity>
                      <excludes>**/target/generated-sources/*.*</excludes>
                      <!-- <linkXRef>false</linkXRef> -->
+                     <includeTestSourceDirectory>true</includeTestSourceDirectory>
                   </configuration>
                   <goals>
                      <goal>check</goal>
@@ -208,6 +209,7 @@
                   <productionBranch>main</productionBranch>
                   <developmentBranch>dev</developmentBranch>
                   <versionTagPrefix>version-</versionTagPrefix>
+                  <releaseBranchPrefix>rel/</releaseBranchPrefix>
                </gitFlowConfig>
                <skipFeatureVersion>true</skipFeatureVersion> <!-- Keep feature names out of versions -->
                <postReleaseGoals>install</postReleaseGoals>  <!-- Let CI run deploys -->
@@ -245,8 +247,7 @@ echo "   See also target/site/jacoco/index.html"
 echo "   and https://www.jacoco.org/jacoco/trunk/doc/counters.html"
 echo "------------------------------------------------------------"
 
-which xpath > /dev/null 2>&1
-if [ "$?" == "0" ]; then
+if which xpath > /dev/null 2>&1; then
    echo "Element\nInstructions Missed\nInstruction Coverage\nBranches Missed\nBranch Coverage\nComplexity Missed\nComplexity Hit\nLines Missed\nLines Hit\nMethods Missed\nMethods Hit\nClasses Missed\nClasses Hit\n" > /tmp/$$.headers
    xpath -n -q -e '/html/body/table/tfoot/tr[1]/td/text()' target/site/jacoco/index.html > /tmp/$$.values
    paste /tmp/$$.headers /tmp/$$.values | tail +2 | awk -v FS='\t' '{printf("%-20s %s\n",$1,$2)}'
@@ -255,8 +256,7 @@ else
    echo "xpath is not installed.  Jacoco coverage summary will not be produced here...";
 fi
 
-which xpath > /dev/null 2>&1
-if [ "$?" == "0" ]; then
+if which html2text > /dev/null 2>&1; then
    echo "Untested classes, per Jacoco:"
    echo "-----------------------------"
    for i in target/site/jacoco/*/index.html; do

qodana.yaml

@@ -0,0 +1,31 @@
+#-------------------------------------------------------------------------------#
+#               Qodana analysis is configured by qodana.yaml file               #
+#             https://www.jetbrains.com/help/qodana/qodana-yaml.html            #
+#-------------------------------------------------------------------------------#
+version: "1.0"
+
+#Specify inspection profile for code analysis
+profile:
+  name: qodana.starter
+
+#Enable inspections
+#include:
+#  - name: <SomeEnabledInspectionId>
+
+#Disable inspections
+#exclude:
+#  - name: <SomeDisabledInspectionId>
+#    paths:
+#      - <path/where/not/run/inspection>
+
+projectJDK: 17 #(Applied in CI/CD pipeline)
+
+#Execute shell command before Qodana execution (Applied in CI/CD pipeline)
+#bootstrap: sh ./prepare-qodana.sh
+
+#Install IDE plugins before Qodana execution (Applied in CI/CD pipeline)
+#plugins:
+#  - id: <plugin.id> #(plugin id can be found at https://plugins.jetbrains.com)
+
+#Specify Qodana linter for analysis (Applied in CI/CD pipeline)
+linter: jetbrains/qodana-jvm:latest

qqq-backend-core/pom.xml

@@ -102,6 +102,16 @@
          <artifactId>fastexcel</artifactId>
          <version>0.12.15</version>
       </dependency>
+      <dependency>
+         <groupId>org.apache.poi</groupId>
+         <artifactId>poi</artifactId>
+         <version>5.2.5</version>
+      </dependency>
+      <dependency>
+         <groupId>org.apache.poi</groupId>
+         <artifactId>poi-ooxml</artifactId>
+         <version>5.2.5</version>
+      </dependency>
       <dependency>
          <groupId>com.auth0</groupId>
          <artifactId>auth0</artifactId>
@@ -163,6 +173,45 @@
          <version>1.12.321</version>
       </dependency>
 
+      <dependency>
+         <groupId>com.amazonaws</groupId>
+         <artifactId>aws-java-sdk-ses</artifactId>
+         <version>1.12.705</version>
+      </dependency>
+
+      <dependency>
+         <groupId>cloud.localstack</groupId>
+         <artifactId>localstack-utils</artifactId>
+         <version>0.2.20</version>
+         <scope>test</scope>
+      </dependency>
+
+      <dependency>
+         <groupId>org.quartz-scheduler</groupId>
+         <artifactId>quartz</artifactId>
+         <version>2.3.2</version>
+      </dependency>
+
+      <!-- bring in a newer version of this lib, which quartz transitively loads through c3p0 -->
+      <dependency>
+         <groupId>com.mchange</groupId>
+         <artifactId>mchange-commons-java</artifactId>
+         <version>0.3.0</version>
+      </dependency>
+
+      <!-- Many of the deps we bring in use slf4j.  This dep maps slf4j to our logger, log4j -->
+      <dependency>
+         <groupId>org.apache.logging.log4j</groupId>
+         <artifactId>log4j-slf4j-impl</artifactId>
+         <version>2.23.0</version>
+      </dependency>
+
+      <dependency>
+         <groupId>com.sun.mail</groupId>
+         <artifactId>jakarta.mail</artifactId>
+         <version>2.0.1</version>
+      </dependency>
+
       <!-- Common deps for all qqq modules -->
       <dependency>
          <groupId>org.apache.maven.plugins</groupId>

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/ActionHelper.java

@@ -37,6 +37,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
 import com.kingsrook.qqq.backend.core.model.session.QSession;
 import com.kingsrook.qqq.backend.core.modules.authentication.QAuthenticationModuleDispatcher;
 import com.kingsrook.qqq.backend.core.modules.authentication.QAuthenticationModuleInterface;
+import com.kingsrook.qqq.backend.core.utils.PrefixedDefaultThreadFactory;
 
 
 /*******************************************************************************
@@ -55,7 +56,7 @@ public class ActionHelper
    /////////////////////////////////////////////////////////////////////////////
    private static Integer         CORE_THREADS    = 8;
    private static Integer         MAX_THREADS     = 500;
-   private static ExecutorService executorService = new ThreadPoolExecutor(CORE_THREADS, MAX_THREADS, 60L, TimeUnit.SECONDS, new SynchronousQueue<>());
+   private static ExecutorService executorService = new ThreadPoolExecutor(CORE_THREADS, MAX_THREADS, 60L, TimeUnit.SECONDS, new SynchronousQueue<>(), new PrefixedDefaultThreadFactory(ActionHelper.class));
 
 
 

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/QBackendTransaction.java

@@ -37,7 +37,7 @@ import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleInterface;
  **
  ** Note:  One would imagine that this class shouldn't ever implement Serializable...
  *******************************************************************************/
-public class QBackendTransaction
+public class QBackendTransaction implements AutoCloseable
 {
 
    /*******************************************************************************

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/async/AsyncJobManager.java

@@ -42,6 +42,7 @@ import com.kingsrook.qqq.backend.core.state.InMemoryStateProvider;
 import com.kingsrook.qqq.backend.core.state.StateProviderInterface;
 import com.kingsrook.qqq.backend.core.state.StateType;
 import com.kingsrook.qqq.backend.core.state.UUIDAndTypeStateKey;
+import com.kingsrook.qqq.backend.core.utils.PrefixedDefaultThreadFactory;
 import com.kingsrook.qqq.backend.core.utils.StringUtils;
 import org.apache.logging.log4j.Level;
 import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
@@ -65,7 +66,7 @@ public class AsyncJobManager
    /////////////////////////////////////////////////////////////////////////////
    private static Integer         CORE_THREADS    = 8;
    private static Integer         MAX_THREADS     = 500;
-   private static ExecutorService executorService = new ThreadPoolExecutor(CORE_THREADS, MAX_THREADS, 60L, TimeUnit.SECONDS, new SynchronousQueue<>());
+   private static ExecutorService executorService = new ThreadPoolExecutor(CORE_THREADS, MAX_THREADS, 60L, TimeUnit.SECONDS, new SynchronousQueue<>(), new PrefixedDefaultThreadFactory(AsyncJobManager.class));
 
 
    private String forcedJobUUID = null;
@@ -159,7 +160,7 @@ public class AsyncJobManager
    private <T extends Serializable> T runAsyncJob(String jobName, AsyncJob<T> asyncJob, UUIDAndTypeStateKey uuidAndTypeStateKey, AsyncJobStatus asyncJobStatus)
    {
       String originalThreadName = Thread.currentThread().getName();
-      Thread.currentThread().setName("Job:" + jobName + ":" + uuidAndTypeStateKey.getUuid().toString().substring(0, 8));
+      Thread.currentThread().setName("Job:" + jobName);
       try
       {
          LOG.debug("Starting job " + uuidAndTypeStateKey.getUuid());
@@ -169,17 +170,24 @@ public class AsyncJobManager
          LOG.debug("Completed job " + uuidAndTypeStateKey.getUuid());
          return (result);
       }
-      catch(Exception e)
+      catch(Throwable t)
       {
          asyncJobStatus.setState(AsyncJobState.ERROR);
-         asyncJobStatus.setCaughtException(e);
+         if(t instanceof Exception e)
+         {
+            asyncJobStatus.setCaughtException(e);
+         }
+         else
+         {
+            asyncJobStatus.setCaughtException(new QException("Caught throwable", t));
+         }
          getStateProvider().put(uuidAndTypeStateKey, asyncJobStatus);
 
          //////////////////////////////////////////////////////
          // if user facing, just log an info, warn otherwise //
          //////////////////////////////////////////////////////
-         LOG.log((e instanceof QUserFacingException) ? Level.INFO : Level.WARN, "Job ended with an exception", e, logPair("jobId", uuidAndTypeStateKey.getUuid()));
-         throw (new CompletionException(e));
+         LOG.log((t instanceof QUserFacingException) ? Level.INFO : Level.WARN, "Job ended with an exception", t, logPair("jobId", uuidAndTypeStateKey.getUuid()));
+         throw (new CompletionException(t));
       }
       finally
       {

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/async/AsyncJobStatus.java

@@ -31,6 +31,7 @@ import java.io.Serializable;
  *******************************************************************************/
 public class AsyncJobStatus implements Serializable
 {
+   private String        jobName;
    private AsyncJobState state;
    private String        message;
    private Integer       current;
@@ -187,4 +188,36 @@ public class AsyncJobStatus implements Serializable
    {
       this.cancelRequested = cancelRequested;
    }
+
+
+
+   /*******************************************************************************
+    ** Getter for jobName
+    *******************************************************************************/
+   public String getJobName()
+   {
+      return (this.jobName);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for jobName
+    *******************************************************************************/
+   public void setJobName(String jobName)
+   {
+      this.jobName = jobName;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for jobName
+    *******************************************************************************/
+   public AsyncJobStatus withJobName(String jobName)
+   {
+      this.jobName = jobName;
+      return (this);
+   }
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/async/AsyncRecordPipeLoop.java

@@ -32,6 +32,7 @@ import com.kingsrook.qqq.backend.core.logging.QLogger;
 import com.kingsrook.qqq.backend.core.utils.SleepUtils;
 import com.kingsrook.qqq.backend.core.utils.lambdas.UnsafeFunction;
 import com.kingsrook.qqq.backend.core.utils.lambdas.UnsafeSupplier;
+import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
 
 
 /*******************************************************************************
@@ -185,8 +186,7 @@ public class AsyncRecordPipeLoop
 
       if(recordCount > 0)
       {
-         LOG.info(String.format("Processed %,d records", recordCount)
-            + String.format(" at end of job [%s] in %,d ms (%.2f records/second).", jobName, (endTime - jobStartTime), 1000d * (recordCount / (.001d + (endTime - jobStartTime)))));
+         LOG.info("End of job summary", logPair("recordCount", recordCount), logPair("jobName", jobName), logPair("millis", endTime - jobStartTime), logPair("recordsPerSecond", 1000d * (recordCount / (.001d + (endTime - jobStartTime)))));
       }
 
       return (recordCount);

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/async/NonPersistedAsyncJobCallback.java

@@ -0,0 +1,62 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.actions.async;
+
+
+import java.util.UUID;
+
+
+/*******************************************************************************
+ ** subclass designed to be used when we want there to be an instance (so code
+ ** doesn't have to all be null-tolerant), but there's no one who will ever be
+ ** reading the status data, so we don't need to store the object in a
+ ** state provider.
+ *******************************************************************************/
+public class NonPersistedAsyncJobCallback extends AsyncJobCallback
+{
+   private final AsyncJobStatus asyncJobStatus;
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public NonPersistedAsyncJobCallback(UUID jobUUID, AsyncJobStatus asyncJobStatus)
+   {
+      super(jobUUID, asyncJobStatus);
+      this.asyncJobStatus = asyncJobStatus;
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   protected void storeUpdatedStatus()
+   {
+      ///////////////////////////////////////////////////////////////////////////////////////
+      // noop - cf. base class, which writes to persistence here (our point is, we do not) //
+      ///////////////////////////////////////////////////////////////////////////////////////
+   }
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/audits/DMLAuditAction.java

@@ -149,7 +149,7 @@ public class DMLAuditAction extends AbstractQActionFunction<DMLAuditInput, DMLAu
             // sort the field names by their labels //
             //////////////////////////////////////////
             List<String> sortedFieldNames = table.getFields().keySet().stream()
-               .sorted(Comparator.comparing(fieldName -> table.getFields().get(fieldName).getLabel()))
+               .sorted(Comparator.comparing(fieldName -> Objects.requireNonNullElse(table.getFields().get(fieldName).getLabel(), fieldName)))
                .toList();
 
             QFieldMetaData primaryKeyField = table.getField(table.getPrimaryKeyField());
@@ -303,7 +303,7 @@ public class DMLAuditAction extends AbstractQActionFunction<DMLAuditInput, DMLAu
          }
          else
          {
-            String formattedValue = getFormattedValueForAuditDetail(record, fieldName, field, value);
+            String formattedValue = getFormattedValueForAuditDetail(table, record, fieldName, field, value);
             detailRecord = new QRecord().withValue("message", "Set " + field.getLabel() + " to " + formattedValue);
             detailRecord.withValue("newValue", formattedValue);
          }
@@ -329,8 +329,8 @@ public class DMLAuditAction extends AbstractQActionFunction<DMLAuditInput, DMLAu
             }
             else
             {
-               String formattedValue    = getFormattedValueForAuditDetail(record, fieldName, field, value);
-               String formattedOldValue = getFormattedValueForAuditDetail(oldRecord, fieldName, field, oldValue);
+               String formattedValue    = getFormattedValueForAuditDetail(table, record, fieldName, field, value);
+               String formattedOldValue = getFormattedValueForAuditDetail(table, oldRecord, fieldName, field, oldValue);
 
                if(oldValue == null)
                {
@@ -464,7 +464,7 @@ public class DMLAuditAction extends AbstractQActionFunction<DMLAuditInput, DMLAu
    /*******************************************************************************
     **
     *******************************************************************************/
-   private static String getFormattedValueForAuditDetail(QRecord record, String fieldName, QFieldMetaData field, Serializable value)
+   private static String getFormattedValueForAuditDetail(QTableMetaData table, QRecord record, String fieldName, QFieldMetaData field, Serializable value)
    {
       String formattedValue = null;
       if(value != null)
@@ -479,7 +479,8 @@ public class DMLAuditAction extends AbstractQActionFunction<DMLAuditInput, DMLAu
          }
          else
          {
-            formattedValue = QValueFormatter.formatValue(field, value);
+            QValueFormatter.setDisplayValuesInRecord(table, table.getFields(), record);
+            formattedValue = record.getDisplayValue(fieldName);
          }
       }
 

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/automation/AutomationStatus.java

@@ -127,7 +127,6 @@ public enum AutomationStatus implements PossibleValueEnum<Integer>
    /*******************************************************************************
     **
     *******************************************************************************/
-   @SuppressWarnings("checkstyle:indentation")
    public String getInsertOrUpdate()
    {
       return switch(this)

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/automation/RecordAutomationStatusUpdater.java

@@ -344,6 +344,9 @@ public class RecordAutomationStatusUpdater
 
 
 
+   /***************************************************************************
+    **
+    ***************************************************************************/
    private record Key(QTableMetaData table, TriggerEvent triggerEvent) {}
 
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/automation/polling/PollingAutomationPerTableRunner.java

@@ -132,6 +132,11 @@ public class PollingAutomationPerTableRunner implements Runnable
        *******************************************************************************/
       String tableName();
 
+      /*******************************************************************************
+       **
+       *******************************************************************************/
+      QTableAutomationDetails tableAutomationDetails();
+
       /*******************************************************************************
        **
        *******************************************************************************/
@@ -143,7 +148,7 @@ public class PollingAutomationPerTableRunner implements Runnable
    /*******************************************************************************
     ** Wrapper for a pair of (tableName, automationStatus)
     *******************************************************************************/
-   public record TableActions(String tableName, AutomationStatus status) implements TableActionsInterface
+   public record TableActions(String tableName, QTableAutomationDetails tableAutomationDetails, AutomationStatus status) implements TableActionsInterface
    {
       /*******************************************************************************
        **
@@ -159,7 +164,7 @@ public class PollingAutomationPerTableRunner implements Runnable
     ** extended version of TableAction, for sharding use-case - adds the shard
     ** details.
     *******************************************************************************/
-   public record ShardedTableActions(String tableName, AutomationStatus status, String shardByFieldName, Serializable shardValue, String shardLabel) implements TableActionsInterface
+   public record ShardedTableActions(String tableName, QTableAutomationDetails tableAutomationDetails, AutomationStatus status, String shardByFieldName, Serializable shardValue, String shardLabel) implements TableActionsInterface
    {
       /*******************************************************************************
        **
@@ -198,8 +203,8 @@ public class PollingAutomationPerTableRunner implements Runnable
                   {
                      Serializable shardId = record.getValue(automationDetails.getShardIdFieldName());
                      String       label   = record.getValueString(automationDetails.getShardLabelFieldName());
-                     tableActionList.add(new ShardedTableActions(table.getName(), AutomationStatus.PENDING_INSERT_AUTOMATIONS, automationDetails.getShardByFieldName(), shardId, label));
-                     tableActionList.add(new ShardedTableActions(table.getName(), AutomationStatus.PENDING_UPDATE_AUTOMATIONS, automationDetails.getShardByFieldName(), shardId, label));
+                     tableActionList.add(new ShardedTableActions(table.getName(), automationDetails, AutomationStatus.PENDING_INSERT_AUTOMATIONS, automationDetails.getShardByFieldName(), shardId, label));
+                     tableActionList.add(new ShardedTableActions(table.getName(), automationDetails, AutomationStatus.PENDING_UPDATE_AUTOMATIONS, automationDetails.getShardByFieldName(), shardId, label));
                   }
                }
                catch(Exception e)
@@ -209,11 +214,11 @@ public class PollingAutomationPerTableRunner implements Runnable
             }
             else
             {
-               ///////////////////////////////////////////////////////////////////
-               // for non-sharded, we just need tabler name & automation status //
-               ///////////////////////////////////////////////////////////////////
-               tableActionList.add(new TableActions(table.getName(), AutomationStatus.PENDING_INSERT_AUTOMATIONS));
-               tableActionList.add(new TableActions(table.getName(), AutomationStatus.PENDING_UPDATE_AUTOMATIONS));
+               //////////////////////////////////////////////////////////////////
+               // for non-sharded, we just need table name & automation status //
+               //////////////////////////////////////////////////////////////////
+               tableActionList.add(new TableActions(table.getName(), automationDetails, AutomationStatus.PENDING_INSERT_AUTOMATIONS));
+               tableActionList.add(new TableActions(table.getName(), automationDetails, AutomationStatus.PENDING_UPDATE_AUTOMATIONS));
             }
          }
       }
@@ -521,7 +526,7 @@ public class PollingAutomationPerTableRunner implements Runnable
          // note - this method - will re-query the objects, so we should have confidence that their data is fresh... //
          //////////////////////////////////////////////////////////////////////////////////////////////////////////////
          List<QRecord> matchingQRecords = getRecordsMatchingActionFilter(table, records, action);
-         LOG.debug("Of the {} records that were pending automations, {} of them match the filter on the action {}", records.size(), matchingQRecords.size(), action);
+         LOG.debug("Of the [" + records.size() + "] records that were pending automations, [" + matchingQRecords.size() + "] of them match the filter on the action:" +  action);
          if(CollectionUtils.nullSafeHasContents(matchingQRecords))
          {
             LOG.debug("  Processing " + matchingQRecords.size() + " records in " + table + " for action " + action);
@@ -596,7 +601,7 @@ public class PollingAutomationPerTableRunner implements Runnable
 
    /*******************************************************************************
     ** Finally, actually run action code against a list of known matching records.
-    ** todo not commit - move to somewhere genericer
+    **
     *******************************************************************************/
    public static void applyActionToMatchingRecords(QTableMetaData table, List<QRecord> records, TableAutomationAction action) throws Exception
    {

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/AbstractPostDeleteCustomizer.java

@@ -47,12 +47,24 @@ import com.kingsrook.qqq.backend.core.model.data.QRecord;
  ** records that the delete action marked in error - the user might want to do
  ** something special with them (idk, try some other way to delete them?)
  *******************************************************************************/
-public abstract class AbstractPostDeleteCustomizer
+public abstract class AbstractPostDeleteCustomizer implements TableCustomizerInterface
 {
    protected DeleteInput deleteInput;
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public List<QRecord> postDelete(DeleteInput deleteInput, List<QRecord> records) throws QException
+   {
+      this.deleteInput = deleteInput;
+      return apply(records);
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/AbstractPostInsertCustomizer.java

@@ -42,12 +42,24 @@ import com.kingsrook.qqq.backend.core.model.data.QRecord;
  **
  ** Note that the full insertInput is available as a field in this class.
  *******************************************************************************/
-public abstract class AbstractPostInsertCustomizer
+public abstract class AbstractPostInsertCustomizer implements TableCustomizerInterface
 {
    protected InsertInput insertInput;
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public List<QRecord> postInsert(InsertInput insertInput, List<QRecord> records) throws QException
+   {
+      this.insertInput = insertInput;
+      return (apply(records));
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/AbstractPostQueryCustomizer.java

@@ -23,16 +23,29 @@ package com.kingsrook.qqq.backend.core.actions.customizers;
 
 
 import java.util.List;
-import com.kingsrook.qqq.backend.core.model.actions.AbstractTableActionInput;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.actions.tables.QueryOrGetInputInterface;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
 
 
 /*******************************************************************************
  **
  *******************************************************************************/
-public abstract class AbstractPostQueryCustomizer
+public abstract class AbstractPostQueryCustomizer implements TableCustomizerInterface
 {
-   protected AbstractTableActionInput input;
+   protected QueryOrGetInputInterface input;
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public List<QRecord> postQuery(QueryOrGetInputInterface queryInput, List<QRecord> records) throws QException
+   {
+      input = queryInput;
+      return apply(records);
+   }
 
 
 
@@ -47,7 +60,7 @@ public abstract class AbstractPostQueryCustomizer
     ** Getter for input
     **
     *******************************************************************************/
-   public AbstractTableActionInput getInput()
+   public QueryOrGetInputInterface getInput()
    {
       return (input);
    }
@@ -58,7 +71,7 @@ public abstract class AbstractPostQueryCustomizer
     ** Setter for input
     **
     *******************************************************************************/
-   public void setInput(AbstractTableActionInput input)
+   public void setInput(QueryOrGetInputInterface input)
    {
       this.input = input;
    }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/AbstractPostUpdateCustomizer.java

@@ -26,6 +26,7 @@ import java.io.Serializable;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Optional;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
@@ -48,7 +49,7 @@ import com.kingsrook.qqq.backend.core.model.data.QRecord;
  ** available (if the backend supports it) - both as a list (`getOldRecordList`)
  ** and as a memoized (by this class) map of primaryKey to record (`getOldRecordMap`).
  *******************************************************************************/
-public abstract class AbstractPostUpdateCustomizer
+public abstract class AbstractPostUpdateCustomizer implements TableCustomizerInterface
 {
    protected UpdateInput   updateInput;
    protected List<QRecord> oldRecordList;
@@ -57,6 +58,19 @@ public abstract class AbstractPostUpdateCustomizer
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public List<QRecord> postUpdate(UpdateInput updateInput, List<QRecord> records, Optional<List<QRecord>> oldRecordList) throws QException
+   {
+      this.updateInput = updateInput;
+      this.oldRecordList = oldRecordList.orElse(null);
+      return apply(records);
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/AbstractPreDeleteCustomizer.java

@@ -50,7 +50,7 @@ import com.kingsrook.qqq.backend.core.model.data.QRecord;
  ** Note that the full deleteInput is available as a field in this class.
  **
  *******************************************************************************/
-public abstract class AbstractPreDeleteCustomizer
+public abstract class AbstractPreDeleteCustomizer implements TableCustomizerInterface
 {
    protected DeleteInput deleteInput;
 
@@ -58,6 +58,19 @@ public abstract class AbstractPreDeleteCustomizer
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public List<QRecord> preDelete(DeleteInput deleteInput, List<QRecord> records, boolean isPreview) throws QException
+   {
+      this.deleteInput = deleteInput;
+      this.isPreview = isPreview;
+      return apply(records);
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/AbstractPreInsertCustomizer.java

@@ -47,7 +47,7 @@ import com.kingsrook.qqq.backend.core.model.data.QRecord;
  **
  ** Note that the full insertInput is available as a field in this class.
  *******************************************************************************/
-public abstract class AbstractPreInsertCustomizer
+public abstract class AbstractPreInsertCustomizer implements TableCustomizerInterface
 {
    protected InsertInput insertInput;
 
@@ -55,10 +55,10 @@ public abstract class AbstractPreInsertCustomizer
 
 
 
-   /////////////////////////////////////////////////////////////////////////////////
-   // allow the customizer to specify when it should be executed as part of the   //
-   // insert action.  default (per method in this class) is AFTER_ALL_VALIDATIONS //
-   /////////////////////////////////////////////////////////////////////////////////
+   /***************************************************************************
+    ** allow the customizer to specify when it should be executed as part of the
+    ** insert action.  default (per method in this class) is AFTER_ALL_VALIDATIONS
+    ***************************************************************************/
    public enum WhenToRun
    {
       BEFORE_ALL_VALIDATIONS,
@@ -70,6 +70,30 @@ public abstract class AbstractPreInsertCustomizer
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public List<QRecord> preInsert(InsertInput insertInput, List<QRecord> records, boolean isPreview) throws QException
+   {
+      this.insertInput = insertInput;
+      this.isPreview = isPreview;
+      return (apply(records));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public WhenToRun whenToRunPreInsert(InsertInput insertInput, boolean isPreview)
+   {
+      return getWhenToRun();
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/AbstractPreUpdateCustomizer.java

@@ -26,6 +26,7 @@ import java.io.Serializable;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Optional;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
@@ -53,7 +54,7 @@ import com.kingsrook.qqq.backend.core.model.data.QRecord;
  ** available (if the backend supports it) - both as a list (`getOldRecordList`)
  ** and as a memoized (by this class) map of primaryKey to record (`getOldRecordMap`).
  *******************************************************************************/
-public abstract class AbstractPreUpdateCustomizer
+public abstract class AbstractPreUpdateCustomizer implements TableCustomizerInterface
 {
    protected UpdateInput   updateInput;
    protected List<QRecord> oldRecordList;
@@ -63,6 +64,20 @@ public abstract class AbstractPreUpdateCustomizer
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public List<QRecord> preUpdate(UpdateInput updateInput, List<QRecord> records, boolean isPreview, Optional<List<QRecord>> oldRecordList) throws QException
+   {
+      this.updateInput = updateInput;
+      this.isPreview = isPreview;
+      this.oldRecordList = oldRecordList.orElse(null);
+      return apply(records);
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/ChildInserterPostInsertCustomizer.java

@@ -28,6 +28,7 @@ import java.util.Iterator;
 import java.util.List;
 import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
 import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction;
+import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.exceptions.QRuntimeException;
 import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
@@ -49,6 +50,9 @@ import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
  *******************************************************************************/
 public abstract class ChildInserterPostInsertCustomizer extends AbstractPostInsertCustomizer
 {
+   /***************************************************************************
+    **
+    ***************************************************************************/
    public enum RelationshipType
    {
       PARENT_POINTS_AT_CHILD,
@@ -97,7 +101,7 @@ public abstract class ChildInserterPostInsertCustomizer extends AbstractPostInse
          List<QRecord>  rs               = records;
          List<QRecord>  childrenToInsert = new ArrayList<>();
          QTableMetaData table            = getInsertInput().getTable();
-         QTableMetaData childTable       = getInsertInput().getInstance().getTable(getChildTableName());
+         QTableMetaData childTable       = QContext.getQInstance().getTable(getChildTableName());
 
          ////////////////////////////////////////////////////////////////////////////////
          // iterate over the inserted records, building a list child records to insert //

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/QCodeLoader.java

@@ -22,6 +22,7 @@
 package com.kingsrook.qqq.backend.core.actions.customizers;
 
 
+import java.lang.reflect.Constructor;
 import java.util.Optional;
 import java.util.function.Function;
 import com.kingsrook.qqq.backend.core.actions.automation.RecordAutomationHandler;
@@ -34,42 +35,35 @@ import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeType;
 import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSource;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.TableAutomationAction;
+import com.kingsrook.qqq.backend.core.utils.lambdas.UnsafeFunction;
+import com.kingsrook.qqq.backend.core.utils.memoization.Memoization;
 import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
 
 
 /*******************************************************************************
  ** Utility to load code for running QQQ customizers.
+ **
+ ** TODO - redo all to go through method that memoizes class & constructor
+ ** lookup.  That memoziation causes 1,000,000 such calls to go from ~500ms
+ ** to ~100ms.
  *******************************************************************************/
 public class QCodeLoader
 {
    private static final QLogger LOG = QLogger.getLogger(QCodeLoader.class);
 
-
-
-   /*******************************************************************************
-    **
-    *******************************************************************************/
-   public static <T, R> Optional<Function<T, R>> getTableCustomizerFunction(QTableMetaData table, String customizerName)
-   {
-      Optional<QCodeReference> codeReference = table.getCustomizer(customizerName);
-      if(codeReference.isPresent())
-      {
-         return (Optional.ofNullable(QCodeLoader.getFunction(codeReference.get())));
-      }
-      return (Optional.empty());
-   }
+   private static Memoization<String, Constructor<?>> constructorMemoization = new Memoization<>();
 
 
 
    /*******************************************************************************
     **
     *******************************************************************************/
-   public static <T> Optional<T> getTableCustomizer(Class<T> expectedClass, QTableMetaData table, String customizerName)
+   public static Optional<TableCustomizerInterface> getTableCustomizer(QTableMetaData table, String customizerName)
    {
       Optional<QCodeReference> codeReference = table.getCustomizer(customizerName);
       if(codeReference.isPresent())
       {
-         return (Optional.ofNullable(QCodeLoader.getAdHoc(expectedClass, codeReference.get())));
+         return (Optional.ofNullable(QCodeLoader.getAdHoc(TableCustomizerInterface.class, codeReference.get())));
       }
       return (Optional.empty());
    }
@@ -102,7 +96,7 @@ public class QCodeLoader
       }
       catch(Exception e)
       {
-         LOG.error("Error initializing customizer", logPair("codeReference", codeReference), e);
+         LOG.error("Error initializing customizer", e, logPair("codeReference", codeReference));
 
          //////////////////////////////////////////////////////////////////////////////////////////////////////////
          // return null here - under the assumption that during normal run-time operations, we'll never hit here //
@@ -141,7 +135,7 @@ public class QCodeLoader
       }
       catch(Exception e)
       {
-         LOG.error("Error initializing customizer", logPair("codeReference", codeReference), e);
+         LOG.error("Error initializing customizer", e, logPair("codeReference", codeReference));
 
          //////////////////////////////////////////////////////////////////////////////////////////////////////////
          // return null here - under the assumption that during normal run-time operations, we'll never hit here //
@@ -175,12 +169,25 @@ public class QCodeLoader
 
       try
       {
-         Class<?> customizerClass = Class.forName(codeReference.getName());
-         return ((T) customizerClass.getConstructor().newInstance());
+         Optional<Constructor<?>> constructor = constructorMemoization.getResultThrowing(codeReference.getName(), (UnsafeFunction<String, Constructor<?>, Exception>) s ->
+         {
+            Class<?> customizerClass = Class.forName(codeReference.getName());
+            return customizerClass.getConstructor();
+         });
+
+         if(constructor.isPresent())
+         {
+            return ((T) constructor.get().newInstance());
+         }
+         else
+         {
+            LOG.error("Could not get constructor for code reference", logPair("codeReference", codeReference));
+            return (null);
+         }
       }
       catch(Exception e)
       {
-         LOG.error("Error initializing customizer", logPair("codeReference", codeReference), e);
+         LOG.error("Error initializing customizer", e, logPair("codeReference", codeReference));
 
          //////////////////////////////////////////////////////////////////////////////////////////////////////////
          // return null here - under the assumption that during normal run-time operations, we'll never hit here //

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/RecordCustomizerUtilityInterface.java

@@ -50,10 +50,7 @@ public interface RecordCustomizerUtilityInterface
    /*******************************************************************************
     ** Container for an old value and a new value.
     *******************************************************************************/
-   @SuppressWarnings("checkstyle:MethodName")
-   record Change(Serializable oldValue, Serializable newValue)
-   {
-   }
+   record Change(Serializable oldValue, Serializable newValue) {}
 
 
    /*******************************************************************************

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/TableCustomizerInterface.java

@@ -0,0 +1,202 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.actions.customizers;
+
+
+import java.util.List;
+import java.util.Optional;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.logging.QLogger;
+import com.kingsrook.qqq.backend.core.model.actions.tables.QueryOrGetInputInterface;
+import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
+
+
+/*******************************************************************************
+ ** Common interface used by all (core) TableCustomizer types (e.g., post-query,
+ ** and {pre,post}-{insert,update,delete}.
+ **
+ ** Note that the abstract-base classes for each action still exist, though have
+ ** been back-ported to be implementors of this interface.  The action classes
+ ** will now expect this type, and call this type's methods.
+ **
+ *******************************************************************************/
+public interface TableCustomizerInterface
+{
+   QLogger LOG = QLogger.getLogger(TableCustomizerInterface.class);
+
+
+   /*******************************************************************************
+    ** custom actions to run after a query (or get!) takes place.
+    **
+    *******************************************************************************/
+   default List<QRecord> postQuery(QueryOrGetInputInterface queryInput, List<QRecord> records) throws QException
+   {
+      LOG.info("A default implementation of postQuery is running...  Probably not expected!", logPair("tableName", queryInput.getTableName()));
+      return (records);
+   }
+
+
+   /*******************************************************************************
+    ** custom actions before an insert takes place.
+    **
+    ** It's important for implementations to be aware of the isPreview field, which
+    ** is set to true when the code is running to give users advice, e.g., on a review
+    ** screen - vs. being false when the action is ACTUALLY happening.  So, if you're doing
+    ** things like storing data, you don't want to do that if isPreview is true!!
+    **
+    ** General implementation would be, to iterate over the records (the inputs to
+    ** the insert action), and look at their values:
+    ** - possibly adding Errors (`addError`) or Warnings (`addWarning`) to the records
+    ** - possibly manipulating values (`setValue`)
+    ** - possibly throwing an exception - if you really don't want the insert operation to continue.
+    ** - doing "whatever else" you may want to do.
+    ** - returning the list of records (can be the input list) that you want to go on to the backend implementation class.
+    *******************************************************************************/
+   default List<QRecord> preInsert(InsertInput insertInput, List<QRecord> records, boolean isPreview) throws QException
+   {
+      LOG.info("A default implementation of preInsert is running...  Probably not expected!", logPair("tableName", insertInput.getTableName()));
+      return (records);
+   }
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   default AbstractPreInsertCustomizer.WhenToRun whenToRunPreInsert(InsertInput insertInput, boolean isPreview)
+   {
+      return (AbstractPreInsertCustomizer.WhenToRun.AFTER_ALL_VALIDATIONS);
+   }
+
+
+   /*******************************************************************************
+    ** custom actions after an insert takes place.
+    **
+    ** General implementation would be, to iterate over the records (the outputs of
+    ** the insert action), and look at their values:
+    ** - possibly adding Errors (`addError`) or Warnings (`addWarning`) to the records
+    ** - possibly throwing an exception - though doing so won't stop the update, and instead
+    **   will just set a warning on all of the updated records...
+    ** - doing "whatever else" you may want to do.
+    ** - returning the list of records (can be the input list) that you want to go back to the caller.
+    *******************************************************************************/
+   default List<QRecord> postInsert(InsertInput insertInput, List<QRecord> records) throws QException
+   {
+      LOG.info("A default implementation of postInsert is running...  Probably not expected!", logPair("tableName", insertInput.getTableName()));
+      return (records);
+   }
+
+
+   /*******************************************************************************
+    ** custom actions before an update takes place.
+    **
+    ** It's important for implementations to be aware of the isPreview field, which
+    ** is set to true when the code is running to give users advice, e.g., on a review
+    ** screen - vs. being false when the action is ACTUALLY happening.  So, if you're doing
+    ** things like storing data, you don't want to do that if isPreview is true!!
+    **
+    ** General implementation would be, to iterate over the records (the inputs to
+    ** the update action), and look at their values:
+    ** - possibly adding Errors (`addError`) or Warnings (`addWarning`) to the records
+    ** - possibly manipulating values (`setValue`)
+    ** - possibly throwing an exception - if you really don't want the update operation to continue.
+    ** - doing "whatever else" you may want to do.
+    ** - returning the list of records (can be the input list) that you want to go on to the backend implementation class.
+    **
+    ** Note, "old records" (e.g., with values freshly fetched from the backend) will be
+    ** available (if the backend supports it)
+    *******************************************************************************/
+   default List<QRecord> preUpdate(UpdateInput updateInput, List<QRecord> records, boolean isPreview, Optional<List<QRecord>> oldRecordList) throws QException
+   {
+      LOG.info("A default implementation of preUpdate is running...  Probably not expected!", logPair("tableName", updateInput.getTableName()));
+      return (records);
+   }
+
+
+   /*******************************************************************************
+    ** custom actions after an update takes place.
+    **
+    ** General implementation would be, to iterate over the records (the outputs of
+    ** the update action), and look at their values:
+    ** - possibly adding Errors (`addError`) or Warnings (`addWarning`) to the records?
+    ** - possibly throwing an exception - though doing so won't stop the update, and instead
+    **   will just set a warning on all of the updated records...
+    ** - doing "whatever else" you may want to do.
+    ** - returning the list of records (can be the input list) that you want to go back to the caller.
+    **
+    ** Note, "old records" (e.g., with values freshly fetched from the backend) will be
+    ** available (if the backend supports it).
+    *******************************************************************************/
+   default List<QRecord> postUpdate(UpdateInput updateInput, List<QRecord> records, Optional<List<QRecord>> oldRecordList) throws QException
+   {
+      LOG.info("A default implementation of postUpdate is running...  Probably not expected!", logPair("tableName", updateInput.getTableName()));
+      return (records);
+   }
+
+
+   /*******************************************************************************
+    ** Custom actions before a delete takes place.
+    **
+    ** It's important for implementations to be aware of the isPreview param, which
+    ** is set to true when the code is running to give users advice, e.g., on a review
+    ** screen - vs. being false when the action is ACTUALLY happening.  So, if you're doing
+    ** things like storing data, you don't want to do that if isPreview is true!!
+    **
+    ** General implementation would be, to iterate over the records (which the DeleteAction
+    ** would look up based on the inputs to the delete action), and look at their values:
+    ** - possibly adding Errors (`addError`) or Warnings (`addWarning`) to the records
+    ** - possibly throwing an exception - if you really don't want the delete operation to continue.
+    ** - doing "whatever else" you may want to do.
+    ** - returning the list of records (can be the input list) - this is how errors
+    **   and warnings are propagated to the DeleteAction.  Note that any records with
+    **   an error will NOT proceed to the backend's delete interface - but those with
+    **   warnings will.
+    *******************************************************************************/
+   default List<QRecord> preDelete(DeleteInput deleteInput, List<QRecord> records, boolean isPreview) throws QException
+   {
+      LOG.info("A default implementation of preDelete is running...  Probably not expected!", logPair("tableName", deleteInput.getTableName()));
+      return (records);
+   }
+
+
+   /*******************************************************************************
+    ** Custom actions after a delete takes place.
+    **
+    ** General implementation would be, to iterate over the records (ones which didn't
+    ** have a delete error), and look at their values:
+    ** - possibly adding Errors (`addError`) or Warnings (`addWarning`) to the records?
+    ** - possibly throwing an exception - though doing so won't stop the delete, and instead
+    **   will just set a warning on all of the deleted records...
+    ** - doing "whatever else" you may want to do.
+    ** - returning the list of records (can be the input list) that you want to go back
+    **   to the caller - this is how errors and warnings are propagated .
+    *******************************************************************************/
+   default List<QRecord> postDelete(DeleteInput deleteInput, List<QRecord> records) throws QException
+   {
+      LOG.info("A default implementation of postDelete is running...  Probably not expected!", logPair("tableName", deleteInput.getTableName()));
+      return (records);
+   }
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/TableCustomizers.java

@@ -29,13 +29,13 @@ package com.kingsrook.qqq.backend.core.actions.customizers;
  *******************************************************************************/
 public enum TableCustomizers
 {
-   POST_QUERY_RECORD("postQueryRecord", AbstractPostQueryCustomizer.class),
-   PRE_INSERT_RECORD("preInsertRecord", AbstractPreInsertCustomizer.class),
-   POST_INSERT_RECORD("postInsertRecord", AbstractPostInsertCustomizer.class),
-   PRE_UPDATE_RECORD("preUpdateRecord", AbstractPreUpdateCustomizer.class),
-   POST_UPDATE_RECORD("postUpdateRecord", AbstractPostUpdateCustomizer.class),
-   PRE_DELETE_RECORD("preDeleteRecord", AbstractPreDeleteCustomizer.class),
-   POST_DELETE_RECORD("postDeleteRecord", AbstractPostDeleteCustomizer.class);
+   POST_QUERY_RECORD("postQueryRecord", TableCustomizerInterface.class),
+   PRE_INSERT_RECORD("preInsertRecord", TableCustomizerInterface.class),
+   POST_INSERT_RECORD("postInsertRecord", TableCustomizerInterface.class),
+   PRE_UPDATE_RECORD("preUpdateRecord", TableCustomizerInterface.class),
+   POST_UPDATE_RECORD("postUpdateRecord", TableCustomizerInterface.class),
+   PRE_DELETE_RECORD("preDeleteRecord", TableCustomizerInterface.class),
+   POST_DELETE_RECORD("postDeleteRecord", TableCustomizerInterface.class);
 
 
    private final String   role;

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/dashboard/AbstractHTMLWidgetRenderer.java

@@ -161,7 +161,7 @@ public abstract class AbstractHTMLWidgetRenderer extends AbstractWidgetRenderer
    public static String linkTableCreateWithDefaultValues(RenderWidgetInput input, String tableName, Map<String, Serializable> defaultValues) throws QException
    {
       String tablePath = QContext.getQInstance().getTablePath(tableName);
-      return (tablePath + "/create?defaultValues=" + URLEncoder.encode(JsonUtils.toJson(defaultValues), Charset.defaultCharset()));
+      return (tablePath + "/create#defaultValues=" + URLEncoder.encode(JsonUtils.toJson(defaultValues), Charset.defaultCharset()));
    }
 
 
@@ -183,7 +183,6 @@ public abstract class AbstractHTMLWidgetRenderer extends AbstractWidgetRenderer
 
 
 
-
    /*******************************************************************************
     **
     *******************************************************************************/

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/dashboard/widgets/AbstractWidgetRenderer.java

@@ -31,6 +31,7 @@ import java.util.Map;
 import java.util.Set;
 import com.kingsrook.qqq.backend.core.actions.values.QValueFormatter;
 import com.kingsrook.qqq.backend.core.actions.values.SearchPossibleValueSourceAction;
+import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
@@ -42,6 +43,7 @@ import com.kingsrook.qqq.backend.core.model.actions.widgets.RenderWidgetOutput;
 import com.kingsrook.qqq.backend.core.model.dashboard.widgets.QWidgetData;
 import com.kingsrook.qqq.backend.core.model.metadata.dashboard.QWidgetMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.dashboard.WidgetDropdownData;
+import com.kingsrook.qqq.backend.core.model.metadata.dashboard.WidgetDropdownType;
 import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValue;
 import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSource;
 import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
@@ -72,80 +74,104 @@ public abstract class AbstractWidgetRenderer
     *******************************************************************************/
    protected boolean setupDropdowns(RenderWidgetInput input, QWidgetMetaData metaData, QWidgetData widgetData) throws QException
    {
-      List<List<Map<String, String>>> pvsData                   = new ArrayList<>();
-      List<String>                    pvsLabels                 = new ArrayList<>();
-      List<String>                    pvsNames                  = new ArrayList<>();
+      List<List<Map<String, String>>> dataList                  = new ArrayList<>();
+      List<String>                    labelList                 = new ArrayList<>();
+      List<String>                    nameList                  = new ArrayList<>();
       List<String>                    missingRequiredSelections = new ArrayList<>();
       for(WidgetDropdownData dropdownData : CollectionUtils.nonNullList(metaData.getDropdowns()))
       {
-         String               possibleValueSourceName = dropdownData.getPossibleValueSourceName();
-         QPossibleValueSource possibleValueSource     = input.getInstance().getPossibleValueSource(possibleValueSourceName);
-
-         /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-         // this looks complicated, but is just look for a label in the dropdown data and if found use it,                                                                                  //
-         // otherwise look for label in PVS and if found use that, otherwise just use the PVS name                                                                                          //
-         /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-         String pvsLabel = dropdownData.getLabel() != null ? dropdownData.getLabel() : (possibleValueSource.getLabel() != null ? possibleValueSource.getLabel() : possibleValueSourceName);
-         pvsLabels.add(pvsLabel);
-         pvsNames.add(possibleValueSourceName);
-
-         SearchPossibleValueSourceInput pvsInput = new SearchPossibleValueSourceInput();
-         pvsInput.setPossibleValueSourceName(possibleValueSourceName);
-
-         if(dropdownData.getForeignKeyFieldName() != null)
+         if(WidgetDropdownType.DATE_PICKER.equals(dropdownData.getType()))
          {
-            ////////////////////////////////////////
-            // look for an id in the query params //
-            ////////////////////////////////////////
-            Integer id = null;
-            if(input.getQueryParams() != null && input.getQueryParams().containsKey("id") && StringUtils.hasContent(input.getQueryParams().get("id")))
+            String name = dropdownData.getName();
+            nameList.add(name);
+            labelList.add(dropdownData.getLabel());
+            dataList.add(new ArrayList<>());
+
+            ////////////////////////////////////////////////////////////////////////////////////////////////////////////
+            // sure that something has been selected, and if not, display a message that a selection needs made       //
+            ////////////////////////////////////////////////////////////////////////////////////////////////////////////
+            if(dropdownData.getIsRequired())
             {
-               id = Integer.parseInt(input.getQueryParams().get("id"));
-            }
-            if(id != null)
-            {
-               pvsInput.setDefaultQueryFilter(new QQueryFilter().withCriteria(
-                  new QFilterCriteria(
-                     dropdownData.getForeignKeyFieldName(),
-                     QCriteriaOperator.EQUALS,
-                     id)));
+               if(!input.getQueryParams().containsKey(name) || !StringUtils.hasContent(input.getQueryParams().get(name)))
+               {
+                  missingRequiredSelections.add(dropdownData.getLabel());
+               }
             }
          }
-
-         SearchPossibleValueSourceOutput output = new SearchPossibleValueSourceAction().execute(pvsInput);
-
-         List<Map<String, String>> dropdownOptionList = new ArrayList<>();
-         pvsData.add(dropdownOptionList);
-
-         //////////////////////////////////////////
-         // sort results, dedupe, and add to map //
-         //////////////////////////////////////////
-         Set<String> exists = new HashSet<>();
-         output.getResults().removeIf(pvs -> !exists.add(pvs.getLabel()));
-         for(QPossibleValue<?> possibleValue : output.getResults())
+         else
          {
-            dropdownOptionList.add(MapBuilder.of(
-               "id", String.valueOf(possibleValue.getId()),
-               "label", possibleValue.getLabel()
-            ));
-         }
-
-         ////////////////////////////////////////////////////////////////////////////////////////////////////////////
-         // because we know the dropdowns and what the field names will be when something is selected, we can make //
-         // sure that something has been selected, and if not, display a message that a selection needs made       //
-         ////////////////////////////////////////////////////////////////////////////////////////////////////////////
-         if(dropdownData.getIsRequired())
-         {
-            if(!input.getQueryParams().containsKey(possibleValueSourceName) || !StringUtils.hasContent(input.getQueryParams().get(possibleValueSourceName)))
+            String possibleValueSourceName = dropdownData.getPossibleValueSourceName();
+            if(possibleValueSourceName != null)
             {
-               missingRequiredSelections.add(pvsLabel);
+               QPossibleValueSource possibleValueSource = QContext.getQInstance().getPossibleValueSource(possibleValueSourceName);
+
+               /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+               // this looks complicated, but is just look for a label in the dropdown data and if found use it,                                                                                  //
+               // otherwise look for label in PVS and if found use that, otherwise just use the PVS name                                                                                          //
+               /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+               String pvsLabel = dropdownData.getLabel() != null ? dropdownData.getLabel() : (possibleValueSource.getLabel() != null ? possibleValueSource.getLabel() : possibleValueSourceName);
+               labelList.add(pvsLabel);
+               nameList.add(possibleValueSourceName);
+
+               SearchPossibleValueSourceInput pvsInput = new SearchPossibleValueSourceInput();
+               pvsInput.setPossibleValueSourceName(possibleValueSourceName);
+
+               if(dropdownData.getForeignKeyFieldName() != null)
+               {
+                  ////////////////////////////////////////
+                  // look for an id in the query params //
+                  ////////////////////////////////////////
+                  Integer id = null;
+                  if(input.getQueryParams() != null && input.getQueryParams().containsKey("id") && StringUtils.hasContent(input.getQueryParams().get("id")))
+                  {
+                     id = Integer.parseInt(input.getQueryParams().get("id"));
+                  }
+                  if(id != null)
+                  {
+                     pvsInput.setDefaultQueryFilter(new QQueryFilter().withCriteria(
+                        new QFilterCriteria(
+                           dropdownData.getForeignKeyFieldName(),
+                           QCriteriaOperator.EQUALS,
+                           id)));
+                  }
+               }
+
+               SearchPossibleValueSourceOutput output = new SearchPossibleValueSourceAction().execute(pvsInput);
+
+               List<Map<String, String>> dropdownOptionList = new ArrayList<>();
+               dataList.add(dropdownOptionList);
+
+               //////////////////////////////////////////
+               // sort results, dedupe, and add to map //
+               //////////////////////////////////////////
+               Set<String> exists = new HashSet<>();
+               output.getResults().removeIf(pvs -> !exists.add(pvs.getLabel()));
+               for(QPossibleValue<?> possibleValue : output.getResults())
+               {
+                  dropdownOptionList.add(MapBuilder.of(
+                     "id", String.valueOf(possibleValue.getId()),
+                     "label", possibleValue.getLabel()
+                  ));
+               }
+
+               ////////////////////////////////////////////////////////////////////////////////////////////////////////////
+               // because we know the dropdowns and what the field names will be when something is selected, we can make //
+               // sure that something has been selected, and if not, display a message that a selection needs made       //
+               ////////////////////////////////////////////////////////////////////////////////////////////////////////////
+               if(dropdownData.getIsRequired())
+               {
+                  if(!input.getQueryParams().containsKey(possibleValueSourceName) || !StringUtils.hasContent(input.getQueryParams().get(possibleValueSourceName)))
+                  {
+                     missingRequiredSelections.add(pvsLabel);
+                  }
+               }
             }
          }
       }
 
-      widgetData.setDropdownNameList(pvsNames);
-      widgetData.setDropdownLabelList(pvsLabels);
-      widgetData.setDropdownDataList(pvsData);
+      widgetData.setDropdownNameList(nameList);
+      widgetData.setDropdownLabelList(labelList);
+      widgetData.setDropdownDataList(dataList);
 
       ////////////////////////////////////////////////////////////////////////////////
       // if there are any missing required dropdowns, build up a message to display //

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/dashboard/widgets/Aggregate2DTableWidgetRenderer.java

@@ -0,0 +1,202 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.actions.dashboard.widgets;
+
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.LinkedHashMap;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import com.kingsrook.qqq.backend.core.actions.tables.AggregateAction;
+import com.kingsrook.qqq.backend.core.context.QContext;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.logging.QLogger;
+import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.Aggregate;
+import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateOperator;
+import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateOutput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateResult;
+import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.GroupBy;
+import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.QFilterOrderByAggregate;
+import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.QFilterOrderByGroupBy;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
+import com.kingsrook.qqq.backend.core.model.actions.widgets.RenderWidgetInput;
+import com.kingsrook.qqq.backend.core.model.actions.widgets.RenderWidgetOutput;
+import com.kingsrook.qqq.backend.core.model.dashboard.widgets.TableData;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+import com.kingsrook.qqq.backend.core.utils.StringUtils;
+import com.kingsrook.qqq.backend.core.utils.ValueUtils;
+
+
+/*******************************************************************************
+ ** Generic widget that does an aggregate query, and presents its results
+ ** as a table, using group-by values as both row & column labels.
+ *******************************************************************************/
+public class Aggregate2DTableWidgetRenderer extends AbstractWidgetRenderer
+{
+   private static final QLogger LOG = QLogger.getLogger(Aggregate2DTableWidgetRenderer.class);
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public RenderWidgetOutput render(RenderWidgetInput input) throws QException
+   {
+      Map<String, Serializable> values = input.getWidgetMetaData().getDefaultValues();
+
+      String         tableName   = ValueUtils.getValueAsString(values.get("tableName"));
+      String         valueField  = ValueUtils.getValueAsString(values.get("valueField"));
+      String         rowField    = ValueUtils.getValueAsString(values.get("rowField"));
+      String         columnField = ValueUtils.getValueAsString(values.get("columnField"));
+      QTableMetaData table       = QContext.getQInstance().getTable(tableName);
+
+      AggregateInput aggregateInput = new AggregateInput();
+      aggregateInput.setTableName(tableName);
+
+      // todo - allow input of "list of columns" (e.g., in case some miss sometimes, or as a version of filter)
+      // todo - max rows, max cols?
+
+      // todo - from input map
+      QQueryFilter filter = new QQueryFilter();
+      aggregateInput.setFilter(filter);
+
+      Aggregate aggregate = new Aggregate(valueField, AggregateOperator.COUNT);
+      aggregateInput.withAggregate(aggregate);
+
+      GroupBy rowGroupBy    = new GroupBy(table.getField(rowField));
+      GroupBy columnGroupBy = new GroupBy(table.getField(columnField));
+      aggregateInput.withGroupBy(rowGroupBy);
+      aggregateInput.withGroupBy(columnGroupBy);
+
+      String orderBys = ValueUtils.getValueAsString(values.get("orderBys"));
+      if(StringUtils.hasContent(orderBys))
+      {
+         for(String orderBy : orderBys.split(","))
+         {
+            switch(orderBy)
+            {
+               case "row" -> filter.addOrderBy(new QFilterOrderByGroupBy(rowGroupBy));
+               case "column" -> filter.addOrderBy(new QFilterOrderByGroupBy(columnGroupBy));
+               case "value" -> filter.addOrderBy(new QFilterOrderByAggregate(aggregate));
+               default -> LOG.warn("Unrecognized orderBy: " + orderBy);
+            }
+         }
+      }
+
+      AggregateOutput aggregateOutput = new AggregateAction().execute(aggregateInput);
+
+      Map<Serializable, Map<Serializable, Serializable>> data       = new LinkedHashMap<>();
+      Set<Serializable>                                  columnsSet = new LinkedHashSet<>();
+
+      for(AggregateResult result : aggregateOutput.getResults())
+      {
+         Serializable column = result.getGroupByValue(columnGroupBy);
+         Serializable row    = result.getGroupByValue(rowGroupBy);
+         Serializable value  = result.getAggregateValue(aggregate);
+
+         Map<Serializable, Serializable> rowMap = data.computeIfAbsent(row, (k) -> new LinkedHashMap<>());
+         rowMap.put(column, value);
+         columnsSet.add(column);
+      }
+
+      // todo - possible values from rows, cols
+
+      ////////////////////////////////////
+      // setup datastructures for table //
+      ////////////////////////////////////
+      List<Map<String, Object>> tableRows    = new ArrayList<>();
+      List<TableData.Column>    tableColumns = new ArrayList<>();
+      tableColumns.add(new TableData.Column("default", table.getField(rowField).getLabel(), "_row", "2fr", "left"));
+
+      for(Serializable column : columnsSet)
+      {
+         tableColumns.add(new TableData.Column("default", String.valueOf(column) /* todo display value */, String.valueOf(column), "1fr", "right"));
+      }
+
+      tableColumns.add(new TableData.Column("default", "Total", "_total", "1fr", "right"));
+
+      TableData tableData = new TableData(null, tableColumns, tableRows)
+         .withRowsPerPage(100)
+         .withFixedStickyLastRow(false)
+         .withHidePaginationDropdown(true);
+
+      Map<Serializable, Integer> columnSums = new HashMap<>();
+      int                        grandTotal = 0;
+      for(Map.Entry<Serializable, Map<Serializable, Serializable>> rowEntry : data.entrySet())
+      {
+         Map<String, Object> rowMap = new HashMap<>();
+         tableRows.add(rowMap);
+
+         rowMap.put("_row", rowEntry.getKey() /* todo display value */);
+         int rowTotal = 0;
+         for(Serializable column : columnsSet)
+         {
+            Serializable value = rowEntry.getValue().get(column);
+            if(value == null)
+            {
+               value = 0; // todo?
+            }
+
+            Integer valueAsInteger = Objects.requireNonNullElse(ValueUtils.getValueAsInteger(value), 0);
+            rowTotal += valueAsInteger;
+            columnSums.putIfAbsent(column, 0);
+            columnSums.put(column, columnSums.get(column) + valueAsInteger);
+
+            rowMap.put(String.valueOf(column), value); // todo format commas?
+         }
+
+         rowMap.put("_total", rowTotal);
+         grandTotal += rowTotal;
+      }
+
+      ///////////////
+      // total row //
+      ///////////////
+      Map<String, Object> totalRowMap = new HashMap<>();
+      tableRows.add(totalRowMap);
+
+      totalRowMap.put("_row", "Total");
+      int rowTotal = 0;
+      for(Serializable column : columnsSet)
+      {
+         Serializable value = columnSums.get(column);
+         if(value == null)
+         {
+            value = 0; // todo?
+         }
+
+         totalRowMap.put(String.valueOf(column), value); // todo format commas?
+      }
+
+      totalRowMap.put("_total", grandTotal);
+
+      return (new RenderWidgetOutput(tableData));
+   }
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/dashboard/widgets/ChildRecordListRenderer.java

@@ -34,8 +34,10 @@ import com.kingsrook.qqq.backend.core.actions.tables.CountAction;
 import com.kingsrook.qqq.backend.core.actions.tables.GetAction;
 import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
 import com.kingsrook.qqq.backend.core.actions.values.QValueFormatter;
+import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.exceptions.QNotFoundException;
+import com.kingsrook.qqq.backend.core.logging.QLogger;
 import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetOutput;
@@ -59,6 +61,7 @@ import com.kingsrook.qqq.backend.core.utils.JsonUtils;
 import com.kingsrook.qqq.backend.core.utils.StringUtils;
 import com.kingsrook.qqq.backend.core.utils.ValueUtils;
 import org.apache.commons.lang.BooleanUtils;
+import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
 
 
 /*******************************************************************************
@@ -66,6 +69,9 @@ import org.apache.commons.lang.BooleanUtils;
  *******************************************************************************/
 public class ChildRecordListRenderer extends AbstractWidgetRenderer
 {
+   private static final QLogger LOG = QLogger.getLogger(ChildRecordListRenderer.class);
+
+
 
    /*******************************************************************************
     **
@@ -137,7 +143,7 @@ public class ChildRecordListRenderer extends AbstractWidgetRenderer
        *******************************************************************************/
       public Builder withCanAddChildRecord(boolean b)
       {
-         widgetMetaData.withDefaultValue("canAddChildRecord", true);
+         widgetMetaData.withDefaultValue("canAddChildRecord", b);
          return (this);
       }
 
@@ -151,6 +157,17 @@ public class ChildRecordListRenderer extends AbstractWidgetRenderer
          widgetMetaData.withDefaultValue("disabledFieldsForNewChildRecords", new HashSet<>(disabledFieldsForNewChildRecords));
          return (this);
       }
+
+
+
+      /*******************************************************************************
+       **
+       *******************************************************************************/
+      public Builder withManageAssociationName(String manageAssociationName)
+      {
+         widgetMetaData.withDefaultValue("manageAssociationName", manageAssociationName);
+         return (this);
+      }
    }
 
 
@@ -161,98 +178,136 @@ public class ChildRecordListRenderer extends AbstractWidgetRenderer
    @Override
    public RenderWidgetOutput render(RenderWidgetInput input) throws QException
    {
-      String         widgetLabel = input.getQueryParams().get("widgetLabel");
-      String         joinName    = input.getQueryParams().get("joinName");
-      QJoinMetaData  join        = input.getInstance().getJoin(joinName);
-      String         id          = input.getQueryParams().get("id");
-      QTableMetaData leftTable   = input.getInstance().getTable(join.getLeftTable());
-      QTableMetaData rightTable  = input.getInstance().getTable(join.getRightTable());
-
-      Integer maxRows = null;
-      if(StringUtils.hasContent(input.getQueryParams().get("maxRows")))
+      try
       {
-         maxRows = ValueUtils.getValueAsInteger(input.getQueryParams().get("maxRows"));
-      }
-      else if(input.getWidgetMetaData().getDefaultValues().containsKey("maxRows"))
-      {
-         maxRows = ValueUtils.getValueAsInteger(input.getWidgetMetaData().getDefaultValues().containsKey("maxRows"));
-      }
+         String         widgetLabel = input.getQueryParams().get("widgetLabel");
+         String         joinName    = input.getQueryParams().get("joinName");
+         QJoinMetaData  join        = QContext.getQInstance().getJoin(joinName);
+         String         id          = input.getQueryParams().get("id");
+         QTableMetaData leftTable   = QContext.getQInstance().getTable(join.getLeftTable());
+         QTableMetaData rightTable  = QContext.getQInstance().getTable(join.getRightTable());
 
-      ////////////////////////////////////////////////////////
-      // fetch the record that we're getting children for.  //
-      // e.g., the left-side of the join, with the input id //
-      ////////////////////////////////////////////////////////
-      GetInput getInput = new GetInput();
-      getInput.setTableName(join.getLeftTable());
-      getInput.setPrimaryKey(id);
-      GetOutput getOutput = new GetAction().execute(getInput);
-      QRecord   record    = getOutput.getRecord();
-
-      if(record == null)
-      {
-         throw (new QNotFoundException("Could not find " + (leftTable == null ? "" : leftTable.getLabel()) + " with primary key " + id));
-      }
-
-      ////////////////////////////////////////////////////////////////////
-      // set up the query - for the table on the right side of the join //
-      ////////////////////////////////////////////////////////////////////
-      QQueryFilter filter = new QQueryFilter();
-      for(JoinOn joinOn : join.getJoinOns())
-      {
-         filter.addCriteria(new QFilterCriteria(joinOn.getRightField(), QCriteriaOperator.EQUALS, List.of(record.getValue(joinOn.getLeftField()))));
-      }
-      filter.setOrderBys(join.getOrderBys());
-      filter.setLimit(maxRows);
-
-      QueryInput queryInput = new QueryInput();
-      queryInput.setTableName(join.getRightTable());
-      queryInput.setShouldTranslatePossibleValues(true);
-      queryInput.setShouldGenerateDisplayValues(true);
-      queryInput.setFilter(filter);
-      QueryOutput queryOutput = new QueryAction().execute(queryInput);
-
-      QValueFormatter.setBlobValuesToDownloadUrls(rightTable, queryOutput.getRecords());
-
-      int totalRows = queryOutput.getRecords().size();
-      if(maxRows != null && (queryOutput.getRecords().size() == maxRows))
-      {
-         /////////////////////////////////////////////////////////////////////////////////////
-         // if the input said to only do some max, and the # of results we got is that max, //
-         // then do a count query, for displaying 1-n of <count>                            //
-         /////////////////////////////////////////////////////////////////////////////////////
-         CountInput countInput = new CountInput();
-         countInput.setTableName(join.getRightTable());
-         countInput.setFilter(filter);
-         totalRows = new CountAction().execute(countInput).getCount();
-      }
-
-      String tablePath   = input.getInstance().getTablePath(rightTable.getName());
-      String viewAllLink = tablePath == null ? null : (tablePath + "?filter=" + URLEncoder.encode(JsonUtils.toJson(filter), Charset.defaultCharset()));
-
-      ChildRecordListData widgetData = new ChildRecordListData(widgetLabel, queryOutput, rightTable, tablePath, viewAllLink, totalRows);
-
-      if(BooleanUtils.isTrue(ValueUtils.getValueAsBoolean(input.getQueryParams().get("canAddChildRecord"))))
-      {
-         widgetData.setCanAddChildRecord(true);
-
-         //////////////////////////////////////////////////////////
-         // new child records must have values from the join-ons //
-         //////////////////////////////////////////////////////////
-         Map<String, Serializable> defaultValuesForNewChildRecords = new HashMap<>();
-         for(JoinOn joinOn : join.getJoinOns())
+         Integer maxRows = null;
+         if(StringUtils.hasContent(input.getQueryParams().get("maxRows")))
          {
-            defaultValuesForNewChildRecords.put(joinOn.getRightField(), record.getValue(joinOn.getLeftField()));
+            maxRows = ValueUtils.getValueAsInteger(input.getQueryParams().get("maxRows"));
          }
-         widgetData.setDefaultValuesForNewChildRecords(defaultValuesForNewChildRecords);
-
-         Map<String, Serializable> widgetValues = input.getWidgetMetaData().getDefaultValues();
-         if(widgetValues.containsKey("disabledFieldsForNewChildRecords"))
+         else if(input.getWidgetMetaData().getDefaultValues().containsKey("maxRows"))
          {
-            widgetData.setDisabledFieldsForNewChildRecords((Set<String>) widgetValues.get("disabledFieldsForNewChildRecords"));
+            maxRows = ValueUtils.getValueAsInteger(input.getWidgetMetaData().getDefaultValues().containsKey("maxRows"));
          }
-      }
 
-      return (new RenderWidgetOutput(widgetData));
+         //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+         // fetch the record that we're getting children for. e.g., the left-side of the join, with the input id                                     //
+         // but - only try this if we were given an id.  note, this widget could be called for on an INSERT screen, where we don't have a record yet //
+         // but we still want to be able to return all the other data in here that otherwise comes from the widget meta data, join, etc.             //
+         //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+         int          totalRows     = 0;
+         QRecord      primaryRecord = null;
+         QQueryFilter filter        = null;
+         QueryOutput  queryOutput   = new QueryOutput(new QueryInput());
+         if(StringUtils.hasContent(id))
+         {
+            GetInput getInput = new GetInput();
+            getInput.setTableName(join.getLeftTable());
+            getInput.setPrimaryKey(id);
+            GetOutput getOutput = new GetAction().execute(getInput);
+            primaryRecord = getOutput.getRecord();
+
+            if(primaryRecord == null)
+            {
+               throw (new QNotFoundException("Could not find " + (leftTable == null ? "" : leftTable.getLabel()) + " with primary key " + id));
+            }
+
+            ////////////////////////////////////////////////////////////////////
+            // set up the query - for the table on the right side of the join //
+            ////////////////////////////////////////////////////////////////////
+            filter = new QQueryFilter();
+            for(JoinOn joinOn : join.getJoinOns())
+            {
+               filter.addCriteria(new QFilterCriteria(joinOn.getRightField(), QCriteriaOperator.EQUALS, List.of(primaryRecord.getValue(joinOn.getLeftField()))));
+            }
+            filter.setOrderBys(join.getOrderBys());
+            filter.setLimit(maxRows);
+
+            QueryInput queryInput = new QueryInput();
+            queryInput.setTableName(join.getRightTable());
+            queryInput.setShouldTranslatePossibleValues(true);
+            queryInput.setShouldGenerateDisplayValues(true);
+            queryInput.setFilter(filter);
+            queryOutput = new QueryAction().execute(queryInput);
+
+            QValueFormatter.setBlobValuesToDownloadUrls(rightTable, queryOutput.getRecords());
+
+            totalRows = queryOutput.getRecords().size();
+            if(maxRows != null && (queryOutput.getRecords().size() == maxRows))
+            {
+               /////////////////////////////////////////////////////////////////////////////////////
+               // if the input said to only do some max, and the # of results we got is that max, //
+               // then do a count query, for displaying 1-n of <count>                            //
+               /////////////////////////////////////////////////////////////////////////////////////
+               CountInput countInput = new CountInput();
+               countInput.setTableName(join.getRightTable());
+               countInput.setFilter(filter);
+               totalRows = new CountAction().execute(countInput).getCount();
+            }
+         }
+
+         String tablePath   = QContext.getQInstance().getTablePath(rightTable.getName());
+         String viewAllLink = tablePath == null ? null : (tablePath + "?filter=" + URLEncoder.encode(JsonUtils.toJson(filter), Charset.defaultCharset()));
+
+         ChildRecordListData widgetData = new ChildRecordListData(widgetLabel, queryOutput, rightTable, tablePath, viewAllLink, totalRows);
+
+         if(BooleanUtils.isTrue(ValueUtils.getValueAsBoolean(input.getQueryParams().get("canAddChildRecord"))))
+         {
+            widgetData.setCanAddChildRecord(true);
+
+            //////////////////////////////////////////////////////////
+            // new child records must have values from the join-ons //
+            //////////////////////////////////////////////////////////
+            Map<String, Serializable> defaultValuesForNewChildRecords = new HashMap<>();
+            if(primaryRecord != null)
+            {
+               for(JoinOn joinOn : join.getJoinOns())
+               {
+                  defaultValuesForNewChildRecords.put(joinOn.getRightField(), primaryRecord.getValue(joinOn.getLeftField()));
+               }
+            }
+
+            widgetData.setDefaultValuesForNewChildRecords(defaultValuesForNewChildRecords);
+
+            Map<String, Serializable> widgetValues = input.getWidgetMetaData().getDefaultValues();
+            if(widgetValues.containsKey("disabledFieldsForNewChildRecords"))
+            {
+               @SuppressWarnings("unchecked")
+               Set<String> disabledFieldsForNewChildRecords = (Set<String>) widgetValues.get("disabledFieldsForNewChildRecords");
+               widgetData.setDisabledFieldsForNewChildRecords(disabledFieldsForNewChildRecords);
+            }
+            else
+            {
+               /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+               // if there are no disabled fields specified - then normally any fields w/ a default value get implicitly disabled //
+               // but - if we didn't look-up the primary record, then we'll want to explicit disable fields from joins            //
+               /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+               if(primaryRecord == null)
+               {
+                  Set<String> implicitlyDisabledFields = new HashSet<>();
+                  widgetData.setDisabledFieldsForNewChildRecords(implicitlyDisabledFields);
+                  for(JoinOn joinOn : join.getJoinOns())
+                  {
+                     implicitlyDisabledFields.add(joinOn.getRightField());
+                  }
+               }
+            }
+         }
+
+         return (new RenderWidgetOutput(widgetData));
+      }
+      catch(Exception e)
+      {
+         LOG.warn("Error rendering child record list", e, logPair("widgetName", () -> input.getWidgetMetaData().getName()));
+         throw (e);
+      }
    }
 
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/dashboard/widgets/DateTimeGroupBy.java

@@ -256,7 +256,6 @@ public enum DateTimeGroupBy
    /*******************************************************************************
     **
     *******************************************************************************/
-   @SuppressWarnings("checkstyle:indentation")
    public Instant roundDown(Instant instant, ZoneId zoneId)
    {
       ZonedDateTime zoned = instant.atZone(zoneId);

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/dashboard/widgets/ProcessAlertWidget.java

@@ -0,0 +1,86 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.actions.dashboard.widgets;
+
+
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.MetaDataProducerInterface;
+import com.kingsrook.qqq.backend.core.model.actions.widgets.RenderWidgetInput;
+import com.kingsrook.qqq.backend.core.model.actions.widgets.RenderWidgetOutput;
+import com.kingsrook.qqq.backend.core.model.dashboard.widgets.AlertData;
+import com.kingsrook.qqq.backend.core.model.dashboard.widgets.WidgetType;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
+import com.kingsrook.qqq.backend.core.model.metadata.dashboard.QWidgetMetaData;
+
+
+/*******************************************************************************
+ ** Widget that can add an Alert to a process screen.
+ **
+ ** In the process, you'll want values:
+ ** - alertType - name of entry in AlertType enum (ERROR, WARNING, SUCCESS)
+ ** - alertHtml - html to display inside the alert (other than its icon)
+ *******************************************************************************/
+public class ProcessAlertWidget extends AbstractWidgetRenderer implements MetaDataProducerInterface<QWidgetMetaData>
+{
+   public static final String NAME = "ProcessAlertWidget";
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public RenderWidgetOutput render(RenderWidgetInput input) throws QException
+   {
+      AlertData.AlertType alertType = AlertData.AlertType.WARNING;
+      if(input.getQueryParams().containsKey("alertType"))
+      {
+         alertType = AlertData.AlertType.valueOf(input.getQueryParams().get("alertType"));
+      }
+
+      String html = "Warning";
+      if(input.getQueryParams().containsKey("alertHtml"))
+      {
+         html = input.getQueryParams().get("alertHtml");
+      }
+
+      return (new RenderWidgetOutput(new AlertData(alertType, html)));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public QWidgetMetaData produce(QInstance qInstance) throws QException
+   {
+      return new QWidgetMetaData()
+         .withType(WidgetType.ALERT.getType())
+         .withGridColumns(12)
+         .withName(NAME)
+         .withIsCard(false)
+         .withShowReloadButton(false)
+         .withCodeReference(new QCodeReference(getClass()));
+   }
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/dashboard/widgets/ProcessWidgetRenderer.java

@@ -24,6 +24,7 @@ package com.kingsrook.qqq.backend.core.actions.dashboard.widgets;
 
 import java.util.HashMap;
 import com.kingsrook.qqq.backend.core.actions.ActionHelper;
+import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.model.actions.widgets.RenderWidgetInput;
 import com.kingsrook.qqq.backend.core.model.actions.widgets.RenderWidgetOutput;
@@ -57,7 +58,7 @@ public class ProcessWidgetRenderer extends AbstractWidgetRenderer
             setupDropdowns(input, widgetMetaData, data);
 
             String           processName     = (String) widgetMetaData.getDefaultValues().get(WIDGET_PROCESS_NAME);
-            QProcessMetaData processMetaData = input.getInstance().getProcess(processName);
+            QProcessMetaData processMetaData = QContext.getQInstance().getProcess(processName);
             data.setProcessMetaData(processMetaData);
 
             data.setDefaultValues(new HashMap<>(input.getQueryParams()));

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/interfaces/QStorageInterface.java

@@ -0,0 +1,69 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.actions.interfaces;
+
+
+import java.io.InputStream;
+import java.io.OutputStream;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.actions.tables.storage.StorageInput;
+
+
+/*******************************************************************************
+ ** Interface for actions that a backend can perform, based on streaming data
+ ** into the backend's storage.
+ *******************************************************************************/
+public interface QStorageInterface
+{
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   OutputStream createOutputStream(StorageInput storageInput) throws QException;
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   InputStream getInputStream(StorageInput storageInput) throws QException;
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   default void makePublic(StorageInput storageInput) throws QException
+   {
+      //////////
+      // noop //
+      //////////
+   }
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   default String getDownloadURL(StorageInput storageInput) throws QException
+   {
+      return (null);
+   }
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/messaging/SendMessageAction.java

@@ -0,0 +1,64 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.actions.messaging;
+
+
+import com.kingsrook.qqq.backend.core.actions.AbstractQActionFunction;
+import com.kingsrook.qqq.backend.core.context.QContext;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.actions.messaging.SendMessageInput;
+import com.kingsrook.qqq.backend.core.model.actions.messaging.SendMessageOutput;
+import com.kingsrook.qqq.backend.core.model.metadata.messaging.QMessagingProviderMetaData;
+import com.kingsrook.qqq.backend.core.modules.messaging.MessagingProviderInterface;
+import com.kingsrook.qqq.backend.core.modules.messaging.QMessagingProviderDispatcher;
+import com.kingsrook.qqq.backend.core.utils.StringUtils;
+
+
+/*******************************************************************************
+ **
+ *******************************************************************************/
+public class SendMessageAction extends AbstractQActionFunction<SendMessageInput, SendMessageOutput>
+{
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public SendMessageOutput execute(SendMessageInput input) throws QException
+   {
+      if(!StringUtils.hasContent(input.getMessagingProviderName()))
+      {
+         throw (new QException("Messaging provider name was not given in SendMessageInput."));
+      }
+
+      QMessagingProviderMetaData messagingProvider = QContext.getQInstance().getMessagingProvider(input.getMessagingProviderName());
+      if(messagingProvider == null)
+      {
+         throw (new QException("Messaging provider named [" + input.getMessagingProviderName() + "] was not found in this QInstance."));
+      }
+
+      MessagingProviderInterface messagingProviderInterface = new QMessagingProviderDispatcher().getMessagingProviderInterface(messagingProvider.getType());
+
+      return (messagingProviderInterface.sendMessage(input));
+   }
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/metadata/JoinGraph.java

@@ -29,6 +29,7 @@ import java.util.List;
 import java.util.Objects;
 import java.util.Set;
 import java.util.TreeSet;
+import com.kingsrook.qqq.backend.core.instances.QMetaDataVariableInterpreter;
 import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
 import com.kingsrook.qqq.backend.core.model.metadata.joins.QJoinMetaData;
 import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
@@ -41,9 +42,19 @@ import com.kingsrook.qqq.backend.core.utils.StringUtils;
  *******************************************************************************/
 public class JoinGraph
 {
-
    private Set<Edge> edges = new HashSet<>();
 
+   ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+   // as an instance grows, with the number of joins (say, more than 50?), especially as they may have a lot of connections, //
+   // it can become very very slow to process a full join graph (e.g., 10 seconds, maybe much worse, per Big-O...)           //
+   // also, it's not frequently useful to look at a join path that's more than a handful of tables long.                     //
+   // thus - this property exists - to limit the max length of a join path.  Keeping it small keeps instance enrichment      //
+   // and validation reasonably performant, at the possible cost of, some join-path that's longer than this limit may not    //
+   // be found - but - chances are, you don't want some 12-element join path to be used anyway, thus, this makes sense.      //
+   // but - it can be adjusted, per system property or ENV var.                                                              //
+   ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+   private int maxPathLength = new QMetaDataVariableInterpreter().getIntegerFromPropertyOrEnvironment("qqq.instance.joinGraph.maxPathLength", "QQQ_INSTANCE_JOIN_GRAPH_MAX_PATH_LENGTH", 3);
+
 
 
    /*******************************************************************************
@@ -303,6 +314,13 @@ public class JoinGraph
 
                if(otherTableName != null)
                {
+                  if(newPath.size() > maxPathLength)
+                  {
+                     ////////////////////////////////////////////////////////////////
+                     // performance hack.  see comment at maxPathLength definition //
+                     ////////////////////////////////////////////////////////////////
+                     continue;
+                  }
 
                   JoinConnectionList newConnectionList = connectionList.copy();
                   JoinConnection     joinConnection    = new JoinConnection(otherTableName, edge.joinName);

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/metadata/MetaDataAction.java

@@ -30,6 +30,7 @@ import java.util.Map;
 import com.kingsrook.qqq.backend.core.actions.ActionHelper;
 import com.kingsrook.qqq.backend.core.actions.permissions.PermissionCheckResult;
 import com.kingsrook.qqq.backend.core.actions.permissions.PermissionsHelper;
+import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.model.actions.metadata.MetaDataInput;
 import com.kingsrook.qqq.backend.core.model.actions.metadata.MetaDataOutput;
@@ -72,7 +73,7 @@ public class MetaDataAction
       // map tables to frontend metadata //
       /////////////////////////////////////
       Map<String, QFrontendTableMetaData> tables = new LinkedHashMap<>();
-      for(Map.Entry<String, QTableMetaData> entry : metaDataInput.getInstance().getTables().entrySet())
+      for(Map.Entry<String, QTableMetaData> entry : QContext.getQInstance().getTables().entrySet())
       {
          String         tableName = entry.getKey();
          QTableMetaData table     = entry.getValue();
@@ -83,7 +84,7 @@ public class MetaDataAction
             continue;
          }
 
-         QBackendMetaData backendForTable = metaDataInput.getInstance().getBackendForTable(tableName);
+         QBackendMetaData backendForTable = QContext.getQInstance().getBackendForTable(tableName);
          tables.put(tableName, new QFrontendTableMetaData(metaDataInput, backendForTable, table, false, false));
          treeNodes.put(tableName, new AppTreeNode(table));
       }
@@ -96,7 +97,7 @@ public class MetaDataAction
       // map processes to frontend metadata //
       ////////////////////////////////////////
       Map<String, QFrontendProcessMetaData> processes = new LinkedHashMap<>();
-      for(Map.Entry<String, QProcessMetaData> entry : metaDataInput.getInstance().getProcesses().entrySet())
+      for(Map.Entry<String, QProcessMetaData> entry : QContext.getQInstance().getProcesses().entrySet())
       {
          String           processName = entry.getKey();
          QProcessMetaData process     = entry.getValue();
@@ -116,7 +117,7 @@ public class MetaDataAction
       // map reports to frontend metadata //
       //////////////////////////////////////
       Map<String, QFrontendReportMetaData> reports = new LinkedHashMap<>();
-      for(Map.Entry<String, QReportMetaData> entry : metaDataInput.getInstance().getReports().entrySet())
+      for(Map.Entry<String, QReportMetaData> entry : QContext.getQInstance().getReports().entrySet())
       {
          String          reportName = entry.getKey();
          QReportMetaData report     = entry.getValue();
@@ -136,7 +137,7 @@ public class MetaDataAction
       // map widgets to frontend metadata //
       //////////////////////////////////////
       Map<String, QFrontendWidgetMetaData> widgets = new LinkedHashMap<>();
-      for(Map.Entry<String, QWidgetMetaDataInterface> entry : metaDataInput.getInstance().getWidgets().entrySet())
+      for(Map.Entry<String, QWidgetMetaDataInterface> entry : QContext.getQInstance().getWidgets().entrySet())
       {
          String                   widgetName = entry.getKey();
          QWidgetMetaDataInterface widget     = entry.getValue();
@@ -154,7 +155,7 @@ public class MetaDataAction
       ///////////////////////////////////////////////////////
       // sort apps - by sortOrder (integer), then by label //
       ///////////////////////////////////////////////////////
-      List<QAppMetaData> sortedApps = metaDataInput.getInstance().getApps().values().stream()
+      List<QAppMetaData> sortedApps = QContext.getQInstance().getApps().values().stream()
          .sorted(Comparator.comparing((QAppMetaData a) -> a.getSortOrder())
             .thenComparing((QAppMetaData a) -> a.getLabel()))
          .toList();
@@ -211,12 +212,14 @@ public class MetaDataAction
       ////////////////////////////////////
       // add branding metadata if found //
       ////////////////////////////////////
-      if(metaDataInput.getInstance().getBranding() != null)
+      if(QContext.getQInstance().getBranding() != null)
       {
-         metaDataOutput.setBranding(metaDataInput.getInstance().getBranding());
+         metaDataOutput.setBranding(QContext.getQInstance().getBranding());
       }
 
-      metaDataOutput.setEnvironmentValues(metaDataInput.getInstance().getEnvironmentValues());
+      metaDataOutput.setEnvironmentValues(QContext.getQInstance().getEnvironmentValues());
+
+      metaDataOutput.setHelpContents(QContext.getQInstance().getHelpContent());
 
       // todo post-customization - can do whatever w/ the result if you want?
 

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/metadata/ProcessMetaDataAction.java

@@ -23,6 +23,7 @@ package com.kingsrook.qqq.backend.core.actions.metadata;
 
 
 import com.kingsrook.qqq.backend.core.actions.ActionHelper;
+import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.exceptions.QNotFoundException;
 import com.kingsrook.qqq.backend.core.model.actions.metadata.ProcessMetaDataInput;
@@ -47,7 +48,7 @@ public class ProcessMetaDataAction
       // todo pre-customization - just get to modify the request?
       ProcessMetaDataOutput processMetaDataOutput = new ProcessMetaDataOutput();
 
-      QProcessMetaData process = processMetaDataInput.getInstance().getProcess(processMetaDataInput.getProcessName());
+      QProcessMetaData process = QContext.getQInstance().getProcess(processMetaDataInput.getProcessName());
       if(process == null)
       {
          throw (new QNotFoundException("Process [" + processMetaDataInput.getProcessName() + "] was not found."));

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/metadata/TableMetaDataAction.java

@@ -23,6 +23,7 @@ package com.kingsrook.qqq.backend.core.actions.metadata;
 
 
 import com.kingsrook.qqq.backend.core.actions.ActionHelper;
+import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.exceptions.QNotFoundException;
 import com.kingsrook.qqq.backend.core.model.actions.metadata.TableMetaDataInput;
@@ -48,12 +49,12 @@ public class TableMetaDataAction
       // todo pre-customization - just get to modify the request?
       TableMetaDataOutput tableMetaDataOutput = new TableMetaDataOutput();
 
-      QTableMetaData table = tableMetaDataInput.getInstance().getTable(tableMetaDataInput.getTableName());
+      QTableMetaData table = QContext.getQInstance().getTable(tableMetaDataInput.getTableName());
       if(table == null)
       {
          throw (new QNotFoundException("Table [" + tableMetaDataInput.getTableName() + "] was not found."));
       }
-      QBackendMetaData backendForTable = tableMetaDataInput.getInstance().getBackendForTable(table.getName());
+      QBackendMetaData backendForTable = QContext.getQInstance().getBackendForTable(table.getName());
       tableMetaDataOutput.setTable(new QFrontendTableMetaData(tableMetaDataInput, backendForTable, table, true, true));
 
       // todo post-customization - can do whatever w/ the result if you want

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/permissions/PermissionsHelper.java

@@ -34,18 +34,22 @@ import com.kingsrook.qqq.backend.core.exceptions.QPermissionDeniedException;
 import com.kingsrook.qqq.backend.core.logging.QLogger;
 import com.kingsrook.qqq.backend.core.model.actions.AbstractActionInput;
 import com.kingsrook.qqq.backend.core.model.actions.AbstractTableActionInput;
+import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
 import com.kingsrook.qqq.backend.core.model.metadata.dashboard.QWidgetMetaDataInterface;
 import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.permissions.DenyBehavior;
 import com.kingsrook.qqq.backend.core.model.metadata.permissions.MetaDataWithName;
 import com.kingsrook.qqq.backend.core.model.metadata.permissions.MetaDataWithPermissionRules;
+import com.kingsrook.qqq.backend.core.model.metadata.permissions.PermissionLevel;
 import com.kingsrook.qqq.backend.core.model.metadata.permissions.QPermissionRules;
 import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.Capability;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 import com.kingsrook.qqq.backend.core.model.session.QSession;
 import com.kingsrook.qqq.backend.core.utils.StringUtils;
+import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
 
 
 /*******************************************************************************
@@ -75,6 +79,12 @@ public class PermissionsHelper
       warnAboutPermissionSubTypeForTables(permissionSubType);
       QTableMetaData table = QContext.getQInstance().getTable(tableName);
 
+      if(table == null)
+      {
+         LOG.info("Throwing a permission denied exception in response to a non-existent table name", logPair("tableName", tableName));
+         throw (new QPermissionDeniedException("Permission denied."));
+      }
+
       commonCheckPermissionThrowing(getEffectivePermissionRules(table, QContext.getQInstance()), permissionSubType, table.getName());
    }
 
@@ -181,7 +191,14 @@ public class PermissionsHelper
     *******************************************************************************/
    public static void checkProcessPermissionThrowing(AbstractActionInput actionInput, String processName, Map<String, Serializable> processValues) throws QPermissionDeniedException
    {
-      QProcessMetaData process                  = QContext.getQInstance().getProcess(processName);
+      QProcessMetaData process = QContext.getQInstance().getProcess(processName);
+
+      if(process == null)
+      {
+         LOG.info("Throwing a permission denied exception in response to a non-existent process name", logPair("processName", processName));
+         throw (new QPermissionDeniedException("Permission denied."));
+      }
+
       QPermissionRules effectivePermissionRules = getEffectivePermissionRules(process, QContext.getQInstance());
 
       if(effectivePermissionRules.getCustomPermissionChecker() != null)
@@ -223,6 +240,13 @@ public class PermissionsHelper
    public static void checkAppPermissionThrowing(AbstractActionInput actionInput, String appName) throws QPermissionDeniedException
    {
       QAppMetaData app = QContext.getQInstance().getApp(appName);
+
+      if(app == null)
+      {
+         LOG.info("Throwing a permission denied exception in response to a non-existent app name", logPair("appName", appName));
+         throw (new QPermissionDeniedException("Permission denied."));
+      }
+
       commonCheckPermissionThrowing(getEffectivePermissionRules(app, QContext.getQInstance()), PrivatePermissionSubType.HAS_ACCESS, app.getName());
    }
 
@@ -252,6 +276,13 @@ public class PermissionsHelper
    public static void checkReportPermissionThrowing(AbstractActionInput actionInput, String reportName) throws QPermissionDeniedException
    {
       QReportMetaData report = QContext.getQInstance().getReport(reportName);
+
+      if(report == null)
+      {
+         LOG.info("Throwing a permission denied exception in response to a non-existent process name", logPair("reportName", reportName));
+         throw (new QPermissionDeniedException("Permission denied."));
+      }
+
       commonCheckPermissionThrowing(getEffectivePermissionRules(report, QContext.getQInstance()), PrivatePermissionSubType.HAS_ACCESS, report.getName());
    }
 
@@ -281,6 +312,13 @@ public class PermissionsHelper
    public static void checkWidgetPermissionThrowing(AbstractActionInput actionInput, String widgetName) throws QPermissionDeniedException
    {
       QWidgetMetaDataInterface widget = QContext.getQInstance().getWidget(widgetName);
+
+      if(widget == null)
+      {
+         LOG.info("Throwing a permission denied exception in response to a non-existent widget name", logPair("widgetName", widgetName));
+         throw (new QPermissionDeniedException("Permission denied."));
+      }
+
       commonCheckPermissionThrowing(getEffectivePermissionRules(widget, QContext.getQInstance()), PrivatePermissionSubType.HAS_ACCESS, widget.getName());
    }
 
@@ -333,9 +371,25 @@ public class PermissionsHelper
          QPermissionRules rules    = getEffectivePermissionRules(tableMetaData, instance);
          String           baseName = getEffectivePermissionBaseName(rules, tableMetaData.getName());
 
-         for(TablePermissionSubType permissionSubType : TablePermissionSubType.values())
+         QBackendMetaData backend = instance.getBackend(tableMetaData.getBackendName());
+         if(tableMetaData.isCapabilityEnabled(backend, Capability.TABLE_INSERT))
          {
-            addEffectiveAvailablePermission(rules, permissionSubType, rs, baseName, tableMetaData, "Table");
+            addEffectiveAvailablePermission(rules, TablePermissionSubType.INSERT, rs, baseName, tableMetaData, "Table");
+         }
+
+         if(tableMetaData.isCapabilityEnabled(backend, Capability.TABLE_UPDATE))
+         {
+            addEffectiveAvailablePermission(rules, TablePermissionSubType.EDIT, rs, baseName, tableMetaData, "Table");
+         }
+
+         if(tableMetaData.isCapabilityEnabled(backend, Capability.TABLE_DELETE))
+         {
+            addEffectiveAvailablePermission(rules, TablePermissionSubType.DELETE, rs, baseName, tableMetaData, "Table");
+         }
+
+         if(tableMetaData.isCapabilityEnabled(backend, Capability.TABLE_QUERY) || tableMetaData.isCapabilityEnabled(backend, Capability.TABLE_GET))
+         {
+            addEffectiveAvailablePermission(rules, TablePermissionSubType.READ, rs, baseName, tableMetaData, "Table");
          }
       }
 
@@ -369,7 +423,10 @@ public class PermissionsHelper
       {
          QPermissionRules rules    = getEffectivePermissionRules(widgetMetaData, instance);
          String           baseName = getEffectivePermissionBaseName(rules, widgetMetaData.getName());
-         addEffectiveAvailablePermission(rules, PrivatePermissionSubType.HAS_ACCESS, rs, baseName, widgetMetaData, "Widget");
+         if(!rules.getLevel().equals(PermissionLevel.NOT_PROTECTED))
+         {
+            addEffectiveAvailablePermission(rules, PrivatePermissionSubType.HAS_ACCESS, rs, baseName, widgetMetaData, "Widget");
+         }
       }
 
       return (rs);
@@ -478,7 +535,6 @@ public class PermissionsHelper
    /*******************************************************************************
     **
     *******************************************************************************/
-   @SuppressWarnings("checkstyle:indentation")
    static PermissionSubType getEffectivePermissionSubType(QPermissionRules rules, PermissionSubType originalPermissionSubType)
    {
       if(rules == null || rules.getLevel() == null)
@@ -493,10 +549,10 @@ public class PermissionsHelper
       if(PrivatePermissionSubType.HAS_ACCESS.equals(originalPermissionSubType))
       {
          return switch(rules.getLevel())
-            {
-               case NOT_PROTECTED -> null;
-               default -> PrivatePermissionSubType.HAS_ACCESS;
-            };
+         {
+            case NOT_PROTECTED -> null;
+            default -> PrivatePermissionSubType.HAS_ACCESS;
+         };
       }
       else
       {
@@ -505,30 +561,30 @@ public class PermissionsHelper
          // permission sub-type to what we expect to be set for the table                                      //
          ////////////////////////////////////////////////////////////////////////////////////////////////////////
          return switch(rules.getLevel())
+         {
+            case NOT_PROTECTED -> null;
+            case HAS_ACCESS_PERMISSION -> PrivatePermissionSubType.HAS_ACCESS;
+            case READ_WRITE_PERMISSIONS ->
             {
-               case NOT_PROTECTED -> null;
-               case HAS_ACCESS_PERMISSION -> PrivatePermissionSubType.HAS_ACCESS;
-               case READ_WRITE_PERMISSIONS ->
+               if(PrivatePermissionSubType.READ.equals(originalPermissionSubType) || PrivatePermissionSubType.WRITE.equals(originalPermissionSubType))
                {
-                  if(PrivatePermissionSubType.READ.equals(originalPermissionSubType) || PrivatePermissionSubType.WRITE.equals(originalPermissionSubType))
-                  {
-                     yield (originalPermissionSubType);
-                  }
-                  else if(TablePermissionSubType.INSERT.equals(originalPermissionSubType) || TablePermissionSubType.EDIT.equals(originalPermissionSubType) || TablePermissionSubType.DELETE.equals(originalPermissionSubType))
-                  {
-                     yield (PrivatePermissionSubType.WRITE);
-                  }
-                  else if(TablePermissionSubType.READ.equals(originalPermissionSubType))
-                  {
-                     yield (PrivatePermissionSubType.READ);
-                  }
-                  else
-                  {
-                     throw new IllegalStateException("Unexpected permissionSubType: " + originalPermissionSubType);
-                  }
+                  yield (originalPermissionSubType);
                }
-               case READ_INSERT_EDIT_DELETE_PERMISSIONS -> originalPermissionSubType;
-            };
+               else if(TablePermissionSubType.INSERT.equals(originalPermissionSubType) || TablePermissionSubType.EDIT.equals(originalPermissionSubType) || TablePermissionSubType.DELETE.equals(originalPermissionSubType))
+               {
+                  yield (PrivatePermissionSubType.WRITE);
+               }
+               else if(TablePermissionSubType.READ.equals(originalPermissionSubType))
+               {
+                  yield (PrivatePermissionSubType.READ);
+               }
+               else
+               {
+                  throw new IllegalStateException("Unexpected permissionSubType: " + originalPermissionSubType);
+               }
+            }
+            case READ_INSERT_EDIT_DELETE_PERMISSIONS -> originalPermissionSubType;
+         };
       }
    }
 

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/processes/CancelProcessAction.java

@@ -0,0 +1,111 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.actions.processes;
+
+
+import java.util.Optional;
+import java.util.UUID;
+import com.kingsrook.qqq.backend.core.actions.ActionHelper;
+import com.kingsrook.qqq.backend.core.context.QContext;
+import com.kingsrook.qqq.backend.core.exceptions.QBadRequestException;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.logging.QLogger;
+import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessState;
+import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput;
+import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessOutput;
+import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
+import com.kingsrook.qqq.backend.core.state.StateType;
+import com.kingsrook.qqq.backend.core.state.UUIDAndTypeStateKey;
+import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
+
+
+/*******************************************************************************
+ ** Action handler for running the cancel step of a qqq process
+ *
+ *******************************************************************************/
+public class CancelProcessAction extends RunProcessAction
+{
+   private static final QLogger LOG = QLogger.getLogger(CancelProcessAction.class);
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public RunProcessOutput execute(RunProcessInput runProcessInput) throws QException
+   {
+      ActionHelper.validateSession(runProcessInput);
+
+      QProcessMetaData process = QContext.getQInstance().getProcess(runProcessInput.getProcessName());
+      if(process == null)
+      {
+         throw new QBadRequestException("Process [" + runProcessInput.getProcessName() + "] is not defined in this instance.");
+      }
+
+      if(runProcessInput.getProcessUUID() == null)
+      {
+         throw (new QBadRequestException("Cannot cancel process - processUUID was not given."));
+      }
+
+      UUIDAndTypeStateKey    stateKey     = new UUIDAndTypeStateKey(UUID.fromString(runProcessInput.getProcessUUID()), StateType.PROCESS_STATUS);
+      Optional<ProcessState> processState = getState(runProcessInput.getProcessUUID());
+      if(processState.isEmpty())
+      {
+         throw (new QBadRequestException("Cannot cancel process - State for process UUID [" + runProcessInput.getProcessUUID() + "] was not found."));
+      }
+
+      RunProcessOutput runProcessOutput = new RunProcessOutput();
+      try
+      {
+         if(process.getCancelStep() != null)
+         {
+            LOG.info("Running cancel step for process", logPair("processName", process.getName()));
+            runBackendStep(runProcessInput, process, runProcessOutput, stateKey, process.getCancelStep(), process, processState.get());
+         }
+         else
+         {
+            LOG.debug("Process does not have a custom cancel step to run.", logPair("processName", process.getName()));
+         }
+      }
+      catch(QException qe)
+      {
+         ////////////////////////////////////////////////////////////
+         // upon exception (e.g., one thrown by a step), throw it. //
+         ////////////////////////////////////////////////////////////
+         throw (qe);
+      }
+      catch(Exception e)
+      {
+         throw (new QException("Error cancelling process", e));
+      }
+      finally
+      {
+         //////////////////////////////////////////////////////
+         // always put the final state in the process result //
+         //////////////////////////////////////////////////////
+         runProcessOutput.setProcessState(processState.get());
+      }
+
+      return (runProcessOutput);
+   }
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/processes/QProcessCallbackFactory.java

@@ -23,11 +23,19 @@ package com.kingsrook.qqq.backend.core.actions.processes;
 
 
 import java.io.Serializable;
+import java.util.Collection;
 import java.util.Collections;
 import java.util.List;
 import java.util.Map;
+import com.kingsrook.qqq.backend.core.context.QContext;
+import com.kingsrook.qqq.backend.core.exceptions.QRuntimeException;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.data.QRecordEntity;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
+import com.kingsrook.qqq.backend.core.utils.StringUtils;
 
 
 /*******************************************************************************
@@ -65,4 +73,56 @@ public class QProcessCallbackFactory
       };
    }
 
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public static QProcessCallback forRecordEntity(QRecordEntity entity)
+   {
+      return forRecord(entity.toQRecord());
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public static QProcessCallback forRecord(QRecord record)
+   {
+      String primaryKeyField = "id";
+      if(StringUtils.hasContent(record.getTableName()))
+      {
+         primaryKeyField = QContext.getQInstance().getTable(record.getTableName()).getPrimaryKeyField();
+      }
+
+      Serializable primaryKeyValue = record.getValue(primaryKeyField);
+      if(primaryKeyValue == null)
+      {
+         throw (new QRuntimeException("Record did not have value in its primary key field [" + primaryKeyField + "]"));
+      }
+
+      return (forPrimaryKey(primaryKeyField, primaryKeyValue));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public static QProcessCallback forPrimaryKey(String fieldName, Serializable value)
+   {
+      return (forFilter(new QQueryFilter().withCriteria(new QFilterCriteria(fieldName, QCriteriaOperator.EQUALS, value))));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public static QProcessCallback forPrimaryKeys(String fieldName, Collection<? extends Serializable> values)
+   {
+      return (forFilter(new QQueryFilter().withCriteria(new QFilterCriteria(fieldName, QCriteriaOperator.IN, values))));
+   }
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/processes/RunBackendStepAction.java

@@ -26,14 +26,17 @@ import java.io.Serializable;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
+import java.util.Objects;
 import java.util.stream.Collectors;
 import com.kingsrook.qqq.backend.core.actions.ActionHelper;
 import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
+import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.exceptions.QUserFacingException;
 import com.kingsrook.qqq.backend.core.logging.QLogger;
 import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
 import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput;
 import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
@@ -62,7 +65,7 @@ public class RunBackendStepAction
    {
       ActionHelper.validateSession(runBackendStepInput);
 
-      QProcessMetaData process = runBackendStepInput.getInstance().getProcess(runBackendStepInput.getProcessName());
+      QProcessMetaData process = QContext.getQInstance().getProcess(runBackendStepInput.getProcessName());
       if(process == null)
       {
          throw new QException("Process [" + runBackendStepInput.getProcessName() + "] is not defined in this instance.");
@@ -71,7 +74,17 @@ public class RunBackendStepAction
       QStepMetaData stepMetaData = process.getStep(runBackendStepInput.getStepName());
       if(stepMetaData == null)
       {
-         throw new QException("Step [" + runBackendStepInput.getStepName() + "] is not defined in the process [" + process.getName() + "]");
+         if(process.getCancelStep() != null && Objects.equals(process.getCancelStep().getName(), runBackendStepInput.getStepName()))
+         {
+            /////////////////////////////////////
+            // special case for cancel step... //
+            /////////////////////////////////////
+            stepMetaData = process.getCancelStep();
+         }
+         else
+         {
+            throw new QException("Step [" + runBackendStepInput.getStepName() + "] is not defined in the process [" + process.getName() + "]");
+         }
       }
 
       if(!(stepMetaData instanceof QBackendStepMetaData backendStepMetaData))
@@ -82,7 +95,7 @@ public class RunBackendStepAction
       //////////////////////////////////////////////////////////////////////////////////////
       // ensure input data is set as needed - use callback object to get anything missing //
       //////////////////////////////////////////////////////////////////////////////////////
-      ensureRecordsAreInRequest(runBackendStepInput, backendStepMetaData);
+      ensureRecordsAreInRequest(runBackendStepInput, backendStepMetaData, process);
       ensureInputFieldsAreInRequest(runBackendStepInput, backendStepMetaData);
 
       ////////////////////////////////////////////////////////////////////
@@ -167,7 +180,7 @@ public class RunBackendStepAction
     ** check if this step uses a record list - and if so, if we need to get one
     ** via the callback
     *******************************************************************************/
-   private void ensureRecordsAreInRequest(RunBackendStepInput runBackendStepInput, QBackendStepMetaData step) throws QException
+   private void ensureRecordsAreInRequest(RunBackendStepInput runBackendStepInput, QBackendStepMetaData step, QProcessMetaData process) throws QException
    {
       QFunctionInputMetaData inputMetaData = step.getInputMetaData();
       if(inputMetaData != null && inputMetaData.getRecordListMetaData() != null)
@@ -190,9 +203,44 @@ public class RunBackendStepAction
 
             queryInput.setFilter(callback.getQueryFilter());
 
+            //////////////////////////////////////////////////////////////////////////////////////////
+            // if process has a max-no of records, set a limit on the process of that number plus 1 //
+            // (the plus 1 being so we can see "oh, you selected more than that many; error!"       //
+            //////////////////////////////////////////////////////////////////////////////////////////
+            if(process.getMaxInputRecords() != null)
+            {
+               if(callback.getQueryFilter() == null)
+               {
+                  queryInput.setFilter(new QQueryFilter());
+               }
+
+               queryInput.getFilter().setLimit(process.getMaxInputRecords() + 1);
+            }
+
             QueryOutput queryOutput = new QueryAction().execute(queryInput);
             runBackendStepInput.setRecords(queryOutput.getRecords());
-            // todo - handle 0 results found?
+
+            ////////////////////////////////////////////////////////////////////////////////
+            // if process defines a max, and more than the max were found, throw an error //
+            ////////////////////////////////////////////////////////////////////////////////
+            if(process.getMaxInputRecords() != null)
+            {
+               if(queryOutput.getRecords().size() > process.getMaxInputRecords())
+               {
+                  throw (new QUserFacingException("Too many records were selected for this process.  At most, only " + process.getMaxInputRecords() + " can be selected."));
+               }
+            }
+
+            /////////////////////////////////////////////////////////////////////////////////
+            // if process defines a min, and fewer than the min were found, throw an error //
+            /////////////////////////////////////////////////////////////////////////////////
+            if(process.getMinInputRecords() != null)
+            {
+               if(queryOutput.getRecords().size() < process.getMinInputRecords())
+               {
+                  throw (new QUserFacingException("Too few records were selected for this process.  At least " + process.getMinInputRecords() + " must be selected."));
+               }
+            }
          }
       }
    }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/processes/RunProcessAction.java

@@ -82,7 +82,7 @@ public class RunProcessAction
    public static final String BASEPULL_THIS_RUNTIME_KEY = "basepullThisRuntimeKey";
    public static final String BASEPULL_LAST_RUNTIME_KEY = "basepullLastRuntimeKey";
    public static final String BASEPULL_TIMESTAMP_FIELD  = "basepullTimestampField";
-   public static final String BASEPULL_CONFIGURATION = "basepullConfiguration";
+   public static final String BASEPULL_CONFIGURATION    = "basepullConfiguration";
 
    ////////////////////////////////////////////////////////////////////////////////////////////////
    // indicator that the timestamp field should be updated - e.g., the execute step is finished. //
@@ -99,7 +99,7 @@ public class RunProcessAction
    {
       ActionHelper.validateSession(runProcessInput);
 
-      QProcessMetaData process = runProcessInput.getInstance().getProcess(runProcessInput.getProcessName());
+      QProcessMetaData process = QContext.getQInstance().getProcess(runProcessInput.getProcessName());
       if(process == null)
       {
          throw new QException("Process [" + runProcessInput.getProcessName() + "] is not defined in this instance.");
@@ -190,7 +190,25 @@ public class RunProcessAction
                // Run backend steps //
                ///////////////////////
                LOG.debug("Running backend step [" + step.getName() + "] in process [" + process.getName() + "]");
-               runBackendStep(runProcessInput, process, runProcessOutput, stateKey, backendStepMetaData, process, processState);
+               RunBackendStepOutput runBackendStepOutput = runBackendStep(runProcessInput, process, runProcessOutput, stateKey, backendStepMetaData, process, processState);
+
+               /////////////////////////////////////////////////////////////////////////////////////////
+               // if the step returned an override lastStepName, use that to determine how we proceed //
+               /////////////////////////////////////////////////////////////////////////////////////////
+               if(runBackendStepOutput.getOverrideLastStepName() != null)
+               {
+                  LOG.debug("Process step [" + lastStepName + "] returned an overrideLastStepName [" + runBackendStepOutput.getOverrideLastStepName() + "]!");
+                  lastStepName = runBackendStepOutput.getOverrideLastStepName();
+               }
+
+               /////////////////////////////////////////////////////////////////////////////////////////////
+               // similarly, if the step produced an updatedFrontendStepList, propagate that data outward //
+               /////////////////////////////////////////////////////////////////////////////////////////////
+               if(runBackendStepOutput.getUpdatedFrontendStepList() != null)
+               {
+                  LOG.debug("Process step [" + lastStepName + "] generated an updatedFrontendStepList [" + runBackendStepOutput.getUpdatedFrontendStepList().stream().map(s -> s.getName()).toList() + "]!");
+                  runProcessOutput.setUpdatedFrontendStepList(runBackendStepOutput.getUpdatedFrontendStepList());
+               }
             }
             else
             {
@@ -317,6 +335,13 @@ public class RunProcessAction
          ///////////////////////////////////////////////////
          runProcessInput.seedFromProcessState(optionalProcessState.get());
 
+         ///////////////////////////////////////////////////////////////////////////////////////////////////
+         // if we're restoring an old state, we can discard a previously stored updatedFrontendStepList - //
+         // it is only needed on the transitional edge from a backend-step to a frontend step, but not    //
+         // in the other directly                                                                         //
+         ///////////////////////////////////////////////////////////////////////////////////////////////////
+         optionalProcessState.get().setUpdatedFrontendStepList(null);
+
          ///////////////////////////////////////////////////////////////////////////
          // if there were values from the caller, put those (back) in the request //
          ///////////////////////////////////////////////////////////////////////////
@@ -339,7 +364,7 @@ public class RunProcessAction
    /*******************************************************************************
     ** Run a single backend step.
     *******************************************************************************/
-   private void runBackendStep(RunProcessInput runProcessInput, QProcessMetaData process, RunProcessOutput runProcessOutput, UUIDAndTypeStateKey stateKey, QBackendStepMetaData backendStep, QProcessMetaData qProcessMetaData, ProcessState processState) throws Exception
+   protected RunBackendStepOutput runBackendStep(RunProcessInput runProcessInput, QProcessMetaData process, RunProcessOutput runProcessOutput, UUIDAndTypeStateKey stateKey, QBackendStepMetaData backendStep, QProcessMetaData qProcessMetaData, ProcessState processState) throws Exception
    {
       RunBackendStepInput runBackendStepInput = new RunBackendStepInput(processState);
       runBackendStepInput.setProcessName(process.getName());
@@ -368,14 +393,16 @@ public class RunProcessAction
          runBackendStepInput.setBasepullLastRunTime((Instant) runProcessInput.getValues().get(BASEPULL_LAST_RUNTIME_KEY));
       }
 
-      RunBackendStepOutput lastFunctionResult = new RunBackendStepAction().execute(runBackendStepInput);
-      storeState(stateKey, lastFunctionResult.getProcessState());
+      RunBackendStepOutput runBackendStepOutput = new RunBackendStepAction().execute(runBackendStepInput);
+      storeState(stateKey, runBackendStepOutput.getProcessState());
 
-      if(lastFunctionResult.getException() != null)
+      if(runBackendStepOutput.getException() != null)
       {
-         runProcessOutput.setException(lastFunctionResult.getException());
-         throw (lastFunctionResult.getException());
+         runProcessOutput.setException(runBackendStepOutput.getException());
+         throw (runBackendStepOutput.getException());
       }
+
+      return (runBackendStepOutput);
    }
 
 
@@ -495,15 +522,15 @@ public class RunProcessAction
       String basepullKeyValue = (basepullConfiguration.getKeyValue() != null) ? basepullConfiguration.getKeyValue() : process.getName();
 
       //////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-      // if backend specifies that it uses variants, look for that data in the session and append to our basepull key //
+      // if process specifies that it uses variants, look for that data in the session and append to our basepull key //
       //////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-      if(process.getSchedule() != null && process.getSchedule().getVariantBackend() != null)
+      if(process.getVariantBackend() != null)
       {
          QSession         session         = QContext.getQSession();
-         QBackendMetaData backendMetaData = QContext.getQInstance().getBackend(process.getSchedule().getVariantBackend());
+         QBackendMetaData backendMetaData = QContext.getQInstance().getBackend(process.getVariantBackend());
          if(session.getBackendVariants() == null || !session.getBackendVariants().containsKey(backendMetaData.getVariantOptionsTableTypeValue()))
          {
-            LOG.info("Could not find Backend Variant information for Backend '" + backendMetaData.getName() + "'");
+            LOG.warn("Could not find Backend Variant information for Backend '" + backendMetaData.getName() + "'");
          }
          else
          {

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/queues/SQSQueuePoller.java

@@ -24,6 +24,7 @@ package com.kingsrook.qqq.backend.core.actions.queues;
 
 import java.util.ArrayList;
 import java.util.List;
+import java.util.Objects;
 import java.util.function.Supplier;
 import com.amazonaws.auth.AWSStaticCredentialsProvider;
 import com.amazonaws.auth.BasicAWSCredentials;
@@ -41,6 +42,8 @@ import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput;
 import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessOutput;
 import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
 import com.kingsrook.qqq.backend.core.model.metadata.queues.QQueueMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.queues.SQSPollerSettings;
+import com.kingsrook.qqq.backend.core.model.metadata.queues.SQSQueueMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.queues.SQSQueueProviderMetaData;
 import com.kingsrook.qqq.backend.core.model.session.QSession;
 
@@ -90,15 +93,17 @@ public class SQSQueuePoller implements Runnable
          }
          queueUrl += queueMetaData.getQueueName();
 
-         while(true)
+         SQSPollerSettings sqsPollerSettings = getSqsPollerSettings(queueProviderMetaData, queueMetaData);
+
+         for(int loop = 0; loop < sqsPollerSettings.getMaxLoops(); loop++)
          {
             ///////////////////////////////
             // fetch a batch of messages //
             ///////////////////////////////
             ReceiveMessageRequest receiveMessageRequest = new ReceiveMessageRequest();
             receiveMessageRequest.setQueueUrl(queueUrl);
-            receiveMessageRequest.setMaxNumberOfMessages(10);
-            receiveMessageRequest.setWaitTimeSeconds(20); // help urge SQS to query multiple servers and find more messages
+            receiveMessageRequest.setMaxNumberOfMessages(sqsPollerSettings.getMaxNumberOfMessages());
+            receiveMessageRequest.setWaitTimeSeconds(sqsPollerSettings.getWaitTimeSeconds()); // larger value (e.g., 20) can help urge SQS to query multiple servers and find more messages
             ReceiveMessageResult receiveMessageResult = sqs.receiveMessage(receiveMessageRequest);
             if(receiveMessageResult.getMessages().isEmpty())
             {
@@ -177,6 +182,47 @@ public class SQSQueuePoller implements Runnable
 
 
 
+   /*******************************************************************************
+    ** For a given queueProvider and queue, get the poller settings to use (using
+    ** default values if none are set at either level).
+    *******************************************************************************/
+   static SQSPollerSettings getSqsPollerSettings(SQSQueueProviderMetaData queueProviderMetaData, QQueueMetaData queueMetaData)
+   {
+      /////////////////////////////////
+      // start with default settings //
+      /////////////////////////////////
+      SQSPollerSettings sqsPollerSettings = new SQSPollerSettings()
+         .withMaxLoops(Integer.MAX_VALUE)
+         .withMaxNumberOfMessages(10)
+         .withWaitTimeSeconds(20);
+
+      /////////////////////////////////////////////////////////////////////
+      // if the queue provider has settings, let them overwrite defaults //
+      /////////////////////////////////////////////////////////////////////
+      if(queueProviderMetaData != null && queueProviderMetaData.getPollerSettings() != null)
+      {
+         SQSPollerSettings providerSettings = queueProviderMetaData.getPollerSettings();
+         sqsPollerSettings.setMaxLoops(Objects.requireNonNullElse(providerSettings.getMaxLoops(), sqsPollerSettings.getMaxLoops()));
+         sqsPollerSettings.setMaxNumberOfMessages(Objects.requireNonNullElse(providerSettings.getMaxNumberOfMessages(), sqsPollerSettings.getMaxNumberOfMessages()));
+         sqsPollerSettings.setWaitTimeSeconds(Objects.requireNonNullElse(providerSettings.getWaitTimeSeconds(), sqsPollerSettings.getWaitTimeSeconds()));
+      }
+
+      ////////////////////////////////////////////////////////////
+      // if the queue has settings, let them overwrite defaults //
+      ////////////////////////////////////////////////////////////
+      if(queueMetaData instanceof SQSQueueMetaData sqsQueueMetaData && sqsQueueMetaData.getPollerSettings() != null)
+      {
+         SQSPollerSettings providerSettings = sqsQueueMetaData.getPollerSettings();
+         sqsPollerSettings.setMaxLoops(Objects.requireNonNullElse(providerSettings.getMaxLoops(), sqsPollerSettings.getMaxLoops()));
+         sqsPollerSettings.setMaxNumberOfMessages(Objects.requireNonNullElse(providerSettings.getMaxNumberOfMessages(), sqsPollerSettings.getMaxNumberOfMessages()));
+         sqsPollerSettings.setWaitTimeSeconds(Objects.requireNonNullElse(providerSettings.getWaitTimeSeconds(), sqsPollerSettings.getWaitTimeSeconds()));
+      }
+
+      return sqsPollerSettings;
+   }
+
+
+
    /*******************************************************************************
     ** Setter for queueProviderMetaData
     **

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/CsvExportStreamer.java

@@ -31,6 +31,7 @@ import com.kingsrook.qqq.backend.core.logging.QLogger;
 import com.kingsrook.qqq.backend.core.model.actions.reporting.ExportInput;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportView;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 import com.kingsrook.qqq.backend.core.utils.StringUtils;
 
@@ -65,12 +66,12 @@ public class CsvExportStreamer implements ExportStreamerInterface
     **
     *******************************************************************************/
    @Override
-   public void start(ExportInput exportInput, List<QFieldMetaData> fields, String label) throws QReportingException
+   public void start(ExportInput exportInput, List<QFieldMetaData> fields, String label, QReportView view) throws QReportingException
    {
       this.exportInput = exportInput;
       this.fields = fields;
       table = exportInput.getTable();
-      outputStream = this.exportInput.getReportOutputStream();
+      outputStream = this.exportInput.getReportDestination().getReportOutputStream();
 
       writeTitleAndHeader();
    }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/DistinctFilteringRecordPipe.java

@@ -0,0 +1,146 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2023.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.actions.reporting;
+
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.UniqueKey;
+
+
+/*******************************************************************************
+ ** Subclass of record pipe that ony allows through distinct records, based on
+ ** the set of fields specified in the constructor as a uniqueKey.
+ *******************************************************************************/
+public class DistinctFilteringRecordPipe extends RecordPipe
+{
+   private UniqueKey         uniqueKey;
+   private Set<Serializable> seenValues = new HashSet<>();
+
+
+
+   /*******************************************************************************
+    ** Constructor
+    **
+    *******************************************************************************/
+   public DistinctFilteringRecordPipe(UniqueKey uniqueKey)
+   {
+      this.uniqueKey = uniqueKey;
+   }
+
+
+
+   /*******************************************************************************
+    ** Constructor that accepts pipe's overrideCapacity (allowed to be null)
+    **
+    *******************************************************************************/
+   public DistinctFilteringRecordPipe(UniqueKey uniqueKey, Integer overrideCapacity)
+   {
+      super(overrideCapacity);
+      this.uniqueKey = uniqueKey;
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public void addRecords(List<QRecord> records) throws QException
+   {
+      List<QRecord> recordsToAdd = new ArrayList<>();
+      for(QRecord record : records)
+      {
+         if(!seenBefore(record))
+         {
+            recordsToAdd.add(record);
+         }
+      }
+
+      if(recordsToAdd.isEmpty())
+      {
+         return;
+      }
+
+      super.addRecords(recordsToAdd);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public void addRecord(QRecord record) throws QException
+   {
+      if(seenBefore(record))
+      {
+         return;
+      }
+
+      super.addRecord(record);
+   }
+
+
+
+   /*******************************************************************************
+    ** return true if we've seen this record before (based on the unique key) -
+    ** also - update the set of seen values!
+    *******************************************************************************/
+   private boolean seenBefore(QRecord record)
+   {
+      Serializable ukValues = extractUKValues(record);
+      if(seenValues.contains(ukValues))
+      {
+         return true;
+      }
+      seenValues.add(ukValues);
+      return false;
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private Serializable extractUKValues(QRecord record)
+   {
+      if(uniqueKey.getFieldNames().size() == 1)
+      {
+         return (record.getValue(uniqueKey.getFieldNames().get(0)));
+      }
+      else
+      {
+         ArrayList<Serializable> rs = new ArrayList<>();
+         for(String fieldName : uniqueKey.getFieldNames())
+         {
+            rs.add(record.getValue(fieldName));
+         }
+         return (rs);
+      }
+   }
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/ExportAction.java

@@ -44,6 +44,7 @@ import com.kingsrook.qqq.backend.core.logging.QLogger;
 import com.kingsrook.qqq.backend.core.model.actions.reporting.ExportInput;
 import com.kingsrook.qqq.backend.core.model.actions.reporting.ExportOutput;
 import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportFormat;
+import com.kingsrook.qqq.backend.core.model.actions.tables.QueryHint;
 import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountOutput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
@@ -52,6 +53,7 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryJoin;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
+import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportView;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.ExposedJoin;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleDispatcher;
@@ -138,7 +140,7 @@ public class ExportAction
       ///////////////////////////////////////////////////////////////////////////////////////////////////////////
       // check if this report format has a max-rows limit -- if so, do a count to verify we're under the limit //
       ///////////////////////////////////////////////////////////////////////////////////////////////////////////
-      ReportFormat reportFormat = exportInput.getReportFormat();
+      ReportFormat reportFormat = exportInput.getReportDestination().getReportFormat();
       verifyCountUnderMax(exportInput, backendModule, reportFormat);
 
       preExecuteRan = true;
@@ -189,6 +191,9 @@ public class ExportAction
       Set<String>     addedJoinNames = new HashSet<>();
       if(CollectionUtils.nullSafeHasContents(exportInput.getFieldNames()))
       {
+         /////////////////////////////////////////////////////////////////////////////////////////////
+         // make sure that any tables being selected from are included as (LEFT) joins in the query //
+         /////////////////////////////////////////////////////////////////////////////////////////////
          for(String fieldName : exportInput.getFieldNames())
          {
             if(fieldName.contains("."))
@@ -197,27 +202,7 @@ public class ExportAction
                String   joinTableName = parts[0];
                if(!addedJoinNames.contains(joinTableName))
                {
-                  QueryJoin queryJoin = new QueryJoin(joinTableName).withType(QueryJoin.Type.LEFT).withSelect(true);
-                  queryJoins.add(queryJoin);
-
-                  /////////////////////////////////////////////////////////////////////////////////////////////
-                  // in at least some cases, we need to let the queryJoin know what join-meta-data to use... //
-                  // This code basically mirrors what QFMD is doing right now, so it's better -              //
-                  // but shouldn't all of this just be in JoinsContext?  it does some of this...             //
-                  /////////////////////////////////////////////////////////////////////////////////////////////
-                  QTableMetaData        table               = exportInput.getTable();
-                  Optional<ExposedJoin> exposedJoinOptional = CollectionUtils.nonNullList(table.getExposedJoins()).stream().filter(ej -> ej.getJoinTable().equals(joinTableName)).findFirst();
-                  if(exposedJoinOptional.isEmpty())
-                  {
-                     throw (new QException("Could not find exposed join between base table " + table.getName() + " and requested join table " + joinTableName));
-                  }
-                  ExposedJoin exposedJoin = exposedJoinOptional.get();
-
-                  if(exposedJoin.getJoinPath().size() == 1)
-                  {
-                     queryJoin.setJoinMetaData(QContext.getQInstance().getJoin(exposedJoin.getJoinPath().get(exposedJoin.getJoinPath().size() - 1)));
-                  }
-
+                  queryJoins.add(new QueryJoin(joinTableName).withType(QueryJoin.Type.LEFT).withSelect(true));
                   addedJoinNames.add(joinTableName);
                }
             }
@@ -232,6 +217,8 @@ public class ExportAction
       }
       queryInput.getFilter().setLimit(exportInput.getLimit());
       queryInput.setShouldTranslatePossibleValues(true);
+      queryInput.withQueryHint(QueryHint.POTENTIALLY_LARGE_NUMBER_OF_RESULTS);
+      queryInput.withQueryHint(QueryHint.MAY_USE_READ_ONLY_BACKEND);
 
       /////////////////////////////////////////////////////////////////
       // tell this query that it needs to put its output into a pipe //
@@ -242,10 +229,19 @@ public class ExportAction
       ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
       // set up a report streamer, which will read rows from the pipe, and write formatted report rows to the output stream //
       ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-      ReportFormat            reportFormat   = exportInput.getReportFormat();
+      ReportFormat            reportFormat   = exportInput.getReportDestination().getReportFormat();
       ExportStreamerInterface reportStreamer = reportFormat.newReportStreamer();
       List<QFieldMetaData>    fields         = getFields(exportInput);
-      reportStreamer.start(exportInput, fields, "Sheet 1");
+
+      //////////////////////////////////////////////////////////
+      // it seems we can pass a view with just a name in here //
+      //////////////////////////////////////////////////////////
+      List<QReportView> views = new ArrayList<>();
+      views.add(new QReportView()
+         .withName("export"));
+
+      reportStreamer.preRun(exportInput.getReportDestination(), views);
+      reportStreamer.start(exportInput, fields, "Sheet 1", views.get(0));
 
       //////////////////////////////////////////
       // run the query action as an async job //
@@ -334,7 +330,7 @@ public class ExportAction
 
       try
       {
-         exportInput.getReportOutputStream().close();
+         exportInput.getReportDestination().getReportOutputStream().close();
       }
       catch(Exception e)
       {

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/ExportStreamerInterface.java

@@ -26,8 +26,10 @@ import java.util.List;
 import java.util.Map;
 import com.kingsrook.qqq.backend.core.exceptions.QReportingException;
 import com.kingsrook.qqq.backend.core.model.actions.reporting.ExportInput;
+import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportDestination;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportView;
 
 
 /*******************************************************************************
@@ -35,20 +37,14 @@ import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
  *******************************************************************************/
 public interface ExportStreamerInterface
 {
-   /*******************************************************************************
-    ** Called once, before any rows are available.  Meant to write a header, for example.
-    *******************************************************************************/
-   void start(ExportInput exportInput, List<QFieldMetaData> fields, String label) throws QReportingException;
 
    /*******************************************************************************
-    ** Called as records flow into the pipe.
-    ******************************************************************************/
-   void addRecords(List<QRecord> recordList) throws QReportingException;
-
-   /*******************************************************************************
-    ** Called once, after all rows are available.  Meant to write a footer, or close resources, for example.
+    ** Called once, before any sheets are actually being produced.
     *******************************************************************************/
-   void finish() throws QReportingException;
+   default void preRun(ReportDestination reportDestination, List<QReportView> views) throws QReportingException
+   {
+      // noop in base class
+   }
 
    /*******************************************************************************
     **
@@ -58,6 +54,20 @@ public interface ExportStreamerInterface
       // noop in base class
    }
 
+   /*******************************************************************************
+    ** Called once per sheet, before any rows are available.  Meant to write a
+    ** header, for example.
+    **
+    ** If multiple sheets are being created, there is no separate end-sheet call.
+    ** Rather, a new one will just get started...
+    *******************************************************************************/
+   void start(ExportInput exportInput, List<QFieldMetaData> fields, String label, QReportView view) throws QReportingException;
+
+   /*******************************************************************************
+    ** Called as records flow into the pipe.
+    ******************************************************************************/
+   void addRecords(List<QRecord> recordList) throws QReportingException;
+
    /*******************************************************************************
     **
     *******************************************************************************/
@@ -65,4 +75,11 @@ public interface ExportStreamerInterface
    {
       addRecords(List.of(record));
    }
+
+   /*******************************************************************************
+    ** Called after all sheets are complete.  Meant to do a final write, or close
+    ** resources, for example.
+    *******************************************************************************/
+   void finish() throws QReportingException;
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/ExportsFullInstanceVerifier.java

@@ -0,0 +1,202 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.actions.reporting;
+
+
+import java.io.ByteArrayOutputStream;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import com.kingsrook.qqq.backend.core.context.QContext;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.logging.QLogger;
+import com.kingsrook.qqq.backend.core.model.actions.reporting.ExportInput;
+import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportDestination;
+import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportFormat;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.Capability;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.ExposedJoin;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
+import com.kingsrook.qqq.backend.core.utils.Pair;
+import com.kingsrook.qqq.backend.core.utils.StringUtils;
+import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
+
+
+/*******************************************************************************
+ ** Utility for verifying that the ExportAction works for all tables, and all
+ ** exposed joins.
+ **
+ ** Meant for use within a unit test, or maybe as part of an instance's boot-up/
+ ** validation.
+ *******************************************************************************/
+public class ExportsFullInstanceVerifier
+{
+   private static final QLogger LOG = QLogger.getLogger(ExportsFullInstanceVerifier.class);
+
+   private boolean filterForAtMostOneRowPerExport = true;
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public void verify(Collection<QTableMetaData> tables) throws QException
+   {
+      Map<Pair<String, String>, Exception> caughtExceptions = new LinkedHashMap<>();
+      for(QTableMetaData table : tables)
+      {
+         if(table.isCapabilityEnabled(QContext.getQInstance().getBackendForTable(table.getName()), Capability.TABLE_QUERY))
+         {
+            LOG.info("Verifying Exports on table", logPair("tableName", table.getName()));
+
+            //////////////////////////////////////////////
+            // run the table by itself (no join fields) //
+            //////////////////////////////////////////////
+            runExport(table.getName(), Collections.emptyList(), "main-table-only", caughtExceptions);
+
+            ///////////////////////////////////////////////////
+            // run once w/ the fields from each exposed join //
+            ///////////////////////////////////////////////////
+            for(ExposedJoin exposedJoin : CollectionUtils.nonNullList(table.getExposedJoins()))
+            {
+               runExport(table.getName(), List.of(exposedJoin), "join-" + exposedJoin.getLabel(), caughtExceptions);
+            }
+
+            /////////////////////////////////////////////////
+            // run w/ all exposed joins (if there are any) //
+            /////////////////////////////////////////////////
+            if(CollectionUtils.nullSafeHasContents(table.getExposedJoins()))
+            {
+               runExport(table.getName(), table.getExposedJoins(), "all-joins", caughtExceptions);
+            }
+         }
+      }
+
+      //////////////////////////////////
+      // log out an exceptions caught //
+      //////////////////////////////////
+      if(!caughtExceptions.isEmpty())
+      {
+         for(Map.Entry<Pair<String, String>, Exception> entry : caughtExceptions.entrySet())
+         {
+            LOG.info("Caught an exception verifying reports", entry.getValue(), logPair("tableName", entry.getKey().getA()), logPair("fieldName", entry.getKey().getB()));
+         }
+         throw (new QException("Reports Verification failed with " + caughtExceptions.size() + " exception" + StringUtils.plural(caughtExceptions.size())));
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private void runExport(String tableName, List<ExposedJoin> exposedJoinList, String description, Map<Pair<String, String>, Exception> caughtExceptions)
+   {
+      try
+      {
+         ////////////////////////////////////////////////////////////////////////////////////
+         // build the list of fieldNames to export - starting with all fields in the table //
+         ////////////////////////////////////////////////////////////////////////////////////
+         List<String> fieldNames = new ArrayList<>();
+         for(QFieldMetaData field : QContext.getQInstance().getTable(tableName).getFields().values())
+         {
+            fieldNames.add(field.getName());
+         }
+
+         ///////////////////////////////////////////////////
+         // add all fields from all exposed joins as well //
+         ///////////////////////////////////////////////////
+         for(ExposedJoin exposedJoin : CollectionUtils.nonNullList(exposedJoinList))
+         {
+            QTableMetaData joinTable = QContext.getQInstance().getTable(exposedJoin.getJoinTable());
+            for(QFieldMetaData field : joinTable.getFields().values())
+            {
+               fieldNames.add(joinTable.getName() + "." + field.getName());
+            }
+         }
+
+         LOG.info("Verifying export", logPair("description", description), logPair("fieldCount", fieldNames.size()));
+
+         QQueryFilter queryFilter = new QQueryFilter();
+
+         /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+         // if caller is okay with a filter that should limit the report to a small number of rows (could be more than 1 for to-many joins), then do so //
+         /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+         if(filterForAtMostOneRowPerExport)
+         {
+            queryFilter.withCriteria(QContext.getQInstance().getTable(tableName).getPrimaryKeyField(), QCriteriaOperator.EQUALS, 1);
+         }
+
+         ExportInput exportInput = new ExportInput();
+         exportInput.setTableName(tableName);
+         exportInput.setFieldNames(fieldNames);
+         exportInput.setReportDestination(new ReportDestination()
+            .withReportOutputStream(new ByteArrayOutputStream())
+            .withReportFormat(ReportFormat.CSV));
+         exportInput.setQueryFilter(queryFilter);
+         new ExportAction().execute(exportInput);
+      }
+      catch(QException e)
+      {
+         caughtExceptions.put(Pair.of(tableName, description), e);
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for filterForAtMostOneRowPerExport
+    *******************************************************************************/
+   public boolean getFilterForAtMostOneRowPerExport()
+   {
+      return (this.filterForAtMostOneRowPerExport);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for filterForAtMostOneRowPerExport
+    *******************************************************************************/
+   public void setFilterForAtMostOneRowPerExport(boolean filterForAtMostOneRowPerExport)
+   {
+      this.filterForAtMostOneRowPerExport = filterForAtMostOneRowPerExport;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for filterForAtMostOneRowPerExport
+    *******************************************************************************/
+   public ExportsFullInstanceVerifier withFilterForAtMostOneRowPerExport(boolean filterForAtMostOneRowPerExport)
+   {
+      this.filterForAtMostOneRowPerExport = filterForAtMostOneRowPerExport;
+      return (this);
+   }
+
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/GenerateReportAction.java

@@ -24,6 +24,8 @@ package com.kingsrook.qqq.backend.core.actions.reporting;
 
 import java.io.Serializable;
 import java.math.BigDecimal;
+import java.time.Instant;
+import java.time.LocalDate;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashMap;
@@ -32,18 +34,23 @@ import java.util.List;
 import java.util.Map;
 import java.util.Objects;
 import java.util.Set;
+import java.util.concurrent.atomic.AtomicInteger;
 import java.util.function.Function;
 import java.util.function.Supplier;
 import java.util.stream.Collectors;
+import com.kingsrook.qqq.backend.core.actions.AbstractQActionFunction;
 import com.kingsrook.qqq.backend.core.actions.async.AsyncRecordPipeLoop;
 import com.kingsrook.qqq.backend.core.actions.customizers.QCodeLoader;
 import com.kingsrook.qqq.backend.core.actions.reporting.customizers.DataSourceQueryInputCustomizer;
 import com.kingsrook.qqq.backend.core.actions.reporting.customizers.ReportViewCustomizer;
+import com.kingsrook.qqq.backend.core.actions.tables.CountAction;
 import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
 import com.kingsrook.qqq.backend.core.actions.values.QValueFormatter;
+import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.exceptions.QFormulaException;
 import com.kingsrook.qqq.backend.core.exceptions.QReportingException;
+import com.kingsrook.qqq.backend.core.exceptions.QUserFacingException;
 import com.kingsrook.qqq.backend.core.instances.QMetaDataVariableInterpreter;
 import com.kingsrook.qqq.backend.core.logging.QLogger;
 import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
@@ -51,10 +58,15 @@ import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutp
 import com.kingsrook.qqq.backend.core.model.actions.reporting.ExportInput;
 import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportFormat;
 import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportInput;
+import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportOutput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.QueryHint;
+import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountOutput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.JoinsContext;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryJoin;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
@@ -68,11 +80,17 @@ import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwith
 import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.BackendStepPostRunInput;
 import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.BackendStepPostRunOutput;
 import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
+import com.kingsrook.qqq.backend.core.utils.ObjectUtils;
 import com.kingsrook.qqq.backend.core.utils.Pair;
 import com.kingsrook.qqq.backend.core.utils.StringUtils;
 import com.kingsrook.qqq.backend.core.utils.aggregates.AggregatesInterface;
 import com.kingsrook.qqq.backend.core.utils.aggregates.BigDecimalAggregates;
+import com.kingsrook.qqq.backend.core.utils.aggregates.InstantAggregates;
 import com.kingsrook.qqq.backend.core.utils.aggregates.IntegerAggregates;
+import com.kingsrook.qqq.backend.core.utils.aggregates.LocalDateAggregates;
+import com.kingsrook.qqq.backend.core.utils.aggregates.LongAggregates;
+import com.kingsrook.qqq.backend.core.utils.aggregates.StringAggregates;
+import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
 
 
 /*******************************************************************************
@@ -87,7 +105,7 @@ import com.kingsrook.qqq.backend.core.utils.aggregates.IntegerAggregates;
  ** - summaries (like pivot tables, but called summary to avoid confusion with "native" pivot tables),
  ** - native pivot tables (not initially supported, due to lack of support in fastexcel...).
  *******************************************************************************/
-public class GenerateReportAction
+public class GenerateReportAction extends AbstractQActionFunction<ReportInput, ReportOutput>
 {
    private static final QLogger LOG = QLogger.getLogger(GenerateReportAction.class);
 
@@ -101,50 +119,67 @@ public class GenerateReportAction
    // Aggregates: (count:47;sum:10,000;max:2,000;min:15)                                                      //
    // salesSummaryReport > [(state:MO),(city:St.Louis)] > salePrice  > (count:47;sum:10,000;max:2,000;min:15) //
    /////////////////////////////////////////////////////////////////////////////////////////////////////////////
-   Map<String, Map<SummaryKey, Map<String, AggregatesInterface<?>>>> summaryAggregates  = new HashMap<>();
-   Map<String, Map<SummaryKey, Map<String, AggregatesInterface<?>>>> varianceAggregates = new HashMap<>();
+   Map<String, Map<SummaryKey, Map<String, AggregatesInterface<?, ?>>>> summaryAggregates  = new HashMap<>();
+   Map<String, Map<SummaryKey, Map<String, AggregatesInterface<?, ?>>>> varianceAggregates = new HashMap<>();
 
-   Map<String, AggregatesInterface<?>> totalAggregates         = new HashMap<>();
-   Map<String, AggregatesInterface<?>> varianceTotalAggregates = new HashMap<>();
+   Map<String, AggregatesInterface<?, ?>> totalAggregates         = new HashMap<>();
+   Map<String, AggregatesInterface<?, ?>> varianceTotalAggregates = new HashMap<>();
 
-   private QReportMetaData         report;
-   private ReportFormat            reportFormat;
    private ExportStreamerInterface reportStreamer;
+   private List<QReportDataSource> dataSources;
+   private List<QReportView>       views;
+
+   private Map<String, Integer> countByDataSource = new HashMap<>();
 
 
 
    /*******************************************************************************
     **
     *******************************************************************************/
-   public void execute(ReportInput reportInput) throws QException
+   public ReportOutput execute(ReportInput reportInput) throws QException
    {
-      report = reportInput.getInstance().getReport(reportInput.getReportName());
-      reportFormat = reportInput.getReportFormat();
+      ReportOutput    reportOutput = new ReportOutput();
+      QReportMetaData report       = getReportMetaData(reportInput);
+
+      this.views = report.getViews();
+      this.dataSources = report.getDataSources();
+
+      ReportFormat reportFormat = reportInput.getReportDestination().getReportFormat();
       if(reportFormat == null)
       {
          throw new QException("Report format was not specified.");
       }
-      reportStreamer = reportFormat.newReportStreamer();
+
+      if(reportInput.getOverrideExportStreamerSupplier() != null)
+      {
+         reportStreamer = reportInput.getOverrideExportStreamerSupplier().get();
+      }
+      else
+      {
+         reportStreamer = reportFormat.newReportStreamer();
+      }
+
+      reportStreamer.preRun(reportInput.getReportDestination(), views);
 
       ////////////////////////////////////////////////////////////////////////////////////////////////
       // foreach data source, do a query (possibly more than 1, if it goes to multiple table views) //
       ////////////////////////////////////////////////////////////////////////////////////////////////
-      for(QReportDataSource dataSource : report.getDataSources())
+      for(QReportDataSource dataSource : dataSources)
       {
          //////////////////////////////////////////////////////////////////////////////
          // make a list of the views that use this data source for various purposes. //
          //////////////////////////////////////////////////////////////////////////////
-         List<QReportView> dataSourceTableViews = report.getViews().stream()
+         List<QReportView> dataSourceTableViews = views.stream()
             .filter(v -> v.getType().equals(ReportType.TABLE))
             .filter(v -> v.getDataSourceName().equals(dataSource.getName()))
             .toList();
 
-         List<QReportView> dataSourceSummaryViews = report.getViews().stream()
+         List<QReportView> dataSourceSummaryViews = views.stream()
             .filter(v -> v.getType().equals(ReportType.SUMMARY))
             .filter(v -> v.getDataSourceName().equals(dataSource.getName()))
             .toList();
 
-         List<QReportView> dataSourceVariantViews = report.getViews().stream()
+         List<QReportView> dataSourceVariantViews = views.stream()
             .filter(v -> v.getType().equals(ReportType.SUMMARY))
             .filter(v -> v.getVarianceDataSourceName() != null && v.getVarianceDataSourceName().equals(dataSource.getName()))
             .toList();
@@ -183,24 +218,50 @@ public class GenerateReportAction
                ////////////////////////////////////////////////////////////////////////////////////
                // start the table-view (e.g., open this tab in xlsx) and then run the query-loop //
                ////////////////////////////////////////////////////////////////////////////////////
-               startTableView(reportInput, dataSource, dataSourceTableView);
+               startTableView(reportInput, dataSource, dataSourceTableView, reportFormat);
                gatherData(reportInput, dataSource, dataSourceTableView, dataSourceSummaryViews, dataSourceVariantViews);
             }
          }
       }
 
+      //////////////////////
+      // add pivot sheets //
+      //////////////////////
+      for(QReportView view : views)
+      {
+         if(view.getType().equals(ReportType.PIVOT))
+         {
+            if(reportFormat.getSupportsNativePivotTables())
+            {
+               startTableView(reportInput, null, view, reportFormat);
+            }
+            else
+            {
+               LOG.warn("Request to render a report with a PIVOT type view, for a format that does not support native pivot tables", logPair("reportFormat", reportFormat));
+            }
+
+            //////////////////////////////////////////////////////////////////////////
+            // there's no data to add to a pivot table, so nothing else to do here. //
+            //////////////////////////////////////////////////////////////////////////
+         }
+      }
+
       outputSummaries(reportInput);
 
+      reportOutput.setTotalRecordCount(countByDataSource.values().stream().mapToInt(Integer::intValue).sum());
+
       reportStreamer.finish();
 
       try
       {
-         reportInput.getReportOutputStream().close();
+         reportInput.getReportDestination().getReportOutputStream().close();
       }
       catch(Exception e)
       {
          throw (new QReportingException("Error completing report", e));
       }
+
+      return (reportOutput);
    }
 
 
@@ -208,28 +269,48 @@ public class GenerateReportAction
    /*******************************************************************************
     **
     *******************************************************************************/
-   private void startTableView(ReportInput reportInput, QReportDataSource dataSource, QReportView reportView) throws QException
+   private QReportMetaData getReportMetaData(ReportInput reportInput) throws QException
    {
-      QTableMetaData table = reportInput.getInstance().getTable(dataSource.getSourceTable());
+      if(reportInput.getReportMetaData() != null)
+      {
+         return reportInput.getReportMetaData();
+      }
 
+      if(StringUtils.hasContent(reportInput.getReportName()))
+      {
+         return QContext.getQInstance().getReport(reportInput.getReportName());
+      }
+
+      throw (new QReportingException("ReportInput did not contain required parameters to identify the report being generated"));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private void startTableView(ReportInput reportInput, QReportDataSource dataSource, QReportView reportView, ReportFormat reportFormat) throws QException
+   {
       QMetaDataVariableInterpreter variableInterpreter = new QMetaDataVariableInterpreter();
       variableInterpreter.addValueMap("input", reportInput.getInputValues());
 
       ExportInput exportInput = new ExportInput();
-      exportInput.setReportFormat(reportFormat);
-      exportInput.setFilename(reportInput.getFilename());
+      exportInput.setReportDestination(reportInput.getReportDestination());
       exportInput.setTitleRow(getTitle(reportView, variableInterpreter));
       exportInput.setIncludeHeaderRow(reportView.getIncludeHeaderRow());
-      exportInput.setReportOutputStream(reportInput.getReportOutputStream());
 
       JoinsContext joinsContext = null;
-      if(StringUtils.hasContent(dataSource.getSourceTable()))
+      if(dataSource != null)
       {
-         joinsContext = new JoinsContext(exportInput.getInstance(), dataSource.getSourceTable(), dataSource.getQueryJoins(), dataSource.getQueryFilter());
+         if(StringUtils.hasContent(dataSource.getSourceTable()))
+         {
+            joinsContext = new JoinsContext(QContext.getQInstance(), dataSource.getSourceTable(), cloneDataSourceQueryJoins(dataSource), dataSource.getQueryFilter() == null ? null : dataSource.getQueryFilter().clone());
+            countDataSourceRecords(reportInput, dataSource, reportFormat);
+         }
       }
 
       List<QFieldMetaData> fields = new ArrayList<>();
-      for(QReportField column : reportView.getColumns())
+      for(QReportField column : CollectionUtils.nonNullList(reportView.getColumns()))
       {
          if(column.getIsVirtual())
          {
@@ -241,7 +322,7 @@ public class GenerateReportAction
             JoinsContext.FieldAndTableNameOrAlias fieldAndTableNameOrAlias = joinsContext == null ? null : joinsContext.getFieldAndTableNameOrAlias(effectiveFieldName);
             if(fieldAndTableNameOrAlias == null || fieldAndTableNameOrAlias.field() == null)
             {
-               throw new QReportingException("Could not find field named [" + effectiveFieldName + "] in dataSource [" + dataSource.getName() + "]");
+               throw new QReportingException("Could not find field named [" + effectiveFieldName + "] in dataSource [" + (dataSource == null ? null : dataSource.getName()) + "]");
             }
 
             QFieldMetaData field = fieldAndTableNameOrAlias.field().clone();
@@ -255,7 +336,7 @@ public class GenerateReportAction
       }
 
       reportStreamer.setDisplayFormats(getDisplayFormatMap(fields));
-      reportStreamer.start(exportInput, fields, reportView.getLabel());
+      reportStreamer.start(exportInput, fields, reportView.getLabel(), reportView);
    }
 
 
@@ -263,7 +344,56 @@ public class GenerateReportAction
    /*******************************************************************************
     **
     *******************************************************************************/
-   private void gatherData(ReportInput reportInput, QReportDataSource dataSource, QReportView tableView, List<QReportView> summaryViews, List<QReportView> variantViews) throws QException
+   private void countDataSourceRecords(ReportInput reportInput, QReportDataSource dataSource, ReportFormat reportFormat) throws QException
+   {
+      QQueryFilter queryFilter = dataSource.getQueryFilter() == null ? new QQueryFilter() : dataSource.getQueryFilter().clone();
+      setInputValuesInQueryFilter(reportInput, queryFilter);
+
+      CountInput countInput = new CountInput();
+      countInput.setTableName(dataSource.getSourceTable());
+      countInput.setFilter(queryFilter);
+      countInput.setQueryJoins(cloneDataSourceQueryJoins(dataSource));
+      CountOutput countOutput = new CountAction().execute(countInput);
+
+      if(countOutput.getCount() != null)
+      {
+         countByDataSource.put(dataSource.getName(), countOutput.getCount());
+
+         if(reportFormat.getMaxRows() != null && countOutput.getCount() > reportFormat.getMaxRows())
+         {
+            throw (new QUserFacingException("The requested report would include more rows ("
+               + String.format("%,d", countOutput.getCount()) + ") than the maximum allowed ("
+               + String.format("%,d", reportFormat.getMaxRows()) + ") for the selected file format (" + reportFormat + ")."));
+         }
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private static List<QueryJoin> cloneDataSourceQueryJoins(QReportDataSource dataSource)
+   {
+      if(dataSource == null || dataSource.getQueryJoins() == null)
+      {
+         return (null);
+      }
+
+      List<QueryJoin> rs = new ArrayList<>();
+      for(QueryJoin queryJoin : dataSource.getQueryJoins())
+      {
+         rs.add(queryJoin.clone());
+      }
+      return (rs);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private Integer gatherData(ReportInput reportInput, QReportDataSource dataSource, QReportView tableView, List<QReportView> summaryViews, List<QReportView> variantViews) throws QException
    {
       ////////////////////////////////////////////////////////////////////////////////////////
       // check if this view has a transform step - if so, set it up now and run its pre-run //
@@ -290,6 +420,9 @@ public class GenerateReportAction
       RunBackendStepInput   finalTransformStepInput  = transformStepInput;
       RunBackendStepOutput  finalTransformStepOutput = transformStepOutput;
 
+      String        tableLabel    = ObjectUtils.tryElse(() -> QContext.getQInstance().getTable(dataSource.getSourceTable()).getLabel(), Objects.requireNonNullElse(dataSource.getSourceTable(), ""));
+      AtomicInteger consumedCount = new AtomicInteger(0);
+
       /////////////////////////////////////////////////////////////////
       // run a record pipe loop, over the query for this data source //
       /////////////////////////////////////////////////////////////////
@@ -305,10 +438,12 @@ public class GenerateReportAction
             queryInput.setRecordPipe(recordPipe);
             queryInput.setTableName(dataSource.getSourceTable());
             queryInput.setFilter(queryFilter);
-            queryInput.setQueryJoins(dataSource.getQueryJoins());
+            queryInput.setQueryJoins(cloneDataSourceQueryJoins(dataSource));
+            queryInput.withQueryHint(QueryHint.POTENTIALLY_LARGE_NUMBER_OF_RESULTS);
+            queryInput.withQueryHint(QueryHint.MAY_USE_READ_ONLY_BACKEND);
 
             queryInput.setShouldTranslatePossibleValues(true);
-            queryInput.setFieldsToTranslatePossibleValues(setupFieldsToTranslatePossibleValues(reportInput, dataSource, new JoinsContext(reportInput.getInstance(), dataSource.getSourceTable(), dataSource.getQueryJoins(), queryInput.getFilter())));
+            queryInput.setFieldsToTranslatePossibleValues(setupFieldsToTranslatePossibleValues(reportInput, dataSource));
 
             if(dataSource.getQueryInputCustomizer() != null)
             {
@@ -345,11 +480,22 @@ public class GenerateReportAction
          if(finalTransformStep != null)
          {
             finalTransformStepInput.setRecords(records);
-            finalTransformStep.run(finalTransformStepInput, finalTransformStepOutput);
+            finalTransformStep.runOnePage(finalTransformStepInput, finalTransformStepOutput);
             records = finalTransformStepOutput.getRecords();
          }
 
-         return (consumeRecords(reportInput, dataSource, records, tableView, summaryViews, variantViews));
+         Integer total = countByDataSource.get(dataSource.getName());
+         if(total != null)
+         {
+            reportInput.getAsyncJobCallback().updateStatus("Processing " + tableLabel + " records", consumedCount.get() + 1, total);
+         }
+         else
+         {
+            reportInput.getAsyncJobCallback().updateStatus("Processing " + tableLabel + " records (" + String.format("%,d", consumedCount.get() + 1) + ")");
+         }
+         consumedCount.getAndAdd(records.size());
+
+         return (consumeRecords(dataSource, records, tableView, summaryViews, variantViews));
       });
 
       ////////////////////////////////////////////////
@@ -359,6 +505,8 @@ public class GenerateReportAction
       {
          transformStep.postRun(new BackendStepPostRunInput(transformStepInput), new BackendStepPostRunOutput(transformStepOutput));
       }
+
+      return consumedCount.get();
    }
 
 
@@ -366,11 +514,11 @@ public class GenerateReportAction
    /*******************************************************************************
     **
     *******************************************************************************/
-   private Set<String> setupFieldsToTranslatePossibleValues(ReportInput reportInput, QReportDataSource dataSource, JoinsContext joinsContext)
+   private Set<String> setupFieldsToTranslatePossibleValues(ReportInput reportInput, QReportDataSource dataSource) throws QException
    {
       Set<String> fieldsToTranslatePossibleValues = new HashSet<>();
 
-      for(QReportView view : report.getViews())
+      for(QReportView view : views)
       {
          for(QReportField column : CollectionUtils.nonNullList(view.getColumns()))
          {
@@ -384,15 +532,16 @@ public class GenerateReportAction
             }
          }
 
-         for(String summaryField : CollectionUtils.nonNullList(view.getPivotFields()))
+         for(String summaryFieldName : CollectionUtils.nonNullList(view.getSummaryFields()))
          {
             ///////////////////////////////////////////////////////////////////////////////
             // all pivotFields that are possible value sources are implicitly translated //
             ///////////////////////////////////////////////////////////////////////////////
-            QTableMetaData table = reportInput.getInstance().getTable(dataSource.getSourceTable());
-            if(table.getField(summaryField).getPossibleValueSourceName() != null)
+            QTableMetaData    mainTable         = QContext.getQInstance().getTable(dataSource.getSourceTable());
+            FieldAndJoinTable fieldAndJoinTable = getFieldAndJoinTable(mainTable, summaryFieldName);
+            if(fieldAndJoinTable.field().getPossibleValueSourceName() != null)
             {
-               fieldsToTranslatePossibleValues.add(summaryField);
+               fieldsToTranslatePossibleValues.add(summaryFieldName);
             }
          }
       }
@@ -405,7 +554,33 @@ public class GenerateReportAction
    /*******************************************************************************
     **
     *******************************************************************************/
-   private void setInputValuesInQueryFilter(ReportInput reportInput, QQueryFilter queryFilter)
+   public static FieldAndJoinTable getFieldAndJoinTable(QTableMetaData mainTable, String fieldName) throws QException
+   {
+      if(fieldName.indexOf('.') > -1)
+      {
+         String joinTableName = fieldName.replaceAll("\\..*", "");
+         String joinFieldName = fieldName.replaceAll(".*\\.", "");
+
+         QTableMetaData joinTable = QContext.getQInstance().getTable(joinTableName);
+         if(joinTable == null)
+         {
+            throw (new QException("Unrecognized join table name: " + joinTableName));
+         }
+
+         return new FieldAndJoinTable(joinTable.getField(joinFieldName), joinTable);
+      }
+      else
+      {
+         return new FieldAndJoinTable(mainTable.getField(fieldName), mainTable);
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private void setInputValuesInQueryFilter(ReportInput reportInput, QQueryFilter queryFilter) throws QException
    {
       if(queryFilter == null || queryFilter.getCriteria() == null)
       {
@@ -420,9 +595,9 @@ public class GenerateReportAction
    /*******************************************************************************
     **
     *******************************************************************************/
-   private Integer consumeRecords(ReportInput reportInput, QReportDataSource dataSource, List<QRecord> records, QReportView tableView, List<QReportView> summaryViews, List<QReportView> variantViews) throws QException
+   private Integer consumeRecords(QReportDataSource dataSource, List<QRecord> records, QReportView tableView, List<QReportView> summaryViews, List<QReportView> variantViews) throws QException
    {
-      QTableMetaData table = reportInput.getInstance().getTable(dataSource.getSourceTable());
+      QTableMetaData table = QContext.getQInstance().getTable(dataSource.getSourceTable());
 
       ////////////////////////////////////////////////////////////////////////////
       // if this record goes on a table view, add it to the report streamer now //
@@ -494,26 +669,27 @@ public class GenerateReportAction
    /*******************************************************************************
     **
     *******************************************************************************/
-   private void addRecordsToSummaryAggregates(QReportView view, QTableMetaData table, List<QRecord> records, Map<String, Map<SummaryKey, Map<String, AggregatesInterface<?>>>> aggregatesMap)
+   private void addRecordsToSummaryAggregates(QReportView view, QTableMetaData table, List<QRecord> records, Map<String, Map<SummaryKey, Map<String, AggregatesInterface<?, ?>>>> aggregatesMap) throws QException
    {
-      Map<SummaryKey, Map<String, AggregatesInterface<?>>> viewAggregates = aggregatesMap.computeIfAbsent(view.getName(), (name) -> new HashMap<>());
+      Map<SummaryKey, Map<String, AggregatesInterface<?, ?>>> viewAggregates = aggregatesMap.computeIfAbsent(view.getName(), (name) -> new HashMap<>());
 
       for(QRecord record : records)
       {
          SummaryKey key = new SummaryKey();
-         for(String summaryField : view.getPivotFields())
+         for(String summaryFieldName : view.getSummaryFields())
          {
-            Serializable summaryValue = record.getValue(summaryField);
-            if(table.getField(summaryField).getPossibleValueSourceName() != null)
+            FieldAndJoinTable fieldAndJoinTable = getFieldAndJoinTable(table, summaryFieldName);
+            Serializable      summaryValue      = record.getValue(summaryFieldName);
+            if(fieldAndJoinTable.field().getPossibleValueSourceName() != null)
             {
                //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
                // so, this is kinda a thing - where we implicitly use possible-value labels (e.g., display values) for pivot fields... //
                //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-               summaryValue = record.getDisplayValue(summaryField);
+               summaryValue = record.getDisplayValue(summaryFieldName);
             }
-            key.add(summaryField, summaryValue);
+            key.add(summaryFieldName, summaryValue);
 
-            if(view.getIncludePivotSubTotals() && key.getKeys().size() < view.getPivotFields().size())
+            if(view.getIncludeSummarySubTotals() && key.getKeys().size() < view.getSummaryFields().size())
             {
                /////////////////////////////////////////////////////////////////////////////////////////
                // be careful here, with these key objects, and their identity, being used as map keys //
@@ -532,9 +708,9 @@ public class GenerateReportAction
    /*******************************************************************************
     **
     *******************************************************************************/
-   private void addRecordToSummaryKeyAggregates(QTableMetaData table, QRecord record, Map<SummaryKey, Map<String, AggregatesInterface<?>>> viewAggregates, SummaryKey key)
+   private void addRecordToSummaryKeyAggregates(QTableMetaData table, QRecord record, Map<SummaryKey, Map<String, AggregatesInterface<?, ?>>> viewAggregates, SummaryKey key)
    {
-      Map<String, AggregatesInterface<?>> keyAggregates = viewAggregates.computeIfAbsent(key, (name) -> new HashMap<>());
+      Map<String, AggregatesInterface<?, ?>> keyAggregates = viewAggregates.computeIfAbsent(key, (name) -> new HashMap<>());
       addRecordToAggregatesMap(table, record, keyAggregates);
    }
 
@@ -543,23 +719,74 @@ public class GenerateReportAction
    /*******************************************************************************
     **
     *******************************************************************************/
-   private void addRecordToAggregatesMap(QTableMetaData table, QRecord record, Map<String, AggregatesInterface<?>> aggregatesMap)
+   private void addRecordToAggregatesMap(QTableMetaData table, QRecord record, Map<String, AggregatesInterface<?, ?>> aggregatesMap)
    {
-      for(QFieldMetaData field : table.getFields().values())
+      //////////////////////////////////////////////////////////////////////////////////////
+      // todo - an optimization could be, to only compute aggregates that we'll need...   //
+      // Only if we measure and see this to be slow - it may be, lots of BigDecimal math? //
+      //////////////////////////////////////////////////////////////////////////////////////
+      for(String fieldName : record.getValues().keySet())
       {
-         if(field.getType().equals(QFieldType.INTEGER))
+         QFieldMetaData field;
+         try
+         {
+            //////////////////////////////////////////////////////
+            // todo - memoize this, if we ever need to optimize //
+            //////////////////////////////////////////////////////
+            FieldAndJoinTable fieldAndJoinTable = getFieldAndJoinTable(table, fieldName);
+            field = fieldAndJoinTable.field();
+         }
+         catch(Exception e)
+         {
+            //////////////////////////////////////////////////////////////////////////////////////
+            // for non-real-fields... let's skip for now - but maybe treat as string in future? //
+            //////////////////////////////////////////////////////////////////////////////////////
+            LOG.debug("Couldn't find field in table qInstance - won't compute aggregates for it", logPair("fieldName", fieldName));
+            continue;
+         }
+
+         if(StringUtils.hasContent(field.getPossibleValueSourceName()))
          {
             @SuppressWarnings("unchecked")
-            AggregatesInterface<Integer> fieldAggregates = (AggregatesInterface<Integer>) aggregatesMap.computeIfAbsent(field.getName(), (name) -> new IntegerAggregates());
-            fieldAggregates.add(record.getValueInteger(field.getName()));
+            AggregatesInterface<String, ?> fieldAggregates = (AggregatesInterface<String, ?>) aggregatesMap.computeIfAbsent(fieldName, (name) -> new StringAggregates());
+            fieldAggregates.add(record.getDisplayValue(fieldName));
+         }
+         else if(field.getType().equals(QFieldType.INTEGER))
+         {
+            @SuppressWarnings("unchecked")
+            AggregatesInterface<Integer, ?> fieldAggregates = (AggregatesInterface<Integer, ?>) aggregatesMap.computeIfAbsent(fieldName, (name) -> new IntegerAggregates());
+            fieldAggregates.add(record.getValueInteger(fieldName));
+         }
+         else if(field.getType().equals(QFieldType.LONG))
+         {
+            @SuppressWarnings("unchecked")
+            AggregatesInterface<Long, ?> fieldAggregates = (AggregatesInterface<Long, ?>) aggregatesMap.computeIfAbsent(fieldName, (name) -> new LongAggregates());
+            fieldAggregates.add(record.getValueLong(fieldName));
          }
          else if(field.getType().equals(QFieldType.DECIMAL))
          {
             @SuppressWarnings("unchecked")
-            AggregatesInterface<BigDecimal> fieldAggregates = (AggregatesInterface<BigDecimal>) aggregatesMap.computeIfAbsent(field.getName(), (name) -> new BigDecimalAggregates());
-            fieldAggregates.add(record.getValueBigDecimal(field.getName()));
+            AggregatesInterface<BigDecimal, ?> fieldAggregates = (AggregatesInterface<BigDecimal, ?>) aggregatesMap.computeIfAbsent(fieldName, (name) -> new BigDecimalAggregates());
+            fieldAggregates.add(record.getValueBigDecimal(fieldName));
+         }
+         else if(field.getType().equals(QFieldType.DATE_TIME))
+         {
+            @SuppressWarnings("unchecked")
+            AggregatesInterface<Instant, ?> fieldAggregates = (AggregatesInterface<Instant, ?>) aggregatesMap.computeIfAbsent(fieldName, (name) -> new InstantAggregates());
+            fieldAggregates.add(record.getValueInstant(fieldName));
+         }
+         else if(field.getType().equals(QFieldType.DATE))
+         {
+            @SuppressWarnings("unchecked")
+            AggregatesInterface<LocalDate, ?> fieldAggregates = (AggregatesInterface<LocalDate, ?>) aggregatesMap.computeIfAbsent(fieldName, (name) -> new LocalDateAggregates());
+            fieldAggregates.add(record.getValueLocalDate(fieldName));
+         }
+         if(field.getType().isStringLike())
+         {
+            @SuppressWarnings("unchecked")
+            AggregatesInterface<String, ?> fieldAggregates = (AggregatesInterface<String, ?>) aggregatesMap.computeIfAbsent(fieldName, (name) -> new StringAggregates());
+            fieldAggregates.add(record.getValueString(fieldName));
          }
-         // todo - more types (dates, at least?)
       }
    }
 
@@ -568,24 +795,27 @@ public class GenerateReportAction
    /*******************************************************************************
     **
     *******************************************************************************/
-   private void outputSummaries(ReportInput reportInput) throws QReportingException, QFormulaException
+   private void outputSummaries(ReportInput reportInput) throws QException
    {
-      List<QReportView> reportViews = report.getViews().stream().filter(v -> v.getType().equals(ReportType.SUMMARY)).toList();
+      List<QReportView> reportViews = views.stream().filter(v -> v.getType().equals(ReportType.SUMMARY)).toList();
       for(QReportView view : reportViews)
       {
-         QReportDataSource dataSource    = report.getDataSource(view.getDataSourceName());
-         QTableMetaData    table         = reportInput.getInstance().getTable(dataSource.getSourceTable());
-         SummaryOutput     summaryOutput = computeSummaryRowsForView(reportInput, view, table);
+         QReportDataSource dataSource = getDataSource(view.getDataSourceName());
+         if(dataSource == null)
+         {
+            throw new QReportingException("Data source for summary view was not found (viewName=" + view.getName() + ", dataSourceName=" + view.getDataSourceName() + ").");
+         }
+
+         QTableMetaData table         = QContext.getQInstance().getTable(dataSource.getSourceTable());
+         SummaryOutput  summaryOutput = computeSummaryRowsForView(reportInput, view, table);
 
          ExportInput exportInput = new ExportInput();
-         exportInput.setReportFormat(reportFormat);
-         exportInput.setFilename(reportInput.getFilename());
+         exportInput.setReportDestination(reportInput.getReportDestination());
          exportInput.setTitleRow(summaryOutput.titleRow);
          exportInput.setIncludeHeaderRow(view.getIncludeHeaderRow());
-         exportInput.setReportOutputStream(reportInput.getReportOutputStream());
 
          reportStreamer.setDisplayFormats(getDisplayFormatMap(view));
-         reportStreamer.start(exportInput, getFields(table, view), view.getLabel());
+         reportStreamer.start(exportInput, getFields(table, view), view.getLabel(), view);
 
          reportStreamer.addRecords(summaryOutput.summaryRows); // todo - what if this set is huge?
 
@@ -598,6 +828,24 @@ public class GenerateReportAction
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private QReportDataSource getDataSource(String dataSourceName)
+   {
+      for(QReportDataSource dataSource : CollectionUtils.nonNullList(dataSources))
+      {
+         if(dataSource.getName().equals(dataSourceName))
+         {
+            return (dataSource);
+         }
+      }
+
+      return (null);
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/
@@ -625,13 +873,13 @@ public class GenerateReportAction
    /*******************************************************************************
     **
     *******************************************************************************/
-   private List<QFieldMetaData> getFields(QTableMetaData table, QReportView view)
+   private List<QFieldMetaData> getFields(QTableMetaData table, QReportView view) throws QException
    {
       List<QFieldMetaData> fields = new ArrayList<>();
-      for(String pivotField : view.getPivotFields())
+      for(String summaryFieldName : view.getSummaryFields())
       {
-         QFieldMetaData field = table.getField(pivotField);
-         fields.add(new QFieldMetaData(pivotField, field.getType()).withLabel(field.getLabel())); // todo do we need the type?  if so need table as input here
+         FieldAndJoinTable fieldAndJoinTable = getFieldAndJoinTable(table, summaryFieldName);
+         fields.add(new QFieldMetaData(summaryFieldName, fieldAndJoinTable.field().getType()).withLabel(fieldAndJoinTable.field().getLabel())); // todo do we need the type?  if so need table as input here
       }
       for(QReportField column : view.getColumns())
       {
@@ -645,9 +893,8 @@ public class GenerateReportAction
    /*******************************************************************************
     **
     *******************************************************************************/
-   private SummaryOutput computeSummaryRowsForView(ReportInput reportInput, QReportView view, QTableMetaData table) throws QReportingException, QFormulaException
+   private SummaryOutput computeSummaryRowsForView(ReportInput reportInput, QReportView view, QTableMetaData table) throws QFormulaException
    {
-      QValueFormatter              valueFormatter      = new QValueFormatter();
       QMetaDataVariableInterpreter variableInterpreter = new QMetaDataVariableInterpreter();
       variableInterpreter.addValueMap("input", reportInput.getInputValues());
       variableInterpreter.addValueMap("total", getSummaryValuesForInterpreter(totalAggregates));
@@ -661,11 +908,11 @@ public class GenerateReportAction
       // create summary rows //
       /////////////////////////
       List<QRecord> summaryRows = new ArrayList<>();
-      for(Map.Entry<SummaryKey, Map<String, AggregatesInterface<?>>> entry : summaryAggregates.getOrDefault(view.getName(), Collections.emptyMap()).entrySet())
+      for(Map.Entry<SummaryKey, Map<String, AggregatesInterface<?, ?>>> entry : summaryAggregates.getOrDefault(view.getName(), Collections.emptyMap()).entrySet())
       {
-         SummaryKey                          summaryKey      = entry.getKey();
-         Map<String, AggregatesInterface<?>> fieldAggregates = entry.getValue();
-         Map<String, Serializable>           summaryValues   = getSummaryValuesForInterpreter(fieldAggregates);
+         SummaryKey                             summaryKey      = entry.getKey();
+         Map<String, AggregatesInterface<?, ?>> fieldAggregates = entry.getValue();
+         Map<String, Serializable>              summaryValues   = getSummaryValuesForInterpreter(fieldAggregates);
          variableInterpreter.addValueMap("pivot", summaryValues);
          variableInterpreter.addValueMap("summary", summaryValues);
 
@@ -674,9 +921,9 @@ public class GenerateReportAction
 
          if(!varianceAggregates.isEmpty())
          {
-            Map<SummaryKey, Map<String, AggregatesInterface<?>>> varianceMap    = varianceAggregates.getOrDefault(view.getName(), Collections.emptyMap());
-            Map<String, AggregatesInterface<?>>                  varianceSubMap = varianceMap.getOrDefault(summaryKey, Collections.emptyMap());
-            Map<String, Serializable>                            varianceValues = getSummaryValuesForInterpreter(varianceSubMap);
+            Map<SummaryKey, Map<String, AggregatesInterface<?, ?>>> varianceMap    = varianceAggregates.getOrDefault(view.getName(), Collections.emptyMap());
+            Map<String, AggregatesInterface<?, ?>>                  varianceSubMap = varianceMap.getOrDefault(summaryKey, Collections.emptyMap());
+            Map<String, Serializable>                               varianceValues = getSummaryValuesForInterpreter(varianceSubMap);
             variableInterpreter.addValueMap("variancePivot", varianceValues);
             variableInterpreter.addValueMap("variance", varianceValues);
          }
@@ -695,7 +942,7 @@ public class GenerateReportAction
          ///////////////////////////////////////////////////////////////////////////////
          // for summary subtotals, add the text "Total" to the last field in this key //
          ///////////////////////////////////////////////////////////////////////////////
-         if(summaryKey.getKeys().size() < view.getPivotFields().size())
+         if(summaryKey.getKeys().size() < view.getSummaryFields().size())
          {
             String fieldName = summaryKey.getKeys().get(summaryKey.getKeys().size() - 1).getA();
             summaryRow.setValue(fieldName, summaryRow.getValueString(fieldName) + " Total");
@@ -719,10 +966,7 @@ public class GenerateReportAction
       //////////////////////////////////////////////////////////////////////////////////////
       if(CollectionUtils.nullSafeHasContents(view.getOrderByFields()))
       {
-         summaryRows.sort((o1, o2) ->
-         {
-            return summaryRowComparator(view, o1, o2);
-         });
+         summaryRows.sort((o1, o2) -> summaryRowComparator(view, o1, o2));
       }
 
       ////////////////
@@ -733,11 +977,11 @@ public class GenerateReportAction
       {
          totalRow = new QRecord();
 
-         for(String pivotField : view.getPivotFields())
+         for(String summaryField : view.getSummaryFields())
          {
             if(totalRow.getValues().isEmpty())
             {
-               totalRow.setValue(pivotField, "Totals");
+               totalRow.setValue(summaryField, "Totals");
             }
          }
 
@@ -757,8 +1001,6 @@ public class GenerateReportAction
             Serializable serializable = getValueForColumn(variableInterpreter, column);
             totalRow.setValue(column.getName(), serializable);
             thisRowValues.put(column.getName(), serializable);
-
-            String formatted = valueFormatter.formatValue(column.getDisplayFormat(), serializable);
          }
       }
 
@@ -781,7 +1023,7 @@ public class GenerateReportAction
             titleValues.add(variableInterpreter.interpret(titleField));
          }
 
-         title = new QValueFormatter().formatStringWithValues(view.getTitleFormat(), titleValues);
+         title = QValueFormatter.formatStringWithValues(view.getTitleFormat(), titleValues);
       }
       else if(StringUtils.hasContent(view.getTitleFormat()))
       {
@@ -857,18 +1099,24 @@ public class GenerateReportAction
    /*******************************************************************************
     **
     *******************************************************************************/
-   private Map<String, Serializable> getSummaryValuesForInterpreter(Map<String, AggregatesInterface<?>> fieldAggregates)
+   private Map<String, Serializable> getSummaryValuesForInterpreter(Map<String, AggregatesInterface<?, ?>> fieldAggregates)
    {
       Map<String, Serializable> summaryValuesForInterpreter = new HashMap<>();
-      for(Map.Entry<String, AggregatesInterface<?>> subEntry : fieldAggregates.entrySet())
+      for(Map.Entry<String, AggregatesInterface<?, ?>> subEntry : fieldAggregates.entrySet())
       {
-         String                 fieldName  = subEntry.getKey();
-         AggregatesInterface<?> aggregates = subEntry.getValue();
+         String                    fieldName  = subEntry.getKey();
+         AggregatesInterface<?, ?> aggregates = subEntry.getValue();
          summaryValuesForInterpreter.put("sum." + fieldName, aggregates.getSum());
          summaryValuesForInterpreter.put("count." + fieldName, aggregates.getCount());
+         summaryValuesForInterpreter.put("count_nums." + fieldName, aggregates.getCount());
          summaryValuesForInterpreter.put("min." + fieldName, aggregates.getMin());
          summaryValuesForInterpreter.put("max." + fieldName, aggregates.getMax());
          summaryValuesForInterpreter.put("average." + fieldName, aggregates.getAverage());
+         summaryValuesForInterpreter.put("product." + fieldName, aggregates.getProduct());
+         summaryValuesForInterpreter.put("var." + fieldName, aggregates.getVariance());
+         summaryValuesForInterpreter.put("varp." + fieldName, aggregates.getVarP());
+         summaryValuesForInterpreter.put("std_dev." + fieldName, aggregates.getStandardDeviation());
+         summaryValuesForInterpreter.put("std_devp." + fieldName, aggregates.getStdDevP());
       }
       return summaryValuesForInterpreter;
    }
@@ -882,4 +1130,27 @@ public class GenerateReportAction
    {
    }
 
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public record FieldAndJoinTable(QFieldMetaData field, QTableMetaData joinTable)
+   {
+
+      /*******************************************************************************
+       **
+       *******************************************************************************/
+      public String getLabel(QTableMetaData mainTable)
+      {
+         if(mainTable.getName().equals(joinTable.getName()))
+         {
+            return (field.getLabel());
+         }
+         else
+         {
+            return (joinTable.getLabel() + ": " + field.getLabel());
+         }
+      }
+   }
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/JsonExportStreamer.java

@@ -26,17 +26,23 @@ import java.io.IOException;
 import java.io.OutputStream;
 import java.io.Serializable;
 import java.nio.charset.StandardCharsets;
+import java.util.Arrays;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Optional;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
 import com.kingsrook.qqq.backend.core.exceptions.QReportingException;
 import com.kingsrook.qqq.backend.core.logging.QLogger;
 import com.kingsrook.qqq.backend.core.model.actions.reporting.ExportInput;
+import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportDestination;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportView;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 import com.kingsrook.qqq.backend.core.utils.JsonUtils;
-import com.kingsrook.qqq.backend.core.utils.StringUtils;
+import com.kingsrook.qqq.backend.core.utils.memoization.Memoization;
 
 
 /*******************************************************************************
@@ -46,13 +52,23 @@ public class JsonExportStreamer implements ExportStreamerInterface
 {
    private static final QLogger LOG = QLogger.getLogger(JsonExportStreamer.class);
 
+   private boolean prettyPrint = true;
+
    private ExportInput          exportInput;
    private QTableMetaData       table;
    private List<QFieldMetaData> fields;
    private OutputStream         outputStream;
 
-   private boolean needComma   = false;
-   private boolean prettyPrint = true;
+   private boolean multipleViews       = false;
+   private boolean haveStartedAnyViews = false;
+
+   private boolean needCommaBeforeRecord = false;
+
+   private byte[] indent       = new byte[0];
+   private String indentString = "";
+
+   private Pattern                     colonLetterPattern    = Pattern.compile(":([A-Z]+)($|[A-Z][a-z])");
+   private Memoization<String, String> fieldLabelMemoization = new Memoization<>();
 
 
 
@@ -69,21 +85,124 @@ public class JsonExportStreamer implements ExportStreamerInterface
     **
     *******************************************************************************/
    @Override
-   public void start(ExportInput exportInput, List<QFieldMetaData> fields, String label) throws QReportingException
+   public void preRun(ReportDestination reportDestination, List<QReportView> views) throws QReportingException
+   {
+      outputStream = reportDestination.getReportOutputStream();
+
+      if(views.size() > 1)
+      {
+         multipleViews = true;
+      }
+
+      if(multipleViews)
+      {
+         try
+         {
+            indentIfPretty(outputStream);
+            outputStream.write('[');
+            newlineIfPretty(outputStream);
+            increaseIndent();
+         }
+         catch(IOException e)
+         {
+            throw (new QReportingException("Error starting report output", e));
+         }
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public void start(ExportInput exportInput, List<QFieldMetaData> fields, String label, QReportView view) throws QReportingException
    {
       this.exportInput = exportInput;
       this.fields = fields;
       table = exportInput.getTable();
-      outputStream = this.exportInput.getReportOutputStream();
+
+      needCommaBeforeRecord = false;
 
       try
       {
+         if(multipleViews)
+         {
+            if(haveStartedAnyViews)
+            {
+               /////////////////////////
+               // close the last view //
+               /////////////////////////
+               newlineIfPretty(outputStream);
+
+               decreaseIndent();
+               indentIfPretty(outputStream);
+               outputStream.write(']');
+               newlineIfPretty(outputStream);
+
+               decreaseIndent();
+               indentIfPretty(outputStream);
+               outputStream.write('}');
+               outputStream.write(',');
+               newlineIfPretty(outputStream);
+            }
+
+            /////////////////////////////////////////////////////////////
+            // open a new view, as an object, with a name & data entry //
+            /////////////////////////////////////////////////////////////
+            indentIfPretty(outputStream);
+            outputStream.write('{');
+            newlineIfPretty(outputStream);
+            increaseIndent();
+
+            indentIfPretty(outputStream);
+            outputStream.write(String.format("""
+               "name":"%s",""", label).getBytes(StandardCharsets.UTF_8));
+            newlineIfPretty(outputStream);
+
+            indentIfPretty(outputStream);
+            outputStream.write("""
+               "data":""".getBytes(StandardCharsets.UTF_8));
+            newlineIfPretty(outputStream);
+         }
+
+         //////////////////////////////////////////////
+         // start the array of entries for this view //
+         //////////////////////////////////////////////
+         indentIfPretty(outputStream);
          outputStream.write('[');
+         increaseIndent();
       }
       catch(IOException e)
       {
          throw (new QReportingException("Error starting report output", e));
       }
+
+      haveStartedAnyViews = true;
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private void increaseIndent()
+   {
+      indent = new byte[indent.length + 3];
+      Arrays.fill(indent, (byte) ' ');
+      indentString = new String(indent);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private void decreaseIndent()
+   {
+      indent = new byte[Math.max(0, indent.length - 3)];
+      Arrays.fill(indent, (byte) ' ');
+      indentString = new String(indent);
    }
 
 
@@ -111,7 +230,7 @@ public class JsonExportStreamer implements ExportStreamerInterface
    {
       try
       {
-         if(needComma)
+         if(needCommaBeforeRecord)
          {
             outputStream.write(',');
          }
@@ -119,21 +238,25 @@ public class JsonExportStreamer implements ExportStreamerInterface
          Map<String, Serializable> mapForJson = new LinkedHashMap<>();
          for(QFieldMetaData field : fields)
          {
-            String labelForJson = StringUtils.lcFirst(field.getLabel().replace(" ", ""));
-            mapForJson.put(labelForJson, qRecord.getValue(field.getName()));
+            mapForJson.put(getLabelForJson(field), qRecord.getValue(field.getName()));
          }
 
          String json = prettyPrint ? JsonUtils.toPrettyJson(mapForJson) : JsonUtils.toJson(mapForJson);
+         if(prettyPrint)
+         {
+            json = json.replaceAll("(?s)\n", "\n" + indentString);
+         }
 
          if(prettyPrint)
          {
             outputStream.write('\n');
          }
 
+         indentIfPretty(outputStream);
          outputStream.write(json.getBytes(StandardCharsets.UTF_8));
 
          outputStream.flush(); // todo - less often?
-         needComma = true;
+         needCommaBeforeRecord = true;
       }
       catch(Exception e)
       {
@@ -143,6 +266,73 @@ public class JsonExportStreamer implements ExportStreamerInterface
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   String getLabelForJson(QFieldMetaData field)
+   {
+      //////////////////////////////////////////////////////////////////////////
+      // memoize, to avoid running these regex/replacements millions of times //
+      //////////////////////////////////////////////////////////////////////////
+      Optional<String> result = fieldLabelMemoization.getResult(field.getName(), fieldName ->
+      {
+         String labelForJson = field.getLabel().replace(" ", "");
+
+         /////////////////////////////////////////////////////////////////////////////
+         // now fix up any field-name-parts after the table: portion of a join name //
+         // lineItem:SKU to become lineItem:sku                                     //
+         // parcel:SLAStatus to become parcel:slaStatus                             //
+         // order:Client to become order:client                                     //
+         /////////////////////////////////////////////////////////////////////////////
+         Matcher allCaps                          = Pattern.compile("^[A-Z]+$").matcher(labelForJson);
+         Matcher startsAllCapsThenNextWordMatcher = Pattern.compile("([A-Z]+)([A-Z][a-z].*)").matcher(labelForJson);
+         Matcher startsOneCapMatcher              = Pattern.compile("([A-Z])(.*)").matcher(labelForJson);
+
+         if(allCaps.matches())
+         {
+            labelForJson = allCaps.replaceAll(m -> m.group().toLowerCase());
+         }
+         else if(startsAllCapsThenNextWordMatcher.matches())
+         {
+            labelForJson = startsAllCapsThenNextWordMatcher.replaceAll(m -> m.group(1).toLowerCase() + m.group(2));
+         }
+         else if(startsOneCapMatcher.matches())
+         {
+            labelForJson = startsOneCapMatcher.replaceAll(m -> m.group(1).toLowerCase() + m.group(2));
+         }
+
+         /////////////////////////////////////////////////////////////////////////////
+         // now fix up any field-name-parts after the table: portion of a join name //
+         // lineItem:SKU to become lineItem:sku                                     //
+         // parcel:SLAStatus to become parcel:slaStatus                             //
+         // order:Client to become order:client                                     //
+         /////////////////////////////////////////////////////////////////////////////
+         Matcher colonThenAllCapsThenEndMatcher      = Pattern.compile("(.*:)([A-Z]+)$").matcher(labelForJson);
+         Matcher colonThenAllCapsThenNextWordMatcher = Pattern.compile("(.*:)([A-Z]+)([A-Z][a-z].*)").matcher(labelForJson);
+         Matcher colonThenOneCapMatcher              = Pattern.compile("(.*:)([A-Z])(.*)").matcher(labelForJson);
+
+         if(colonThenAllCapsThenEndMatcher.matches())
+         {
+            labelForJson = colonThenAllCapsThenEndMatcher.replaceAll(m -> m.group(1) + m.group(2).toLowerCase());
+         }
+         else if(colonThenAllCapsThenNextWordMatcher.matches())
+         {
+            labelForJson = colonThenAllCapsThenNextWordMatcher.replaceAll(m -> m.group(1) + m.group(2).toLowerCase() + m.group(3));
+         }
+         else if(colonThenOneCapMatcher.matches())
+         {
+            labelForJson = colonThenOneCapMatcher.replaceAll(m -> m.group(1) + m.group(2).toLowerCase() + m.group(3));
+         }
+
+         System.out.println("Label: " + labelForJson);
+         return (labelForJson);
+      });
+
+      return result.orElse(field.getName());
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/
@@ -162,11 +352,34 @@ public class JsonExportStreamer implements ExportStreamerInterface
    {
       try
       {
-         if(prettyPrint)
-         {
-            outputStream.write('\n');
-         }
+         //////////////////////////////////////////////
+         // close the array of entries for this view //
+         //////////////////////////////////////////////
+         newlineIfPretty(outputStream);
+
+         decreaseIndent();
+         indentIfPretty(outputStream);
          outputStream.write(']');
+         newlineIfPretty(outputStream);
+
+         if(multipleViews)
+         {
+            ////////////////////////////////////////////
+            // close this view, if there are multiple //
+            ////////////////////////////////////////////
+            decreaseIndent();
+            indentIfPretty(outputStream);
+            outputStream.write('}');
+            newlineIfPretty(outputStream);
+
+            /////////////////////////////
+            // close the list of views //
+            /////////////////////////////
+            decreaseIndent();
+            indentIfPretty(outputStream);
+            outputStream.write(']');
+            newlineIfPretty(outputStream);
+         }
       }
       catch(IOException e)
       {
@@ -174,4 +387,30 @@ public class JsonExportStreamer implements ExportStreamerInterface
       }
    }
 
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private void newlineIfPretty(OutputStream outputStream) throws IOException
+   {
+      if(prettyPrint)
+      {
+         outputStream.write('\n');
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private void indentIfPretty(OutputStream outputStream) throws IOException
+   {
+      if(prettyPrint)
+      {
+         outputStream.write(indent);
+      }
+   }
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/ListOfMapsExportStreamer.java

@@ -31,6 +31,7 @@ import com.kingsrook.qqq.backend.core.logging.QLogger;
 import com.kingsrook.qqq.backend.core.model.actions.reporting.ExportInput;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportView;
 
 
 /*******************************************************************************
@@ -87,7 +88,7 @@ public class ListOfMapsExportStreamer implements ExportStreamerInterface
     **
     *******************************************************************************/
    @Override
-   public void start(ExportInput exportInput, List<QFieldMetaData> fields, String label) throws QReportingException
+   public void start(ExportInput exportInput, List<QFieldMetaData> fields, String label, QReportView view) throws QReportingException
    {
       this.exportInput = exportInput;
       this.fields = fields;

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/RecordPipe.java

@@ -24,6 +24,7 @@ package com.kingsrook.qqq.backend.core.actions.reporting;
 
 import java.util.ArrayList;
 import java.util.List;
+import java.util.Objects;
 import java.util.concurrent.ArrayBlockingQueue;
 import java.util.concurrent.TimeUnit;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
@@ -43,8 +44,9 @@ public class RecordPipe
 
    private static final long BLOCKING_SLEEP_MILLIS = 100;
    private static final long MAX_SLEEP_LOOP_MILLIS = 300_000; // 5 minutes
+   private static final int  DEFAULT_CAPACITY      = 1_000;
 
-   private int capacity = 1_000;
+   private int capacity = DEFAULT_CAPACITY;
    private ArrayBlockingQueue<QRecord> queue = new ArrayBlockingQueue<>(capacity);
 
    private boolean isTerminated = false;
@@ -70,11 +72,13 @@ public class RecordPipe
 
    /*******************************************************************************
     ** Construct a record pipe, with an alternative capacity for the internal queue.
+    **
+    ** overrideCapacity is allowed to be null - in which case, DEFAULT_CAPACITY is used.
     *******************************************************************************/
    public RecordPipe(Integer overrideCapacity)
    {
-      this.capacity = overrideCapacity;
-      queue = new ArrayBlockingQueue<>(overrideCapacity);
+      this.capacity = Objects.requireNonNullElse(overrideCapacity, DEFAULT_CAPACITY);
+      queue = new ArrayBlockingQueue<>(this.capacity);
    }
 
 
@@ -138,7 +142,7 @@ public class RecordPipe
          {
             if(now - sleepLoopStartTime > MAX_SLEEP_LOOP_MILLIS)
             {
-               LOG.warn("Giving up adding record to pipe, due to pipe being full for more than {} millis", MAX_SLEEP_LOOP_MILLIS);
+               LOG.warn("Giving up adding record to pipe, due to pipe being full for more than " + MAX_SLEEP_LOOP_MILLIS + " millis");
                throw (new IllegalStateException("Giving up adding record to pipe, due to pipe staying full too long."));
             }
             LOG.trace("Record pipe.add failed (due to full pipe).  Blocking.");

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/ReportUtils.java

@@ -0,0 +1,51 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.actions.reporting;
+
+
+import java.util.List;
+import java.util.Optional;
+import com.kingsrook.qqq.backend.core.exceptions.QReportingException;
+import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportView;
+
+
+/*******************************************************************************
+ **
+ *******************************************************************************/
+public class ReportUtils
+{
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public static QReportView getSourceViewForPivotTableView(List<QReportView> views, QReportView pivotTableView) throws QReportingException
+   {
+      Optional<QReportView> sourceView = views.stream().filter(v -> v.getName().equals(pivotTableView.getPivotTableSourceViewName())).findFirst();
+      if(sourceView.isEmpty())
+      {
+         throw (new QReportingException("Could not find data view [" + pivotTableView.getPivotTableSourceViewName() + "] for pivot table view [" + pivotTableView.getName() + "]"));
+      }
+
+      return sourceView.get();
+   }
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/excel/fastexcel/BoldHeaderAndFooterFastExcelStyler.java

@@ -1,6 +1,6 @@
 /*
  * QQQ - Low-code Application Framework for Engineers.
- * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
  * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
  * contact@kingsrook.com
  * https://github.com/Kingsrook/
@@ -19,7 +19,7 @@
  * along with this program.  If not, see <https://www.gnu.org/licenses/>.
  */
 
-package com.kingsrook.qqq.backend.core.actions.reporting.excelformatting;
+package com.kingsrook.qqq.backend.core.actions.reporting.excel.fastexcel;
 
 
 import org.dhatim.fastexcel.BorderSide;
@@ -30,7 +30,7 @@ import org.dhatim.fastexcel.StyleSetter;
 /*******************************************************************************
  ** Version of excel styler that does bold headers and footers, with basic borders.
  *******************************************************************************/
-public class BoldHeaderAndFooterExcelStyler implements ExcelStylerInterface
+public class BoldHeaderAndFooterFastExcelStyler implements FastExcelStylerInterface
 {
 
    /*******************************************************************************
@@ -60,6 +60,9 @@ public class BoldHeaderAndFooterExcelStyler implements ExcelStylerInterface
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
    @Override
    public void styleTotalsRow(StyleSetter totalsRowStyle)
    {

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/excel/fastexcel/ExcelFastexcelExportStreamer.java

@@ -1,6 +1,6 @@
 /*
  * QQQ - Low-code Application Framework for Engineers.
- * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
  * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
  * contact@kingsrook.com
  * https://github.com/Kingsrook/
@@ -19,7 +19,7 @@
  * along with this program.  If not, see <https://www.gnu.org/licenses/>.
  */
 
-package com.kingsrook.qqq.backend.core.actions.reporting;
+package com.kingsrook.qqq.backend.core.actions.reporting.excel.fastexcel;
 
 
 import java.io.OutputStream;
@@ -34,8 +34,8 @@ import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
-import com.kingsrook.qqq.backend.core.actions.reporting.excelformatting.ExcelStylerInterface;
-import com.kingsrook.qqq.backend.core.actions.reporting.excelformatting.PlainExcelStyler;
+import com.kingsrook.qqq.backend.core.actions.reporting.ExportStreamerInterface;
+import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.exceptions.QReportingException;
 import com.kingsrook.qqq.backend.core.logging.QLogger;
 import com.kingsrook.qqq.backend.core.model.actions.reporting.ExportInput;
@@ -43,7 +43,9 @@ import com.kingsrook.qqq.backend.core.model.data.QRecord;
 import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.DisplayFormat;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportView;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+import com.kingsrook.qqq.backend.core.utils.ObjectUtils;
 import com.kingsrook.qqq.backend.core.utils.StringUtils;
 import com.kingsrook.qqq.backend.core.utils.ValueUtils;
 import org.dhatim.fastexcel.StyleSetter;
@@ -52,19 +54,19 @@ import org.dhatim.fastexcel.Worksheet;
 
 
 /*******************************************************************************
- ** Excel export format implementation
+ ** Excel export format implementation - built using fastexcel library
  *******************************************************************************/
-public class ExcelExportStreamer implements ExportStreamerInterface
+public class ExcelFastexcelExportStreamer implements ExportStreamerInterface
 {
-   private static final QLogger LOG = QLogger.getLogger(ExcelExportStreamer.class);
+   private static final QLogger LOG = QLogger.getLogger(ExcelFastexcelExportStreamer.class);
 
    private ExportInput          exportInput;
    private QTableMetaData       table;
    private List<QFieldMetaData> fields;
    private OutputStream         outputStream;
 
-   private ExcelStylerInterface excelStylerInterface = new PlainExcelStyler();
-   private Map<String, String>  excelCellFormats;
+   private FastExcelStylerInterface fastExcelStylerInterface = new PlainFastExcelStyler();
+   private Map<String, String>      excelCellFormats;
 
    private Workbook  workbook;
    private Worksheet worksheet;
@@ -76,7 +78,7 @@ public class ExcelExportStreamer implements ExportStreamerInterface
    /*******************************************************************************
     **
     *******************************************************************************/
-   public ExcelExportStreamer()
+   public ExcelFastexcelExportStreamer()
    {
    }
 
@@ -105,14 +107,14 @@ public class ExcelExportStreamer implements ExportStreamerInterface
     ** Starts a new worksheet in the current workbook.  Can be called multiple times.
     *******************************************************************************/
    @Override
-   public void start(ExportInput exportInput, List<QFieldMetaData> fields, String label) throws QReportingException
+   public void start(ExportInput exportInput, List<QFieldMetaData> fields, String label, QReportView view) throws QReportingException
    {
       try
       {
          this.exportInput = exportInput;
          this.fields = fields;
          table = exportInput.getTable();
-         outputStream = this.exportInput.getReportOutputStream();
+         outputStream = this.exportInput.getReportDestination().getReportOutputStream();
          this.row = 0;
          this.sheetCount++;
 
@@ -121,8 +123,8 @@ public class ExcelExportStreamer implements ExportStreamerInterface
          /////////////////////////////////////////////////////////////////////////////////////////////////////
          if(workbook == null)
          {
-            String    appName  = "QQQ";
-            QInstance instance = exportInput.getInstance();
+            String    appName = ObjectUtils.tryAndRequireNonNullElse(() -> QContext.getQInstance().getBranding().getAppName(), "QQQ");
+            QInstance instance = QContext.getQInstance();
             if(instance != null && instance.getBranding() != null && instance.getBranding().getCompanyName() != null)
             {
                appName = instance.getBranding().getCompanyName();
@@ -167,7 +169,7 @@ public class ExcelExportStreamer implements ExportStreamerInterface
             worksheet.range(row, 0, row, fields.size() - 1).merge();
 
             StyleSetter titleStyle = worksheet.range(row, 0, row, fields.size() - 1).style();
-            excelStylerInterface.styleTitleRow(titleStyle);
+            fastExcelStylerInterface.styleTitleRow(titleStyle);
             titleStyle.set();
 
             row++;
@@ -187,7 +189,7 @@ public class ExcelExportStreamer implements ExportStreamerInterface
             }
 
             StyleSetter headerStyle = worksheet.range(row, 0, row, fields.size() - 1).style();
-            excelStylerInterface.styleHeaderRow(headerStyle);
+            fastExcelStylerInterface.styleHeaderRow(headerStyle);
             headerStyle.set();
 
             row++;
@@ -315,7 +317,7 @@ public class ExcelExportStreamer implements ExportStreamerInterface
       writeRecord(record);
 
       StyleSetter totalsRowStyle = worksheet.range(row, 0, row, fields.size() - 1).style();
-      excelStylerInterface.styleTotalsRow(totalsRowStyle);
+      fastExcelStylerInterface.styleTotalsRow(totalsRowStyle);
       totalsRowStyle.set();
    }
 

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/excel/fastexcel/FastExcelStylerInterface.java

@@ -1,6 +1,6 @@
 /*
  * QQQ - Low-code Application Framework for Engineers.
- * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
  * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
  * contact@kingsrook.com
  * https://github.com/Kingsrook/
@@ -19,7 +19,7 @@
  * along with this program.  If not, see <https://www.gnu.org/licenses/>.
  */
 
-package com.kingsrook.qqq.backend.core.actions.reporting.excelformatting;
+package com.kingsrook.qqq.backend.core.actions.reporting.excel.fastexcel;
 
 
 import org.dhatim.fastexcel.StyleSetter;
@@ -29,7 +29,7 @@ import org.dhatim.fastexcel.StyleSetter;
  ** Interface for classes that know how to apply styles to an Excel stream being
  ** built by fastexcel.
  *******************************************************************************/
-public interface ExcelStylerInterface
+public interface FastExcelStylerInterface
 {
 
    /*******************************************************************************

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/excel/fastexcel/PlainFastExcelStyler.java

@@ -0,0 +1,43 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.actions.reporting.excel.fastexcel;
+
+
+import org.dhatim.fastexcel.StyleSetter;
+
+
+/*******************************************************************************
+ ** Excel styler that does nothing - just takes defaults (which are all no-op) from the interface.
+ *******************************************************************************/
+public class PlainFastExcelStyler implements FastExcelStylerInterface
+{
+
+   /*******************************************************************************
+    ** ... sorry, but adding this gives us test coverage on this class, even though
+    ** we're just deferring to super...
+    *******************************************************************************/
+   @Override
+   public void styleHeaderRow(StyleSetter headerRowStyle)
+   {
+      FastExcelStylerInterface.super.styleHeaderRow(headerRowStyle);
+   }
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/excel/poi/BoldHeaderAndFooterPoiExcelStyler.java

@@ -0,0 +1,93 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.actions.reporting.excel.poi;
+
+
+import org.apache.poi.ss.usermodel.BorderStyle;
+import org.apache.poi.ss.usermodel.CreationHelper;
+import org.apache.poi.ss.usermodel.Font;
+import org.apache.poi.ss.usermodel.HorizontalAlignment;
+import org.apache.poi.xssf.usermodel.XSSFCellStyle;
+import org.apache.poi.xssf.usermodel.XSSFWorkbook;
+
+
+/*******************************************************************************
+ ** Version of POI excel styler that does bold headers and footers, with basic borders.
+ *******************************************************************************/
+public class BoldHeaderAndFooterPoiExcelStyler implements PoiExcelStylerInterface
+{
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public XSSFCellStyle createStyleForTitle(XSSFWorkbook workbook, CreationHelper createHelper)
+   {
+      Font font = workbook.createFont();
+      font.setFontHeightInPoints((short) 14);
+      font.setBold(true);
+
+      XSSFCellStyle cellStyle = workbook.createCellStyle();
+      cellStyle.setFont(font);
+      cellStyle.setAlignment(HorizontalAlignment.CENTER);
+
+      return (cellStyle);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public XSSFCellStyle createStyleForHeader(XSSFWorkbook workbook, CreationHelper createHelper)
+   {
+      Font font = workbook.createFont();
+      font.setBold(true);
+
+      XSSFCellStyle cellStyle = workbook.createCellStyle();
+      cellStyle.setFont(font);
+      cellStyle.setBorderBottom(BorderStyle.THIN);
+
+      return (cellStyle);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public XSSFCellStyle createStyleForFooter(XSSFWorkbook workbook, CreationHelper createHelper)
+   {
+      Font font = workbook.createFont();
+      font.setBold(true);
+
+      XSSFCellStyle cellStyle = workbook.createCellStyle();
+      cellStyle.setFont(font);
+      cellStyle.setBorderTop(BorderStyle.THIN);
+      cellStyle.setBorderBottom(BorderStyle.DOUBLE);
+
+      return (cellStyle);
+   }
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/excel/poi/ExcelPoiBasedStreamingExportStreamer.java

@@ -0,0 +1,819 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.actions.reporting.excel.poi;
+
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.io.OutputStreamWriter;
+import java.io.Serializable;
+import java.io.Writer;
+import java.time.Instant;
+import java.time.LocalDate;
+import java.time.LocalDateTime;
+import java.time.ZoneId;
+import java.time.ZonedDateTime;
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipInputStream;
+import java.util.zip.ZipOutputStream;
+import com.kingsrook.qqq.backend.core.actions.reporting.ExportStreamerInterface;
+import com.kingsrook.qqq.backend.core.actions.reporting.ReportUtils;
+import com.kingsrook.qqq.backend.core.exceptions.QReportingException;
+import com.kingsrook.qqq.backend.core.instances.QInstanceEnricher;
+import com.kingsrook.qqq.backend.core.logging.QLogger;
+import com.kingsrook.qqq.backend.core.model.actions.reporting.ExportInput;
+import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportDestination;
+import com.kingsrook.qqq.backend.core.model.actions.reporting.pivottable.PivotTableGroupBy;
+import com.kingsrook.qqq.backend.core.model.actions.reporting.pivottable.PivotTableValue;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.DisplayFormat;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
+import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportField;
+import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportView;
+import com.kingsrook.qqq.backend.core.model.metadata.reporting.ReportType;
+import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
+import com.kingsrook.qqq.backend.core.utils.StringUtils;
+import com.kingsrook.qqq.backend.core.utils.ValueUtils;
+import org.apache.poi.ss.SpreadsheetVersion;
+import org.apache.poi.ss.usermodel.CreationHelper;
+import org.apache.poi.ss.usermodel.DataConsolidateFunction;
+import org.apache.poi.ss.usermodel.DateUtil;
+import org.apache.poi.ss.util.AreaReference;
+import org.apache.poi.ss.util.CellReference;
+import org.apache.poi.ss.util.WorkbookUtil;
+import org.apache.poi.xssf.usermodel.XSSFCell;
+import org.apache.poi.xssf.usermodel.XSSFCellStyle;
+import org.apache.poi.xssf.usermodel.XSSFPivotTable;
+import org.apache.poi.xssf.usermodel.XSSFRow;
+import org.apache.poi.xssf.usermodel.XSSFSheet;
+import org.apache.poi.xssf.usermodel.XSSFWorkbook;
+
+
+/*******************************************************************************
+ ** Excel export format implementation using POI library, but with modifications
+ ** to actually stream output rather than use any temp files.
+ **
+ ** For a rough outline:
+ ** - create a basically empty Excel workbook using POI - empty meaning, without
+ ** data rows.
+ ** - have POI write that workbook out into a byte[] - which will be a zip full
+ ** of xml (e.g., xlsx).
+ ** - then open a new ZipOutputStream wrapper around the OutputStream we took in
+ ** as the report destination (e.g., streamed into storage or http output)
+ ** - Copy over all entries from the xlsx into our new zip-output-stream, other than
+ ** ones that are the actual sheets that we want to put data into.
+ ** - For the sheet entries, use the StreamedPoiSheetWriter class to write the
+ ** report's data directly out as Excel XML (not using POI).
+ ** - Pivot tables require a bit of an additional hack - to write a "pivot cache
+ ** definition", which, while we won't put all the data in it (because we'll tell
+ ** it refreshOnLoad="true"), we will at least need to know how many cols & rows
+ ** are in the data-sheet (which we wouldn't know until we streamed that sheet!)
+ *******************************************************************************/
+public class ExcelPoiBasedStreamingExportStreamer implements ExportStreamerInterface
+{
+   private static final QLogger LOG = QLogger.getLogger(ExcelPoiBasedStreamingExportStreamer.class);
+
+   private List<QReportView>    views;
+   private ExportInput          exportInput;
+   private List<QFieldMetaData> fields;
+   private OutputStream         outputStream;
+   private ZipOutputStream      zipOutputStream;
+
+   public static final String EXCEL_DATE_FORMAT      = "yyyy-MM-dd";
+   public static final String EXCEL_DATE_TIME_FORMAT = "yyyy-MM-dd H:mm:ss";
+
+   private PoiExcelStylerInterface    poiExcelStylerInterface = getStylerInterface();
+   private Map<String, String>        excelCellFormats;
+   private Map<String, XSSFCellStyle> styles                  = new HashMap<>();
+
+   private int rowNo      = 0;
+   private int sheetIndex = 1;
+
+   private Map<String, String> pivotViewToCacheDefinitionReferenceMap = new HashMap<>();
+
+   private Writer              activeSheetWriter = null;
+   private StreamedSheetWriter sheetWriter       = null;
+
+   private QReportView                       currentView      = null;
+   private Map<String, List<QFieldMetaData>> fieldsPerView    = new HashMap<>();
+   private Map<String, Integer>              rowsPerView      = new HashMap<>();
+   private Map<String, String>               labelViewsByName = new HashMap<>();
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public ExcelPoiBasedStreamingExportStreamer()
+   {
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public void preRun(ReportDestination reportDestination, List<QReportView> views) throws QReportingException
+   {
+      try
+      {
+         this.outputStream = reportDestination.getReportOutputStream();
+         this.views = views;
+
+         ///////////////////////////////////////////////////////////////////////////////
+         // create 'template' workbook through poi - with sheets corresponding to our //
+         // actual file this will be a zip file (stream), with entries for all of the //
+         // files in the final xlsx but without any data, so it'll be small           //
+         ///////////////////////////////////////////////////////////////////////////////
+         XSSFWorkbook workbook = new XSSFWorkbook();
+         createStyles(workbook);
+
+         //////////////////////////////////////////////////////////////////////////////////////////////////
+         // for each of the sheets, create it in the workbook, and put a reference to it in the sheetMap //
+         //////////////////////////////////////////////////////////////////////////////////////////////////
+         Map<String, XSSFSheet> sheetMapByExcelReference = new HashMap<>();
+         Map<String, XSSFSheet> sheetMapByViewName       = new HashMap<>();
+
+         int sheetCounter = 1;
+         for(QReportView view : views)
+         {
+            String label = Objects.requireNonNullElse(view.getLabel(), "Sheet " + sheetCounter);
+            label = WorkbookUtil.createSafeSheetName(label);
+
+            /////////////////////////////////////////////////////////////////////////////////////////////
+            // track the actually-used sheet labels (needed for referencing in pivot table generation) //
+            /////////////////////////////////////////////////////////////////////////////////////////////
+            labelViewsByName.put(view.getName(), label);
+
+            XSSFSheet sheet          = workbook.createSheet(label);
+            String    sheetReference = sheet.getPackagePart().getPartName().getName().substring(1);
+            sheetMapByExcelReference.put(sheetReference, sheet);
+            sheetMapByViewName.put(view.getName(), sheet);
+            sheetCounter++;
+         }
+
+         ////////////////////////////////////////////////////
+         // if any views are pivot tables, create them now //
+         ////////////////////////////////////////////////////
+         List<String> pivotViewNames = new ArrayList<>();
+         for(QReportView view : views)
+         {
+            if(ReportType.PIVOT.equals(view.getType()))
+            {
+               pivotViewNames.add(view.getName());
+
+               XSSFSheet   pivotTableSheet = Objects.requireNonNull(sheetMapByViewName.get(view.getName()), "Could not get pivot table sheet view by name: " + view.getName());
+               XSSFSheet   dataSheet       = Objects.requireNonNull(sheetMapByViewName.get(view.getPivotTableSourceViewName()), "Could not get pivot table source sheet by view name: " + view.getPivotTableSourceViewName());
+               QReportView dataView        = ReportUtils.getSourceViewForPivotTableView(views, view);
+               createPivotTableTemplate(pivotTableSheet, view, dataSheet, dataView);
+            }
+         }
+         Iterator<String> pivotViewNameIterator = pivotViewNames.iterator();
+
+         /////////////////////////////////////////////////////////
+         // write that template worksheet zip out to byte array //
+         /////////////////////////////////////////////////////////
+         ByteArrayOutputStream templateBAOS = new ByteArrayOutputStream();
+         workbook.write(templateBAOS);
+         templateBAOS.close();
+         byte[] templateBytes = templateBAOS.toByteArray();
+
+         /////////////////////////////////////////////////////////////////////////////////////////////
+         // open up a zipOutputStream around the output stream that the report is to be written to. //
+         // note:  this stream is closed in the finish method - see more comments there.            //
+         /////////////////////////////////////////////////////////////////////////////////////////////
+         this.zipOutputStream = new ZipOutputStream(this.outputStream);
+
+         /////////////////////////////////////////////////////////////////////////////////////////////////
+         // copy over all the entries in the template zip that aren't the sheets into the output stream //
+         /////////////////////////////////////////////////////////////////////////////////////////////////
+         ZipInputStream zipInputStream   = new ZipInputStream(new ByteArrayInputStream(templateBytes));
+         ZipEntry       zipTemplateEntry = null;
+         byte[]         buffer           = new byte[2048];
+         while((zipTemplateEntry = zipInputStream.getNextEntry()) != null)
+         {
+            if(zipTemplateEntry.getName().matches(".*/pivotCacheDefinition.*.xml"))
+            {
+               //////////////////////////////////////////////////////////////////////////////////////////////////////
+               // if this zip entry is a pivotCacheDefinition, then don't write it to the output stream right now. //
+               // instead, just map the pivot view's name to the zipTemplateEntry name                             //
+               //////////////////////////////////////////////////////////////////////////////////////////////////////
+               if(!pivotViewNameIterator.hasNext())
+               {
+                  throw new QReportingException("Found a pivot cache definition [" + zipTemplateEntry.getName() + "] in the template ZIP, but no (more) corresponding pivot view names");
+               }
+
+               String pivotViewName = pivotViewNameIterator.next();
+               LOG.info("Holding on a pivot cache definition zip template entry [" + pivotViewName + "] [" + zipTemplateEntry.getName() + "]...");
+               pivotViewToCacheDefinitionReferenceMap.put(pivotViewName, zipTemplateEntry.getName());
+            }
+            else if(!sheetMapByExcelReference.containsKey(zipTemplateEntry.getName()))
+            {
+               ////////////////////////////////////////////////////////////////////////////////////////////////////////////
+               // else, if we don't have this zipTemplateEntry name in our map of sheets, then this is a kinda "meta"    //
+               // file that we don't really care about (e.g., not our sheet data), so just copy it to the output stream. //
+               ////////////////////////////////////////////////////////////////////////////////////////////////////////////
+               LOG.info("Copying zip template entry [" + zipTemplateEntry.getName() + "] to output stream");
+               zipOutputStream.putNextEntry(new ZipEntry(zipTemplateEntry.getName()));
+
+               int length;
+               while((length = zipInputStream.read(buffer)) > 0)
+               {
+                  zipOutputStream.write(buffer, 0, length);
+               }
+
+               zipInputStream.closeEntry();
+            }
+            else
+            {
+               ////////////////////////////////////////////////////////////////////////////////////
+               // else - this is a sheet - so again, don't write it yet - stream its data below. //
+               ////////////////////////////////////////////////////////////////////////////////////
+               LOG.info("Skipping presumed sheet zip template entry [" + zipTemplateEntry.getName() + "] to output stream");
+            }
+         }
+
+         zipInputStream.close();
+      }
+      catch(Exception e)
+      {
+         throw (new QReportingException("Error preparing to generate spreadsheet", e));
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private void createPivotTableTemplate(XSSFSheet pivotTableSheet, QReportView pivotView, XSSFSheet dataSheet, QReportView dataView) throws QReportingException
+   {
+      /////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      // write just enough data to the dataView's sheet so that we can refer to it for creating the pivot table. //
+      // we need to do this, because POI will try to create the pivotCache referring to the data sheet, and if   //
+      // there isn't any data there, it'll crash.                                                                //
+      /////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      XSSFRow headerRow = dataSheet.createRow(0);
+      int     columnNo  = 0;
+      for(QReportField column : dataView.getColumns())
+      {
+         XSSFCell cell = headerRow.createCell(columnNo++);
+         cell.setCellValue(QInstanceEnricher.nameToLabel(column.getName()));
+      }
+
+      XSSFRow valuesRow = dataSheet.createRow(1);
+      columnNo = 0;
+      for(QReportField column : dataView.getColumns())
+      {
+         XSSFCell cell = valuesRow.createCell(columnNo++);
+         cell.setCellValue("Value " + columnNo);
+      }
+
+      /////////////////////////////////////////////////////////////////////////////////////////////////////
+      // for this template version of the pivot table, tell it there are only 2 rows in the source sheet //
+      // as that's all that we wrote above (a header and 1 fake value row)                               //
+      /////////////////////////////////////////////////////////////////////////////////////////////////////
+      int           rows       = 2;
+      String        colsLetter = CellReference.convertNumToColString(dataView.getColumns().size() - 1);
+      AreaReference source     = new AreaReference("A1:" + colsLetter + rows, SpreadsheetVersion.EXCEL2007);
+      CellReference position   = new CellReference("A1");
+
+      //////////////////////////////////////////////////////////////////
+      // tell poi all about our pivot table - rows, cols, and columns //
+      //////////////////////////////////////////////////////////////////
+      XSSFPivotTable pivotTable = pivotTableSheet.createPivotTable(source, position, dataSheet);
+
+      for(PivotTableGroupBy row : CollectionUtils.nonNullList(pivotView.getPivotTableDefinition().getRows()))
+      {
+         int rowLabelColumnIndex = getColumnIndex(dataView.getColumns(), row.getFieldName());
+         pivotTable.addRowLabel(rowLabelColumnIndex);
+      }
+
+      for(PivotTableGroupBy column : CollectionUtils.nonNullList(pivotView.getPivotTableDefinition().getColumns()))
+      {
+         int colLabelColumnIndex = getColumnIndex(dataView.getColumns(), column.getFieldName());
+         pivotTable.addColLabel(colLabelColumnIndex);
+      }
+
+      for(PivotTableValue value : CollectionUtils.nonNullList(pivotView.getPivotTableDefinition().getValues()))
+      {
+         int columnLabelColumnIndex = getColumnIndex(dataView.getColumns(), value.getFieldName());
+
+         String labelPrefix = value.getFunction().name() + " of ";
+         String label       = labelPrefix + QInstanceEnricher.nameToLabel(value.getFieldName());
+         String valueFormat = null;
+
+         Optional<QReportField> optSourceField = dataView.getColumns().stream().filter(c -> c.getName().equals(value.getFieldName())).findFirst();
+         if(optSourceField.isPresent())
+         {
+            QReportField sourceField = optSourceField.get();
+
+            if(StringUtils.hasContent(sourceField.getLabel()))
+            {
+               label = labelPrefix + sourceField.getLabel();
+            }
+
+            if(StringUtils.hasContent(sourceField.getDisplayFormat()))
+            {
+               valueFormat = DisplayFormat.getExcelFormat(sourceField.getDisplayFormat());
+            }
+            else
+            {
+               if(QFieldType.DATE.equals(sourceField.getType()))
+               {
+                  valueFormat = EXCEL_DATE_FORMAT;
+               }
+               else if(QFieldType.DATE_TIME.equals(sourceField.getType()))
+               {
+                  valueFormat = EXCEL_DATE_TIME_FORMAT;
+               }
+            }
+         }
+
+         pivotTable.addColumnLabel(DataConsolidateFunction.valueOf(value.getFunction().name()), columnLabelColumnIndex, label, valueFormat);
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private int getColumnIndex(List<QReportField> columns, String fieldName) throws QReportingException
+   {
+      for(int i = 0; i < columns.size(); i++)
+      {
+         if(columns.get(i).getName().equals(fieldName))
+         {
+            return (i);
+         }
+      }
+
+      throw (new QReportingException("Could not find column by name [" + fieldName + "]"));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private void createStyles(XSSFWorkbook workbook)
+   {
+      CreationHelper createHelper = workbook.getCreationHelper();
+
+      XSSFCellStyle dateStyle = workbook.createCellStyle();
+      dateStyle.setDataFormat(createHelper.createDataFormat().getFormat(EXCEL_DATE_FORMAT));
+      styles.put("date", dateStyle);
+
+      XSSFCellStyle dateTimeStyle = workbook.createCellStyle();
+      dateTimeStyle.setDataFormat(createHelper.createDataFormat().getFormat(EXCEL_DATE_TIME_FORMAT));
+      styles.put("datetime", dateTimeStyle);
+
+      styles.put("title", poiExcelStylerInterface.createStyleForTitle(workbook, createHelper));
+      styles.put("header", poiExcelStylerInterface.createStyleForHeader(workbook, createHelper));
+      styles.put("footer", poiExcelStylerInterface.createStyleForFooter(workbook, createHelper));
+
+      XSSFCellStyle footerDateStyle = poiExcelStylerInterface.createStyleForFooter(workbook, createHelper);
+      footerDateStyle.setDataFormat(createHelper.createDataFormat().getFormat(EXCEL_DATE_FORMAT));
+      styles.put("footer-date", footerDateStyle);
+
+      XSSFCellStyle footerDateTimeStyle = poiExcelStylerInterface.createStyleForFooter(workbook, createHelper);
+      footerDateTimeStyle.setDataFormat(createHelper.createDataFormat().getFormat(EXCEL_DATE_TIME_FORMAT));
+      styles.put("footer-datetime", footerDateTimeStyle);
+   }
+
+
+
+   /*******************************************************************************
+    ** Starts a new worksheet in the current workbook.  Can be called multiple times.
+    *******************************************************************************/
+   @Override
+   public void start(ExportInput exportInput, List<QFieldMetaData> fields, String label, QReportView view) throws QReportingException
+   {
+      try
+      {
+         /////////////////////////////////////////
+         // close previous sheet if one is open //
+         /////////////////////////////////////////
+         closeLastSheetIfOpen();
+
+         if(currentView != null)
+         {
+            this.rowsPerView.put(currentView.getName(), rowNo);
+         }
+
+         this.currentView = view;
+         this.exportInput = exportInput;
+         this.fields = fields;
+         this.rowNo = 0;
+
+         this.fieldsPerView.put(view.getName(), fields);
+
+         //////////////////////////////////////////
+         // start the new sheet as:              //
+         // - a new entry in the zipOutputStream //
+         // - with a new output stream writer    //
+         // - and with a SpreadsheetWriter       //
+         //////////////////////////////////////////
+         zipOutputStream.putNextEntry(new ZipEntry("xl/worksheets/sheet" + this.sheetIndex++ + ".xml"));
+         activeSheetWriter = new OutputStreamWriter(zipOutputStream);
+         sheetWriter = new StreamedSheetWriter(activeSheetWriter);
+
+         if(ReportType.PIVOT.equals(view.getType()))
+         {
+            writePivotTable(view, ReportUtils.getSourceViewForPivotTableView(views, view));
+         }
+         else
+         {
+            sheetWriter.beginSheet();
+
+            ////////////////////////////////////////////////
+            // put the title and header rows in the sheet //
+            ////////////////////////////////////////////////
+            writeTitleAndHeader();
+         }
+      }
+      catch(Exception e)
+      {
+         throw (new QReportingException("Error starting worksheet", e));
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private void writeTitleAndHeader() throws QReportingException
+   {
+      try
+      {
+         ///////////////
+         // title row //
+         ///////////////
+         if(StringUtils.hasContent(exportInput.getTitleRow()))
+         {
+            sheetWriter.insertRow(rowNo++);
+            sheetWriter.createCell(0, exportInput.getTitleRow(), styles.get("title").getIndex());
+            sheetWriter.endRow();
+         }
+
+         ////////////////
+         // header row //
+         ////////////////
+         if(exportInput.getIncludeHeaderRow())
+         {
+            sheetWriter.insertRow(rowNo++);
+            XSSFCellStyle headerStyle = styles.get("header");
+
+            int col = 0;
+            for(QFieldMetaData column : fields)
+            {
+               sheetWriter.createCell(col, column.getLabel(), headerStyle.getIndex());
+               col++;
+            }
+
+            sheetWriter.endRow();
+         }
+      }
+      catch(Exception e)
+      {
+         throw (new QReportingException("Error starting Excel report"));
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public void addRecords(List<QRecord> qRecords) throws QReportingException
+   {
+      LOG.info("Consuming [" + qRecords.size() + "] records from the pipe");
+
+      try
+      {
+         for(QRecord qRecord : qRecords)
+         {
+            writeRecord(qRecord);
+         }
+      }
+      catch(Exception e)
+      {
+         LOG.error("Exception generating excel file", e);
+         try
+         {
+            outputStream.close();
+         }
+         catch(IOException ex)
+         {
+            LOG.warn("Secondary error closing excel output stream", e);
+         }
+
+         throw (new QReportingException("Error generating Excel report", e));
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private void writeRecord(QRecord qRecord) throws IOException
+   {
+      writeRecord(qRecord, false);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private void writeRecord(QRecord qRecord, boolean isFooter) throws IOException
+   {
+      sheetWriter.insertRow(rowNo++);
+
+      int styleIndex         = -1;
+      int dateStyleIndex     = styles.get("date").getIndex();
+      int dateTimeStyleIndex = styles.get("datetime").getIndex();
+      if(isFooter)
+      {
+         styleIndex = styles.get("footer").getIndex();
+         dateStyleIndex = styles.get("footer-date").getIndex();
+         dateTimeStyleIndex = styles.get("footer-datetime").getIndex();
+      }
+
+      int col = 0;
+      for(QFieldMetaData field : fields)
+      {
+         Serializable value = qRecord.getValue(field.getName());
+
+         if(value != null)
+         {
+            if(value instanceof String s)
+            {
+               sheetWriter.createCell(col, s, styleIndex);
+            }
+            else if(value instanceof Number n)
+            {
+               sheetWriter.createCell(col, n.doubleValue(), styleIndex);
+
+               if(excelCellFormats != null)
+               {
+                  String format = excelCellFormats.get(field.getName());
+                  if(format != null)
+                  {
+                     ////////////////////////////////////////////////////////////////////////////////////////////
+                     // todo - so - for this streamed/zip approach, we need to know all styles before we start //
+                     // any sheets.  but, right now Report action only calls us with per-sheet styles when     //
+                     // it's starting individual sheets.  so, we can't quite support this at this time.        //
+                     // "just" need to change GenerateReportAction to look up all cell formats for all sheets  //
+                     // before preRun is called... and change all existing streamer classes to handle that too //
+                     ////////////////////////////////////////////////////////////////////////////////////////////
+                     // worksheet.style(rowNo, col).format(format).set();
+                  }
+               }
+            }
+            else if(value instanceof Boolean b)
+            {
+               sheetWriter.createCell(col, b, styleIndex);
+            }
+            else if(value instanceof Date d)
+            {
+               sheetWriter.createCell(col, DateUtil.getExcelDate(d), dateStyleIndex);
+            }
+            else if(value instanceof LocalDate d)
+            {
+               sheetWriter.createCell(col, DateUtil.getExcelDate(d), dateStyleIndex);
+            }
+            else if(value instanceof LocalDateTime d)
+            {
+               sheetWriter.createCell(col, DateUtil.getExcelDate(d), dateStyleIndex);
+            }
+            else if(value instanceof ZonedDateTime d)
+            {
+               sheetWriter.createCell(col, DateUtil.getExcelDate(d.toLocalDateTime()), dateTimeStyleIndex);
+            }
+            else if(value instanceof Instant i)
+            {
+               sheetWriter.createCell(col, DateUtil.getExcelDate(i.atZone(ZoneId.systemDefault()).toLocalDateTime()), dateTimeStyleIndex);
+            }
+            else
+            {
+               sheetWriter.createCell(col, ValueUtils.getValueAsString(value), styleIndex);
+            }
+         }
+
+         col++;
+      }
+
+      sheetWriter.endRow();
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public void addTotalsRow(QRecord record) throws QReportingException
+   {
+      try
+      {
+         writeRecord(record, true);
+      }
+      catch(Exception e)
+      {
+         throw (new QReportingException("Error adding totals row", e));
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public void finish() throws QReportingException
+   {
+      try
+      {
+         //////////////////////////////////////////////
+         // close the last open sheet if one is open //
+         //////////////////////////////////////////////
+         closeLastSheetIfOpen();
+
+         /////////////////////////////////////////////////////////////////////////////////////
+         // so, we DO need a close here, on the zipOutputStream, to finish its "zippiness". //
+         // even though, doing so also closes the outputStream from the caller that this    //
+         // zipOutputStream is wrapping (and the caller will likely call close on too)...   //
+         /////////////////////////////////////////////////////////////////////////////////////
+         zipOutputStream.close();
+      }
+      catch(Exception e)
+      {
+         throw (new QReportingException("Error finishing Excel report", e));
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private void closeLastSheetIfOpen() throws IOException
+   {
+      //////////////////////////////////////////////////////////////////////////////////////////////////////
+      // if we have an active sheet writer:                                                               //
+      // - end the current sheet in the spreadsheet writer (write some closing xml, unless it's a pivot!) //
+      // - flush the contents through the activeSheetWriter                                               //
+      // - close the zip entry in the output stream.                                                      //
+      //////////////////////////////////////////////////////////////////////////////////////////////////////
+      if(activeSheetWriter != null)
+      {
+         if(!ReportType.PIVOT.equals(currentView.getType()))
+         {
+            sheetWriter.endSheet();
+         }
+
+         activeSheetWriter.flush();
+         zipOutputStream.closeEntry();
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** display formats is a map of field name to Excel format strings (e.g., $#,##0.00)
+    *******************************************************************************/
+   @Override
+   public void setDisplayFormats(Map<String, String> displayFormats)
+   {
+      this.excelCellFormats = new HashMap<>();
+      for(Map.Entry<String, String> entry : displayFormats.entrySet())
+      {
+         String excelFormat = DisplayFormat.getExcelFormat(entry.getValue());
+         if(excelFormat != null)
+         {
+            excelCellFormats.put(entry.getKey(), excelFormat);
+         }
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private void writePivotTable(QReportView pivotTableView, QReportView dataView) throws QReportingException
+   {
+      try
+      {
+         //////////////////////////////////////////////////////////////////////////////////
+         // write the xml file that is the pivot table sheet.                            //
+         // note that the ZipEntry here will have been started above in the start method //
+         //////////////////////////////////////////////////////////////////////////////////
+         activeSheetWriter.write("""
+            <?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+              <worksheet xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main" xmlns:r="http://schemas.openxmlformats.org/officeDocument/2006/relationships" xmlns:mx="http://schemas.microsoft.com/office/mac/excel/2008/main" xmlns:mc="http://schemas.openxmlformats.org/markup-compatibility/2006" xmlns:mv="urn:schemas-microsoft-com:mac:vml" xmlns:x14="http://schemas.microsoft.com/office/spreadsheetml/2009/9/main" xmlns:x15="http://schemas.microsoft.com/office/spreadsheetml/2010/11/main" xmlns:x14ac="http://schemas.microsoft.com/office/spreadsheetml/2009/9/ac" xmlns:xm="http://schemas.microsoft.com/office/excel/2006/main">
+                 <sheetPr>
+                   <outlinePr summaryBelow="0" summaryRight="0"/>
+                 </sheetPr>
+                 <sheetViews>
+                   <sheetView workbookViewId="0"/>
+                 </sheetViews>
+                 <sheetFormatPr customHeight="1" defaultColWidth="14.43" defaultRowHeight="15.0"/>
+                 <sheetData>
+                   <row r="1"/>
+                 </sheetData>
+              </worksheet>
+            """);
+         activeSheetWriter.flush();
+
+         ////////////////////////////////////////////////////////////////////////////
+         // start a new zip entry, for this pivot view's cacheDefinition reference //
+         ////////////////////////////////////////////////////////////////////////////
+         zipOutputStream.putNextEntry(new ZipEntry(pivotViewToCacheDefinitionReferenceMap.get(pivotTableView.getName())));
+
+         /////////////////////////////////////////////////////////
+         // prepare the xml for each field (e.g., w/ its label) //
+         /////////////////////////////////////////////////////////
+         List<String> cachedFieldElements = new ArrayList<>();
+         for(QFieldMetaData column : this.fieldsPerView.get(dataView.getName()))
+         {
+            cachedFieldElements.add(String.format("""
+               <cacheField numFmtId="0" name="%s">
+                  <sharedItems/>
+               </cacheField>
+               """, column.getLabel()));
+         }
+
+         /////////////////////////////////////////////////////////////////////////////////////
+         // write the xml file that is the pivot cache definition (structure only, no data) //
+         /////////////////////////////////////////////////////////////////////////////////////
+         activeSheetWriter = new OutputStreamWriter(zipOutputStream);
+         activeSheetWriter.write(String.format("""
+               <?xml version="1.0" encoding="UTF-8"?>
+               <pivotCacheDefinition xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main" xmlns:r="http://schemas.openxmlformats.org/officeDocument/2006/relationships" createdVersion="3" minRefreshableVersion="3" refreshedVersion="3" refreshedBy="Apache POI" refreshedDate="1.711395767702E12" refreshOnLoad="true" r:id="rId1">
+                 <cacheSource type="worksheet">
+                   <worksheetSource sheet="%s" ref="A1:%s%d"/>
+                 </cacheSource>
+                 <cacheFields count="%d">
+                   %s
+                 </cacheFields>
+               </pivotCacheDefinition>
+               """,
+            StreamedSheetWriter.cleanseValue(labelViewsByName.get(dataView.getName())),
+            CellReference.convertNumToColString(dataView.getColumns().size() - 1),
+            rowsPerView.get(dataView.getName()),
+            dataView.getColumns().size(),
+            StringUtils.join("\n", cachedFieldElements)));
+      }
+      catch(Exception e)
+      {
+         throw (new QReportingException("Error writing pivot table", e));
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   protected PoiExcelStylerInterface getStylerInterface()
+   {
+      return (new PlainPoiExcelStyler());
+   }
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/excel/poi/PlainPoiExcelStyler.java

@@ -0,0 +1,46 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.actions.reporting.excel.poi;
+
+
+import org.apache.poi.ss.usermodel.CreationHelper;
+import org.apache.poi.xssf.usermodel.XSSFCellStyle;
+import org.apache.poi.xssf.usermodel.XSSFWorkbook;
+
+
+/*******************************************************************************
+ ** Excel styler that does nothing - just takes defaults (which are all no-op) from the interface.
+ *******************************************************************************/
+public class PlainPoiExcelStyler implements PoiExcelStylerInterface
+{
+
+   /*******************************************************************************
+    ** ... sorry, but adding this gives us test coverage on this class, even though
+    ** we're just deferring to super...
+    *******************************************************************************/
+   @Override
+   public XSSFCellStyle createStyleForHeader(XSSFWorkbook workbook, CreationHelper createHelper)
+   {
+      return PoiExcelStylerInterface.super.createStyleForHeader(workbook, createHelper);
+   }
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/excel/poi/PoiExcelStylerInterface.java

@@ -0,0 +1,59 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.actions.reporting.excel.poi;
+
+
+import org.apache.poi.ss.usermodel.CreationHelper;
+import org.apache.poi.xssf.usermodel.XSSFCellStyle;
+import org.apache.poi.xssf.usermodel.XSSFWorkbook;
+
+
+/*******************************************************************************
+ ** Interface for classes that know how to apply styles to an Excel stream being
+ ** built by POI.
+ *******************************************************************************/
+public interface PoiExcelStylerInterface
+{
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   default XSSFCellStyle createStyleForTitle(XSSFWorkbook workbook, CreationHelper createHelper)
+   {
+      return (workbook.createCellStyle());
+   }
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   default XSSFCellStyle createStyleForHeader(XSSFWorkbook workbook, CreationHelper createHelper)
+   {
+      return (workbook.createCellStyle());
+   }
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   default XSSFCellStyle createStyleForFooter(XSSFWorkbook workbook, CreationHelper createHelper)
+   {
+      return (workbook.createCellStyle());
+   }
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/excel/poi/StreamedSheetWriter.java

@@ -0,0 +1,242 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.actions.reporting.excel.poi;
+
+
+import java.io.IOException;
+import java.io.Writer;
+import java.util.HashMap;
+import java.util.Map;
+import org.apache.poi.ss.util.CellReference;
+
+
+/*******************************************************************************
+ ** Write excel formatted XML to a Writer.
+ ** Originally from https://coderanch.com/t/548897/java/Generate-large-excel-POI
+ *******************************************************************************/
+public class StreamedSheetWriter
+{
+   private final Writer writer;
+   private       int    rowNo;
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public StreamedSheetWriter(Writer writer)
+   {
+      this.writer = writer;
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public void beginSheet() throws IOException
+   {
+      writer.write("""
+         <?xml version="1.0" encoding="UTF-8"?>
+            <worksheet xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main">
+               <sheetData>""");
+
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public void endSheet() throws IOException
+   {
+      writer.write("""
+            </sheetData>
+         </worksheet>""");
+   }
+
+
+
+   /*******************************************************************************
+    ** Insert a new row
+    **
+    ** @param rowNo 0-based row number
+    *******************************************************************************/
+   public void insertRow(int rowNo) throws IOException
+   {
+      writer.write("<row r=\"" + (rowNo + 1) + "\">\n");
+      this.rowNo = rowNo;
+   }
+
+
+
+   /*******************************************************************************
+    ** Insert row end marker
+    *******************************************************************************/
+   public void endRow() throws IOException
+   {
+      writer.write("</row>\n");
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public void createCell(int columnIndex, String value, int styleIndex) throws IOException
+   {
+      String ref = new CellReference(rowNo, columnIndex).formatAsString();
+      writer.write("<c r=\"" + ref + "\" t=\"inlineStr\"");
+      if(styleIndex != -1)
+      {
+         writer.write(" s=\"" + styleIndex + "\"");
+      }
+
+      String cleanValue = cleanseValue(value);
+
+      writer.write(">");
+      writer.write("<is><t>" + cleanValue + "</t></is>");
+      writer.write("</c>");
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public static String cleanseValue(String value)
+   {
+      if(value != null)
+      {
+         StringBuilder rs = new StringBuilder();
+         for(int i = 0; i < value.length(); i++)
+         {
+            char c = value.charAt(i);
+            if(c == '&')
+            {
+               rs.append("&amp;");
+            }
+            else if(c == '<')
+            {
+               rs.append("&lt;");
+            }
+            else if(c == '>')
+            {
+               rs.append("&gt;");
+            }
+            else if(c == '\'')
+            {
+               rs.append("&apos;");
+            }
+            else if(c == '"')
+            {
+               rs.append("&quot;");
+            }
+            else if (c < 32 && c != '\t' && c != '\n')
+            {
+               rs.append(' ');
+            }
+            else
+            {
+               rs.append(c);
+            }
+         }
+
+         Map<String, Integer> m = new HashMap<>();
+         m.computeIfAbsent("s", (s) -> 3);
+
+         value = rs.toString();
+      }
+
+      return (value);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public void createCell(int columnIndex, String value) throws IOException
+   {
+      createCell(columnIndex, value, -1);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public void createCell(int columnIndex, double value, int styleIndex) throws IOException
+   {
+      String ref = new CellReference(rowNo, columnIndex).formatAsString();
+      writer.write("<c r=\"" + ref + "\" t=\"n\"");
+      if(styleIndex != -1)
+      {
+         writer.write(" s=\"" + styleIndex + "\"");
+      }
+      writer.write(">");
+      writer.write("<v>" + value + "</v>");
+      writer.write("</c>");
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public void createCell(int columnIndex, double value) throws IOException
+   {
+      createCell(columnIndex, value, -1);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public void createCell(int columnIndex, Boolean value) throws IOException
+   {
+      createCell(columnIndex, value, -1);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public void createCell(int columnIndex, Boolean value, int styleIndex) throws IOException
+   {
+      String ref = new CellReference(rowNo, columnIndex).formatAsString();
+      writer.write("<c r=\"" + ref + "\" t=\"b\"");
+      if(styleIndex != -1)
+      {
+         writer.write(" s=\"" + styleIndex + "\"");
+      }
+      writer.write(">");
+      if(value != null)
+      {
+         writer.write("<v>" + (value ? 1 : 0) + "</v>");
+      }
+      writer.write("</c>");
+   }
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/scripts/ExecuteCodeAction.java

@@ -77,7 +77,6 @@ public class ExecuteCodeAction
    /*******************************************************************************
     **
     *******************************************************************************/
-   @SuppressWarnings("checkstyle:indentation")
    public void run(ExecuteCodeInput input, ExecuteCodeOutput output) throws QException, QCodeException
    {
       QCodeReference codeReference = input.getCodeReference();

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/scripts/RecordScriptTestInterface.java

@@ -23,6 +23,7 @@ package com.kingsrook.qqq.backend.core.actions.scripts;
 
 
 import java.io.Serializable;
+import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;
 import com.kingsrook.qqq.backend.core.actions.scripts.logging.BuildScriptLogAndScriptLogLineExecutionLogger;
@@ -97,7 +98,7 @@ public class RecordScriptTestInterface implements TestScriptActionInterface
          }
 
          QueryOutput queryOutput = new QueryAction().execute(new QueryInput(tableName)
-            .withFilter(new QQueryFilter(new QFilterCriteria(table.getPrimaryKeyField(), QCriteriaOperator.IN, recordPrimaryKeyList.split(","))))
+            .withFilter(new QQueryFilter(new QFilterCriteria(table.getPrimaryKeyField(), QCriteriaOperator.IN, Arrays.stream(recordPrimaryKeyList.split(",")).toList())))
             .withIncludeAssociations(true));
          if(CollectionUtils.nullSafeIsEmpty(queryOutput.getRecords()))
          {

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/scripts/RunAdHocRecordScriptAction.java

@@ -154,8 +154,9 @@ public class RunAdHocRecordScriptAction
          Method   qRecordListToApiRecordList = apiScriptUtilsClass.getMethod("qRecordListToApiRecordList", List.class, String.class, String.class, String.class);
          Object   apiRecordList              = qRecordListToApiRecordList.invoke(null, input.getRecordList(), input.getTableName(), scriptRevision.getApiName(), scriptRevision.getApiVersion());
 
-         // noinspection unchecked
-         return (ArrayList<? extends Serializable>) apiRecordList;
+         @SuppressWarnings("unchecked")
+         ArrayList<? extends Serializable> rs = (ArrayList<? extends Serializable>) apiRecordList;
+         return rs;
       }
       catch(ClassNotFoundException e)
       {

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/scripts/logging/BuildScriptLogAndScriptLogLineExecutionLogger.java

@@ -94,7 +94,7 @@ public class BuildScriptLogAndScriptLogLineExecutionLogger implements QCodeExecu
    protected QRecord buildDetailLogRecord(String logLine)
    {
       return (new QRecord()
-         .withValue("scriptLogId", scriptLog.getValue("id"))
+         .withValue("scriptLogId", scriptLog == null ? null : scriptLog.getValue("id"))
          .withValue("timestamp", Instant.now())
          .withValue("text", truncate(logLine)));
    }
@@ -145,6 +145,14 @@ public class BuildScriptLogAndScriptLogLineExecutionLogger implements QCodeExecu
       {
          this.executeCodeInput = executeCodeInput;
          this.scriptLog = buildHeaderRecord(executeCodeInput);
+
+         if(scriptLogLines != null)
+         {
+            for(QRecord scriptLogLine : scriptLogLines)
+            {
+               scriptLogLine.setValue("scriptLogId", scriptLog.getValue("id"));
+            }
+         }
       }
       catch(Exception e)
       {

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/AggregateAction.java

@@ -22,9 +22,12 @@
 package com.kingsrook.qqq.backend.core.actions.tables;
 
 
+import java.util.Collections;
 import com.kingsrook.qqq.backend.core.actions.ActionHelper;
 import com.kingsrook.qqq.backend.core.actions.interfaces.AggregateInterface;
 import com.kingsrook.qqq.backend.core.actions.tables.helpers.QueryStatManager;
+import com.kingsrook.qqq.backend.core.actions.values.ValueBehaviorApplier;
+import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.logging.QLogger;
 import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateInput;
@@ -58,6 +61,11 @@ public class AggregateAction
       QTableMetaData   table   = aggregateInput.getTable();
       QBackendMetaData backend = aggregateInput.getBackend();
 
+      ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      // apply any available field behaviors to the filter (noting that, if anything changes, a new filter is returned) //
+      ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      aggregateInput.setFilter(ValueBehaviorApplier.applyFieldBehaviorsToFilter(QContext.getQInstance(), table, aggregateInput.getFilter(), Collections.emptySet()));
+
       QueryStat queryStat = QueryStatManager.newQueryStat(backend, table, aggregateInput.getFilter());
 
       QBackendModuleDispatcher qBackendModuleDispatcher = new QBackendModuleDispatcher();
@@ -67,6 +75,10 @@ public class AggregateAction
       aggregateInterface.setQueryStat(queryStat);
       AggregateOutput aggregateOutput = aggregateInterface.execute(aggregateInput);
 
+      // todo, maybe, not real important? ValueBehaviorApplier.applyFieldBehaviors(ValueBehaviorApplier.Action.READ, QContext.getQInstance(), table, aggregateOutput.getResults(), null);
+      //  issue being, the signature there... it takes a list of QRecords, which aren't what we have...
+      //  do we want to ... idk, refactor all these behavior deals?  hmm... maybe a new interface/ for ones that do reads?  not sure.
+
       QueryStatManager.getInstance().add(queryStat);
 
       return aggregateOutput;

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/CountAction.java

@@ -22,9 +22,12 @@
 package com.kingsrook.qqq.backend.core.actions.tables;
 
 
+import java.util.Collections;
 import com.kingsrook.qqq.backend.core.actions.ActionHelper;
 import com.kingsrook.qqq.backend.core.actions.interfaces.CountInterface;
 import com.kingsrook.qqq.backend.core.actions.tables.helpers.QueryStatManager;
+import com.kingsrook.qqq.backend.core.actions.values.ValueBehaviorApplier;
+import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.logging.QLogger;
 import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput;
@@ -58,6 +61,11 @@ public class CountAction
       QTableMetaData   table   = countInput.getTable();
       QBackendMetaData backend = countInput.getBackend();
 
+      ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      // apply any available field behaviors to the filter (noting that, if anything changes, a new filter is returned) //
+      ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      countInput.setFilter(ValueBehaviorApplier.applyFieldBehaviorsToFilter(QContext.getQInstance(), table, countInput.getFilter(), Collections.emptySet()));
+
       QueryStat queryStat = QueryStatManager.newQueryStat(backend, table, countInput.getFilter());
 
       QBackendModuleDispatcher qBackendModuleDispatcher = new QBackendModuleDispatcher();

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/DeleteAction.java

@@ -35,9 +35,8 @@ import java.util.Set;
 import java.util.stream.Collectors;
 import com.kingsrook.qqq.backend.core.actions.ActionHelper;
 import com.kingsrook.qqq.backend.core.actions.audits.DMLAuditAction;
-import com.kingsrook.qqq.backend.core.actions.customizers.AbstractPostDeleteCustomizer;
-import com.kingsrook.qqq.backend.core.actions.customizers.AbstractPreDeleteCustomizer;
 import com.kingsrook.qqq.backend.core.actions.customizers.QCodeLoader;
+import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizerInterface;
 import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizers;
 import com.kingsrook.qqq.backend.core.actions.interfaces.DeleteInterface;
 import com.kingsrook.qqq.backend.core.actions.tables.helpers.ValidateRecordSecurityLockHelper;
@@ -250,7 +249,7 @@ public class DeleteAction
       //////////////////////////////////////////////////////////////
       // finally, run the post-delete customizer, if there is one //
       //////////////////////////////////////////////////////////////
-      Optional<AbstractPostDeleteCustomizer> postDeleteCustomizer = QCodeLoader.getTableCustomizer(AbstractPostDeleteCustomizer.class, table, TableCustomizers.POST_DELETE_RECORD.getRole());
+      Optional<TableCustomizerInterface> postDeleteCustomizer = QCodeLoader.getTableCustomizer(table, TableCustomizers.POST_DELETE_RECORD.getRole());
       if(postDeleteCustomizer.isPresent() && oldRecordList.isPresent())
       {
          ////////////////////////////////////////////////////////////////////////////
@@ -260,8 +259,7 @@ public class DeleteAction
 
          try
          {
-            postDeleteCustomizer.get().setDeleteInput(deleteInput);
-            List<QRecord> postCustomizerResult = postDeleteCustomizer.get().apply(recordsForCustomizer);
+            List<QRecord> postCustomizerResult = postDeleteCustomizer.get().postDelete(deleteInput, recordsForCustomizer);
 
             ///////////////////////////////////////////////////////
             // check if any records got errors in the customizer //
@@ -327,13 +325,11 @@ public class DeleteAction
       ///////////////////////////////////////////////////////////////////////////
       // after all validations, run the pre-delete customizer, if there is one //
       ///////////////////////////////////////////////////////////////////////////
-      Optional<AbstractPreDeleteCustomizer> preDeleteCustomizer = QCodeLoader.getTableCustomizer(AbstractPreDeleteCustomizer.class, table, TableCustomizers.PRE_DELETE_RECORD.getRole());
+      Optional<TableCustomizerInterface> preDeleteCustomizer = QCodeLoader.getTableCustomizer(table, TableCustomizers.PRE_DELETE_RECORD.getRole());
       List<QRecord>                         customizerResult    = oldRecordList.get();
       if(preDeleteCustomizer.isPresent())
       {
-         preDeleteCustomizer.get().setDeleteInput(deleteInput);
-         preDeleteCustomizer.get().setIsPreview(isPreview);
-         customizerResult = preDeleteCustomizer.get().apply(oldRecordList.get());
+         customizerResult = preDeleteCustomizer.get().preDelete(deleteInput, oldRecordList.get(), isPreview);
       }
 
       /////////////////////////////////////////////////////////////////////////
@@ -474,7 +470,7 @@ public class DeleteAction
                QRecord recordWithError = new QRecord();
                recordsWithErrors.add(recordWithError);
                recordWithError.setValue(primaryKeyField.getName(), primaryKeyValue);
-               recordWithError.addError(new NotFoundStatusMessage("No record was found to delete for " + primaryKeyField.getLabel() + " = " + primaryKeyValue));
+               recordWithError.addError(new NotFoundStatusMessage("No record was found to delete for " + Objects.requireNonNullElse(primaryKeyField.getLabel(), primaryKeyField.getName()) + " = " + primaryKeyValue));
             }
          }
       }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/GetAction.java

@@ -23,19 +23,23 @@ package com.kingsrook.qqq.backend.core.actions.tables;
 
 
 import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
 import com.kingsrook.qqq.backend.core.actions.ActionHelper;
-import com.kingsrook.qqq.backend.core.actions.customizers.AbstractPostQueryCustomizer;
 import com.kingsrook.qqq.backend.core.actions.customizers.QCodeLoader;
+import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizerInterface;
 import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizers;
 import com.kingsrook.qqq.backend.core.actions.interfaces.GetInterface;
 import com.kingsrook.qqq.backend.core.actions.tables.helpers.GetActionCacheHelper;
 import com.kingsrook.qqq.backend.core.actions.values.QPossibleValueTranslator;
 import com.kingsrook.qqq.backend.core.actions.values.QValueFormatter;
+import com.kingsrook.qqq.backend.core.actions.values.ValueBehaviorApplier;
 import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.logging.QLogger;
 import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetOutput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
@@ -45,10 +49,16 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.AdornmentType;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.FieldBehavior;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.FieldFilterBehavior;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleDispatcher;
 import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleInterface;
+import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
+import com.kingsrook.qqq.backend.core.utils.ObjectUtils;
+import com.kingsrook.qqq.backend.core.utils.Pair;
+import com.kingsrook.qqq.backend.core.utils.memoization.Memoization;
 
 
 /*******************************************************************************
@@ -57,20 +67,14 @@ import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleInterface;
  *******************************************************************************/
 public class GetAction
 {
-   private Optional<AbstractPostQueryCustomizer> postGetRecordCustomizer;
+   private static final QLogger LOG = QLogger.getLogger(GetAction.class);
+
+   private Optional<TableCustomizerInterface> postGetRecordCustomizer;
 
    private GetInput                 getInput;
    private QPossibleValueTranslator qPossibleValueTranslator;
 
-
-
-   /*******************************************************************************
-    **
-    *******************************************************************************/
-   public QRecord executeForRecord(GetInput getInput) throws QException
-   {
-      return (execute(getInput).getRecord());
-   }
+   private Memoization<Pair<String, String>, List<FieldFilterBehavior<?>>> getFieldFilterBehaviorMemoization = new Memoization<>();
 
 
 
@@ -87,7 +91,7 @@ public class GetAction
          throw (new QException("Requested to Get a record from an unrecognized table: " + getInput.getTableName()));
       }
 
-      postGetRecordCustomizer = QCodeLoader.getTableCustomizer(AbstractPostQueryCustomizer.class, table, TableCustomizers.POST_QUERY_RECORD.getRole());
+      postGetRecordCustomizer = QCodeLoader.getTableCustomizer(table, TableCustomizers.POST_QUERY_RECORD.getRole());
       this.getInput = getInput;
 
       QBackendModuleDispatcher qBackendModuleDispatcher = new QBackendModuleDispatcher();
@@ -107,11 +111,15 @@ public class GetAction
       }
 
       GetOutput getOutput;
+      boolean   usingDefaultGetInterface = false;
       if(getInterface == null)
       {
          getInterface = new DefaultGetInterface();
+         usingDefaultGetInterface = true;
       }
 
+      getInput = applyFieldBehaviors(getInput);
+
       getInterface.validateInput(getInput);
       getOutput = getInterface.execute(getInput);
 
@@ -123,10 +131,11 @@ public class GetAction
          new GetActionCacheHelper().handleCaching(getInput, getOutput);
       }
 
-      ////////////////////////////////////////////////////////
-      // if the record is found, perform post-actions on it //
-      ////////////////////////////////////////////////////////
-      if(getOutput.getRecord() != null)
+      ////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      // if the record is found, perform post-actions on it                                                         //
+      // unless the defaultGetInterface was used - as it just does a query, and the query will do the post-actions. //
+      ////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      if(getOutput.getRecord() != null && !usingDefaultGetInterface)
       {
          getOutput.setRecord(postRecordActions(getOutput.getRecord()));
       }
@@ -136,6 +145,119 @@ public class GetAction
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private GetInput applyFieldBehaviors(GetInput getInput)
+   {
+      QTableMetaData table = getInput.getTable();
+
+      try
+      {
+         if(getInput.getPrimaryKey() != null)
+         {
+            ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+            // if the input has a primary key, get its behaviors, then apply, and update the pkey in the input if the value is different //
+            ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+            List<FieldFilterBehavior<?>> fieldFilterBehaviors = getFieldFilterBehaviors(table, table.getPrimaryKeyField());
+            for(FieldFilterBehavior<?> fieldFilterBehavior : CollectionUtils.nonNullList(fieldFilterBehaviors))
+            {
+               QFilterCriteria pkeyCriteria    = new QFilterCriteria(table.getPrimaryKeyField(), QCriteriaOperator.EQUALS, getInput.getPrimaryKey());
+               QFilterCriteria updatedCriteria = ValueBehaviorApplier.apply(pkeyCriteria, QContext.getQInstance(), table, table.getField(table.getPrimaryKeyField()), fieldFilterBehavior);
+               if(updatedCriteria != pkeyCriteria)
+               {
+                  getInput.setPrimaryKey(updatedCriteria.getValues().get(0));
+               }
+            }
+         }
+         else if(getInput.getUniqueKey() != null)
+         {
+            ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+            // if the input has a unique key, get its behaviors, then apply, and update the ukey values in the input if any are different //
+            ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+            Map<String, Serializable> updatedUniqueKey = new HashMap<>(getInput.getUniqueKey());
+            for(String fieldName : getInput.getUniqueKey().keySet())
+            {
+               List<FieldFilterBehavior<?>> fieldFilterBehaviors = getFieldFilterBehaviors(table, fieldName);
+               for(FieldFilterBehavior<?> fieldFilterBehavior : CollectionUtils.nonNullList(fieldFilterBehaviors))
+               {
+                  QFilterCriteria ukeyCriteria    = new QFilterCriteria(fieldName, QCriteriaOperator.EQUALS, updatedUniqueKey.get(fieldName));
+                  QFilterCriteria updatedCriteria = ValueBehaviorApplier.apply(ukeyCriteria, QContext.getQInstance(), table, table.getField(table.getPrimaryKeyField()), fieldFilterBehavior);
+                  updatedUniqueKey.put(fieldName, updatedCriteria.getValues().get(0));
+               }
+            }
+            getInput.setUniqueKey(updatedUniqueKey);
+         }
+      }
+      catch(Exception e)
+      {
+         LOG.warn("Error applying field behaviors to get input - will run with original inputs", e);
+      }
+
+      return (getInput);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private List<FieldFilterBehavior<?>> getFieldFilterBehaviors(QTableMetaData tableMetaData, String fieldName)
+   {
+      Pair<String, String> key = new Pair<>(tableMetaData.getName(), fieldName);
+      return getFieldFilterBehaviorMemoization.getResult(key, (p) ->
+      {
+         List<FieldFilterBehavior<?>> rs = new ArrayList<>();
+         for(FieldBehavior<?> fieldBehavior : CollectionUtils.nonNullCollection(tableMetaData.getFields().get(fieldName).getBehaviors()))
+         {
+            if(fieldBehavior instanceof FieldFilterBehavior<?> fieldFilterBehavior)
+            {
+               rs.add(fieldFilterBehavior);
+            }
+         }
+         return (rs);
+      }).orElse(null);
+   }
+
+
+
+   /*******************************************************************************
+    ** shorthand way to call for the most common use-case, when you just want the
+    ** output record to be returned.
+    *******************************************************************************/
+   public QRecord executeForRecord(GetInput getInput) throws QException
+   {
+      return (execute(getInput).getRecord());
+   }
+
+
+
+   /*******************************************************************************
+    ** more shorthand way to call for the most common use-case, when you just want the
+    ** output record to be returned, and you just want to pass in a table name and primary key.
+    *******************************************************************************/
+   public static QRecord execute(String tableName, Serializable primaryKey) throws QException
+   {
+      GetAction getAction = new GetAction();
+      GetInput  getInput  = new GetInput(tableName).withPrimaryKey(primaryKey);
+      return getAction.executeForRecord(getInput);
+   }
+
+
+
+   /*******************************************************************************
+    ** more shorthand way to call for the most common use-case, when you just want the
+    ** output record to be returned, and you just want to pass in a table name and unique key
+    *******************************************************************************/
+   public static QRecord execute(String tableName, Map<String, Serializable> uniqueKey) throws QException
+   {
+      GetAction getAction = new GetAction();
+      GetInput  getInput  = new GetInput(tableName).withUniqueKey(uniqueKey);
+      return getAction.executeForRecord(getInput);
+   }
+
+
+
    /*******************************************************************************
     ** Run a GetAction by using the QueryAction instead (e.g., with a filter made
     ** from the pkey/ukey, and returning the single record if found).
@@ -202,7 +324,7 @@ public class GetAction
       }
       else
       {
-         throw (new QException("No primaryKey or uniqueKey was passed to Get"));
+         throw (new QException("Unable to get " + ObjectUtils.tryElse(() -> queryInput.getTable().getLabel(), queryInput.getTableName()) + ".  Missing required input."));
       }
 
       queryInput.setFilter(filter);
@@ -216,19 +338,21 @@ public class GetAction
     ** Run the necessary actions on a record.  This may include setting display values,
     ** translating possible values, and running post-record customizations.
     *******************************************************************************/
-   public QRecord postRecordActions(QRecord record)
+   public QRecord postRecordActions(QRecord record) throws QException
    {
       QRecord returnRecord = record;
       if(this.postGetRecordCustomizer.isPresent())
       {
-         returnRecord = postGetRecordCustomizer.get().apply(List.of(record)).get(0);
+         returnRecord = postGetRecordCustomizer.get().postQuery(getInput, List.of(record)).get(0);
       }
 
+      ValueBehaviorApplier.applyFieldBehaviors(ValueBehaviorApplier.Action.READ, QContext.getQInstance(), getInput.getTable(), List.of(record), null);
+
       if(getInput.getShouldTranslatePossibleValues())
       {
          if(qPossibleValueTranslator == null)
          {
-            qPossibleValueTranslator = new QPossibleValueTranslator(getInput.getInstance(), getInput.getSession());
+            qPossibleValueTranslator = new QPossibleValueTranslator(QContext.getQInstance(), QContext.getQSession());
          }
          qPossibleValueTranslator.translatePossibleValuesInRecords(getInput.getTable(), List.of(returnRecord));
       }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/InsertAction.java

@@ -28,6 +28,7 @@ import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
+import java.util.Objects;
 import java.util.Optional;
 import java.util.Set;
 import java.util.stream.Collectors;
@@ -37,9 +38,9 @@ import com.kingsrook.qqq.backend.core.actions.QBackendTransaction;
 import com.kingsrook.qqq.backend.core.actions.audits.DMLAuditAction;
 import com.kingsrook.qqq.backend.core.actions.automation.AutomationStatus;
 import com.kingsrook.qqq.backend.core.actions.automation.RecordAutomationStatusUpdater;
-import com.kingsrook.qqq.backend.core.actions.customizers.AbstractPostInsertCustomizer;
 import com.kingsrook.qqq.backend.core.actions.customizers.AbstractPreInsertCustomizer;
 import com.kingsrook.qqq.backend.core.actions.customizers.QCodeLoader;
+import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizerInterface;
 import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizers;
 import com.kingsrook.qqq.backend.core.actions.interfaces.InsertInterface;
 import com.kingsrook.qqq.backend.core.actions.tables.helpers.UniqueKeyHelper;
@@ -61,6 +62,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.tables.Association;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.UniqueKey;
 import com.kingsrook.qqq.backend.core.model.statusmessages.BadInputStatusMessage;
+import com.kingsrook.qqq.backend.core.model.statusmessages.DuplicateKeyBadInputStatusMessage;
 import com.kingsrook.qqq.backend.core.model.statusmessages.QWarningMessage;
 import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleDispatcher;
 import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleInterface;
@@ -168,13 +170,12 @@ public class InsertAction extends AbstractQActionFunction<InsertInput, InsertOut
       //////////////////////////////////////////////////////////////
       // finally, run the post-insert customizer, if there is one //
       //////////////////////////////////////////////////////////////
-      Optional<AbstractPostInsertCustomizer> postInsertCustomizer = QCodeLoader.getTableCustomizer(AbstractPostInsertCustomizer.class, table, TableCustomizers.POST_INSERT_RECORD.getRole());
+      Optional<TableCustomizerInterface> postInsertCustomizer = QCodeLoader.getTableCustomizer(table, TableCustomizers.POST_INSERT_RECORD.getRole());
       if(postInsertCustomizer.isPresent())
       {
          try
          {
-            postInsertCustomizer.get().setInsertInput(insertInput);
-            insertOutput.setRecords(postInsertCustomizer.get().apply(insertOutput.getRecords()));
+            insertOutput.setRecords(postInsertCustomizer.get().postInsert(insertInput, insertOutput.getRecords()));
          }
          catch(Exception e)
          {
@@ -226,37 +227,40 @@ public class InsertAction extends AbstractQActionFunction<InsertInput, InsertOut
     *******************************************************************************/
    public void performValidations(InsertInput insertInput, boolean isPreview) throws QException
    {
+      if(CollectionUtils.nullSafeIsEmpty(insertInput.getRecords()))
+      {
+         return;
+      }
+
       QTableMetaData table = insertInput.getTable();
 
       ///////////////////////////////////////////////////////////////////
       // load the pre-insert customizer and set it up, if there is one //
       // then we'll run it based on its WhenToRun value                //
       ///////////////////////////////////////////////////////////////////
-      Optional<AbstractPreInsertCustomizer> preInsertCustomizer = QCodeLoader.getTableCustomizer(AbstractPreInsertCustomizer.class, table, TableCustomizers.PRE_INSERT_RECORD.getRole());
+      Optional<TableCustomizerInterface> preInsertCustomizer = QCodeLoader.getTableCustomizer(table, TableCustomizers.PRE_INSERT_RECORD.getRole());
       if(preInsertCustomizer.isPresent())
       {
-         preInsertCustomizer.get().setInsertInput(insertInput);
-         preInsertCustomizer.get().setIsPreview(isPreview);
-         runPreInsertCustomizerIfItIsTime(insertInput, preInsertCustomizer, AbstractPreInsertCustomizer.WhenToRun.BEFORE_ALL_VALIDATIONS);
+         runPreInsertCustomizerIfItIsTime(insertInput, isPreview, preInsertCustomizer, AbstractPreInsertCustomizer.WhenToRun.BEFORE_ALL_VALIDATIONS);
       }
 
       setDefaultValuesInRecords(table, insertInput.getRecords());
 
-      ValueBehaviorApplier.applyFieldBehaviors(ValueBehaviorApplier.Action.INSERT, insertInput.getInstance(), table, insertInput.getRecords(), null);
+      ValueBehaviorApplier.applyFieldBehaviors(ValueBehaviorApplier.Action.INSERT, QContext.getQInstance(), table, insertInput.getRecords(), null);
 
-      runPreInsertCustomizerIfItIsTime(insertInput, preInsertCustomizer, AbstractPreInsertCustomizer.WhenToRun.BEFORE_UNIQUE_KEY_CHECKS);
+      runPreInsertCustomizerIfItIsTime(insertInput, isPreview, preInsertCustomizer, AbstractPreInsertCustomizer.WhenToRun.BEFORE_UNIQUE_KEY_CHECKS);
       setErrorsIfUniqueKeyErrors(insertInput, table);
 
-      runPreInsertCustomizerIfItIsTime(insertInput, preInsertCustomizer, AbstractPreInsertCustomizer.WhenToRun.BEFORE_REQUIRED_FIELD_CHECKS);
+      runPreInsertCustomizerIfItIsTime(insertInput, isPreview, preInsertCustomizer, AbstractPreInsertCustomizer.WhenToRun.BEFORE_REQUIRED_FIELD_CHECKS);
       if(insertInput.getInputSource().shouldValidateRequiredFields())
       {
          validateRequiredFields(insertInput);
       }
 
-      runPreInsertCustomizerIfItIsTime(insertInput, preInsertCustomizer, AbstractPreInsertCustomizer.WhenToRun.BEFORE_SECURITY_CHECKS);
+      runPreInsertCustomizerIfItIsTime(insertInput, isPreview, preInsertCustomizer, AbstractPreInsertCustomizer.WhenToRun.BEFORE_SECURITY_CHECKS);
       ValidateRecordSecurityLockHelper.validateSecurityFields(insertInput.getTable(), insertInput.getRecords(), ValidateRecordSecurityLockHelper.Action.INSERT);
 
-      runPreInsertCustomizerIfItIsTime(insertInput, preInsertCustomizer, AbstractPreInsertCustomizer.WhenToRun.AFTER_ALL_VALIDATIONS);
+      runPreInsertCustomizerIfItIsTime(insertInput, isPreview, preInsertCustomizer, AbstractPreInsertCustomizer.WhenToRun.AFTER_ALL_VALIDATIONS);
    }
 
 
@@ -290,13 +294,13 @@ public class InsertAction extends AbstractQActionFunction<InsertInput, InsertOut
    /*******************************************************************************
     **
     *******************************************************************************/
-   private void runPreInsertCustomizerIfItIsTime(InsertInput insertInput, Optional<AbstractPreInsertCustomizer> preInsertCustomizer, AbstractPreInsertCustomizer.WhenToRun whenToRun) throws QException
+   private void runPreInsertCustomizerIfItIsTime(InsertInput insertInput, boolean isPreview, Optional<TableCustomizerInterface> preInsertCustomizer, AbstractPreInsertCustomizer.WhenToRun whenToRun) throws QException
    {
       if(preInsertCustomizer.isPresent())
       {
-         if(whenToRun.equals(preInsertCustomizer.get().getWhenToRun()))
+         if(whenToRun.equals(preInsertCustomizer.get().whenToRunPreInsert(insertInput, isPreview)))
          {
-            insertInput.setRecords(preInsertCustomizer.get().apply(insertInput.getRecords()));
+            insertInput.setRecords(preInsertCustomizer.get().preInsert(insertInput, insertInput.getRecords(), isPreview));
          }
       }
    }
@@ -321,7 +325,7 @@ public class InsertAction extends AbstractQActionFunction<InsertInput, InsertOut
             {
                if(record.getValue(requiredField.getName()) == null || (requiredField.getType().isStringLike() && record.getValueString(requiredField.getName()).trim().equals("")))
                {
-                  record.addError(new BadInputStatusMessage("Missing value in required field: " + requiredField.getLabel()));
+                  record.addError(new BadInputStatusMessage("Missing value in required field: " + Objects.requireNonNullElse(requiredField.getLabel(), requiredField.getName())));
                }
             }
          }
@@ -416,7 +420,7 @@ public class InsertAction extends AbstractQActionFunction<InsertInput, InsertOut
                Optional<List<Serializable>> keyValues = UniqueKeyHelper.getKeyValues(table, uniqueKey, record);
                if(keyValues.isPresent() && (existingKeys.get(uniqueKey).contains(keyValues.get()) || keysInThisList.get(uniqueKey).contains(keyValues.get())))
                {
-                  record.addError(new BadInputStatusMessage("Another record already exists with this " + uniqueKey.getDescription(table)));
+                  record.addError(new DuplicateKeyBadInputStatusMessage("Another record already exists with this " + uniqueKey.getDescription(table)));
                   foundDupe = true;
                   break;
                }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/QueryAction.java

@@ -25,14 +25,15 @@ package com.kingsrook.qqq.backend.core.actions.tables;
 import java.io.Serializable;
 import java.util.ArrayList;
 import java.util.Collection;
+import java.util.Collections;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
 import java.util.Set;
 import com.kingsrook.qqq.backend.core.actions.ActionHelper;
-import com.kingsrook.qqq.backend.core.actions.customizers.AbstractPostQueryCustomizer;
 import com.kingsrook.qqq.backend.core.actions.customizers.QCodeLoader;
+import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizerInterface;
 import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizers;
 import com.kingsrook.qqq.backend.core.actions.interfaces.QueryInterface;
 import com.kingsrook.qqq.backend.core.actions.reporting.BufferedRecordPipe;
@@ -41,6 +42,7 @@ import com.kingsrook.qqq.backend.core.actions.tables.helpers.QueryActionCacheHel
 import com.kingsrook.qqq.backend.core.actions.tables.helpers.QueryStatManager;
 import com.kingsrook.qqq.backend.core.actions.values.QPossibleValueTranslator;
 import com.kingsrook.qqq.backend.core.actions.values.QValueFormatter;
+import com.kingsrook.qqq.backend.core.actions.values.ValueBehaviorApplier;
 import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.logging.QLogger;
@@ -73,7 +75,7 @@ public class QueryAction
 {
    private static final QLogger LOG = QLogger.getLogger(QueryAction.class);
 
-   private Optional<AbstractPostQueryCustomizer> postQueryRecordCustomizer;
+   private Optional<TableCustomizerInterface> postQueryRecordCustomizer;
 
    private QueryInput               queryInput;
    private QueryInterface           queryInterface;
@@ -100,7 +102,7 @@ public class QueryAction
       }
 
       QBackendMetaData backend = queryInput.getBackend();
-      postQueryRecordCustomizer = QCodeLoader.getTableCustomizer(AbstractPostQueryCustomizer.class, table, TableCustomizers.POST_QUERY_RECORD.getRole());
+      postQueryRecordCustomizer = QCodeLoader.getTableCustomizer(table, TableCustomizers.POST_QUERY_RECORD.getRole());
       this.queryInput = queryInput;
 
       if(queryInput.getRecordPipe() != null)
@@ -117,6 +119,11 @@ public class QueryAction
          }
       }
 
+      ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      // apply any available field behaviors to the filter (noting that, if anything changes, a new filter is returned) //
+      ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      queryInput.setFilter(ValueBehaviorApplier.applyFieldBehaviorsToFilter(QContext.getQInstance(), table, queryInput.getFilter(), Collections.emptySet()));
+
       QueryStat queryStat = QueryStatManager.newQueryStat(backend, table, queryInput.getFilter());
 
       QBackendModuleDispatcher qBackendModuleDispatcher = new QBackendModuleDispatcher();
@@ -151,6 +158,22 @@ public class QueryAction
 
 
 
+   /*******************************************************************************
+    ** shorthand way to call for the most common use-case, when you just want the
+    ** records to be returned, and you just want to pass in a table name and filter.
+    *******************************************************************************/
+   public static List<QRecord> execute(String tableName, QQueryFilter filter) throws QException
+   {
+      QueryAction queryAction = new QueryAction();
+      QueryInput  queryInput  = new QueryInput();
+      queryInput.setTableName(tableName);
+      queryInput.setFilter(filter);
+      QueryOutput queryOutput = queryAction.execute(queryInput);
+      return (queryOutput.getRecords());
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/
@@ -169,6 +192,7 @@ public class QueryAction
             nextLevelQueryInput.setTableName(association.getAssociatedTableName());
             nextLevelQueryInput.setIncludeAssociations(true);
             nextLevelQueryInput.setAssociationNamesToInclude(buildNextLevelAssociationNamesToInclude(association.getName(), queryInput.getAssociationNamesToInclude()));
+            nextLevelQueryInput.setTransaction(queryInput.getTransaction());
 
             QQueryFilter filter = new QQueryFilter();
             nextLevelQueryInput.setFilter(filter);
@@ -264,14 +288,16 @@ public class QueryAction
    {
       if(this.postQueryRecordCustomizer.isPresent())
       {
-         records = postQueryRecordCustomizer.get().apply(records);
+         records = postQueryRecordCustomizer.get().postQuery(queryInput, records);
       }
 
+      ValueBehaviorApplier.applyFieldBehaviors(ValueBehaviorApplier.Action.READ, QContext.getQInstance(), queryInput.getTable(), records, null);
+
       if(queryInput.getShouldTranslatePossibleValues())
       {
          if(qPossibleValueTranslator == null)
          {
-            qPossibleValueTranslator = new QPossibleValueTranslator(queryInput.getInstance(), queryInput.getSession());
+            qPossibleValueTranslator = new QPossibleValueTranslator(QContext.getQInstance(), QContext.getQSession());
          }
          qPossibleValueTranslator.translatePossibleValuesInRecords(queryInput.getTable(), records, queryInput.getQueryJoins(), queryInput.getFieldsToTranslatePossibleValues());
       }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/ReplaceAction.java

@@ -47,6 +47,7 @@ import com.kingsrook.qqq.backend.core.model.data.QRecord;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.UniqueKey;
 import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
+import org.apache.commons.lang.BooleanUtils;
 
 
 /*******************************************************************************
@@ -79,9 +80,11 @@ public class ReplaceAction extends AbstractQActionFunction<ReplaceInput, Replace
 
       try
       {
-         QTableMetaData table           = input.getTable();
-         UniqueKey      uniqueKey       = input.getKey();
-         String         primaryKeyField = table.getPrimaryKeyField();
+         QTableMetaData table                     = input.getTable();
+         UniqueKey      uniqueKey                 = input.getKey();
+         String         primaryKeyField           = table.getPrimaryKeyField();
+         boolean        allowNullKeyValuesToEqual = BooleanUtils.isTrue(input.getAllowNullKeyValuesToEqual());
+
          if(transaction == null)
          {
             transaction = QBackendTransaction.openFor(new InsertInput(input.getTableName()));
@@ -98,10 +101,11 @@ public class ReplaceAction extends AbstractQActionFunction<ReplaceInput, Replace
             // originally it was thought that we'd need to pass the filter in here           //
             // but, it's been decided not to.  the filter only applies to what we can delete //
             ///////////////////////////////////////////////////////////////////////////////////
-            Map<List<Serializable>, Serializable> existingKeys = UniqueKeyHelper.getExistingKeys(transaction, table, page, uniqueKey);
+            Map<List<Serializable>, Serializable> existingKeys = UniqueKeyHelper.getExistingKeys(transaction, table, page, uniqueKey, allowNullKeyValuesToEqual);
+
             for(QRecord record : page)
             {
-               Optional<List<Serializable>> keyValues = UniqueKeyHelper.getKeyValues(table, uniqueKey, record);
+               Optional<List<Serializable>> keyValues = UniqueKeyHelper.getKeyValues(table, uniqueKey, record, allowNullKeyValuesToEqual);
                if(keyValues.isPresent())
                {
                   if(existingKeys.containsKey(keyValues.get()))
@@ -153,6 +157,17 @@ public class ReplaceAction extends AbstractQActionFunction<ReplaceInput, Replace
             output.setDeleteOutput(deleteOutput);
          }
 
+         if(input.getSetPrimaryKeyInInsertedRecords())
+         {
+            for(int i = 0; i < insertList.size(); i++)
+            {
+               if(i < insertOutput.getRecords().size())
+               {
+                  insertList.get(i).setValue(primaryKeyField, insertOutput.getRecords().get(i).getValue(primaryKeyField));
+               }
+            }
+         }
+
          if(weOwnTheTransaction)
          {
             transaction.commit();

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/StorageAction.java

@@ -0,0 +1,120 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.actions.tables;
+
+
+import java.io.InputStream;
+import java.io.OutputStream;
+import com.kingsrook.qqq.backend.core.actions.ActionHelper;
+import com.kingsrook.qqq.backend.core.actions.interfaces.QStorageInterface;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.actions.tables.storage.StorageInput;
+import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleDispatcher;
+import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleInterface;
+
+
+/*******************************************************************************
+ ** Action to do (generally, "mass") storage operations in a backend.
+ **
+ ** e.g., store a (potentially large) file - specifically - by working with it
+ ** as either an InputStream or OutputStream.
+ **
+ ** May not be implemented in all backends.
+ **
+ *******************************************************************************/
+public class StorageAction
+{
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public OutputStream createOutputStream(StorageInput storageInput) throws QException
+   {
+      QBackendModuleInterface qBackendModuleInterface = preAction(storageInput);
+      QStorageInterface       storageInterface        = qBackendModuleInterface.getStorageInterface();
+      return (storageInterface.createOutputStream(storageInput));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public InputStream getInputStream(StorageInput storageInput) throws QException
+   {
+      QBackendModuleInterface qBackendModuleInterface = preAction(storageInput);
+      QStorageInterface       storageInterface        = qBackendModuleInterface.getStorageInterface();
+      return (storageInterface.getInputStream(storageInput));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private QBackendModuleInterface preAction(StorageInput storageInput) throws QException
+   {
+      ActionHelper.validateSession(storageInput);
+
+      if(storageInput.getTableName() == null)
+      {
+         throw (new QException("Table name was not specified in storage input"));
+      }
+
+      QTableMetaData table = storageInput.getTable();
+      if(table == null)
+      {
+         throw (new QException("A table named [" + storageInput.getTableName() + "] was not found in the active QInstance"));
+      }
+
+      QBackendMetaData         backend                  = storageInput.getBackend();
+      QBackendModuleDispatcher qBackendModuleDispatcher = new QBackendModuleDispatcher();
+      QBackendModuleInterface  qModule                  = qBackendModuleDispatcher.getQBackendModule(backend);
+      return (qModule);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public void makePublic(StorageInput storageInput) throws QException
+   {
+      QBackendModuleInterface qBackendModuleInterface = preAction(storageInput);
+      QStorageInterface       storageInterface        = qBackendModuleInterface.getStorageInterface();
+      storageInterface.makePublic(storageInput);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public String getDownloadURL(StorageInput storageInput) throws QException
+   {
+      QBackendModuleInterface qBackendModuleInterface = preAction(storageInput);
+      QStorageInterface       storageInterface        = qBackendModuleInterface.getStorageInterface();
+      return (storageInterface.getDownloadURL(storageInput));
+   }
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/UpdateAction.java

@@ -24,6 +24,7 @@ package com.kingsrook.qqq.backend.core.actions.tables;
 
 import java.io.Serializable;
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -34,9 +35,8 @@ import com.kingsrook.qqq.backend.core.actions.ActionHelper;
 import com.kingsrook.qqq.backend.core.actions.audits.DMLAuditAction;
 import com.kingsrook.qqq.backend.core.actions.automation.AutomationStatus;
 import com.kingsrook.qqq.backend.core.actions.automation.RecordAutomationStatusUpdater;
-import com.kingsrook.qqq.backend.core.actions.customizers.AbstractPostUpdateCustomizer;
-import com.kingsrook.qqq.backend.core.actions.customizers.AbstractPreUpdateCustomizer;
 import com.kingsrook.qqq.backend.core.actions.customizers.QCodeLoader;
+import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizerInterface;
 import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizers;
 import com.kingsrook.qqq.backend.core.actions.interfaces.UpdateInterface;
 import com.kingsrook.qqq.backend.core.actions.tables.helpers.ValidateRecordSecurityLockHelper;
@@ -69,6 +69,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.tables.Association;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 import com.kingsrook.qqq.backend.core.model.statusmessages.BadInputStatusMessage;
 import com.kingsrook.qqq.backend.core.model.statusmessages.NotFoundStatusMessage;
+import com.kingsrook.qqq.backend.core.model.statusmessages.QErrorMessage;
 import com.kingsrook.qqq.backend.core.model.statusmessages.QWarningMessage;
 import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleDispatcher;
 import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleInterface;
@@ -192,14 +193,12 @@ public class UpdateAction
       //////////////////////////////////////////////////////////////
       // finally, run the post-update customizer, if there is one //
       //////////////////////////////////////////////////////////////
-      Optional<AbstractPostUpdateCustomizer> postUpdateCustomizer = QCodeLoader.getTableCustomizer(AbstractPostUpdateCustomizer.class, table, TableCustomizers.POST_UPDATE_RECORD.getRole());
+      Optional<TableCustomizerInterface> postUpdateCustomizer = QCodeLoader.getTableCustomizer(table, TableCustomizers.POST_UPDATE_RECORD.getRole());
       if(postUpdateCustomizer.isPresent())
       {
          try
          {
-            postUpdateCustomizer.get().setUpdateInput(updateInput);
-            oldRecordList.ifPresent(l -> postUpdateCustomizer.get().setOldRecordList(l));
-            updateOutput.setRecords(postUpdateCustomizer.get().apply(updateOutput.getRecords()));
+            updateOutput.setRecords(postUpdateCustomizer.get().postUpdate(updateInput, updateOutput.getRecords(), oldRecordList));
          }
          catch(Exception e)
          {
@@ -253,7 +252,7 @@ public class UpdateAction
          behaviorsToOmit = Set.of(DynamicDefaultValueBehavior.MODIFY_DATE);
       }
 
-      ValueBehaviorApplier.applyFieldBehaviors(ValueBehaviorApplier.Action.UPDATE, updateInput.getInstance(), table, updateInput.getRecords(), behaviorsToOmit);
+      ValueBehaviorApplier.applyFieldBehaviors(ValueBehaviorApplier.Action.UPDATE, QContext.getQInstance(), table, updateInput.getRecords(), behaviorsToOmit);
       validatePrimaryKeysAreGiven(updateInput);
 
       if(oldRecordList.isPresent())
@@ -273,13 +272,10 @@ public class UpdateAction
       ///////////////////////////////////////////////////////////////////////////
       // after all validations, run the pre-update customizer, if there is one //
       ///////////////////////////////////////////////////////////////////////////
-      Optional<AbstractPreUpdateCustomizer> preUpdateCustomizer = QCodeLoader.getTableCustomizer(AbstractPreUpdateCustomizer.class, table, TableCustomizers.PRE_UPDATE_RECORD.getRole());
+      Optional<TableCustomizerInterface> preUpdateCustomizer = QCodeLoader.getTableCustomizer(table, TableCustomizers.PRE_UPDATE_RECORD.getRole());
       if(preUpdateCustomizer.isPresent())
       {
-         preUpdateCustomizer.get().setUpdateInput(updateInput);
-         preUpdateCustomizer.get().setIsPreview(isPreview);
-         oldRecordList.ifPresent(l -> preUpdateCustomizer.get().setOldRecordList(l));
-         updateInput.setRecords(preUpdateCustomizer.get().apply(updateInput.getRecords()));
+         updateInput.setRecords(preUpdateCustomizer.get().preUpdate(updateInput, updateInput.getRecords(), isPreview, oldRecordList));
       }
    }
 
@@ -340,6 +336,9 @@ public class UpdateAction
       QTableMetaData table           = updateInput.getTable();
       QFieldMetaData primaryKeyField = table.getField(table.getPrimaryKeyField());
 
+      /////////////////////////////////////////////////////////////
+      // todo - evolve to use lock tree (e.g., from multi-locks) //
+      /////////////////////////////////////////////////////////////
       List<RecordSecurityLock> onlyWriteLocks = RecordSecurityLockFilters.filterForOnlyWriteLocks(CollectionUtils.nonNullList(table.getRecordSecurityLocks()));
 
       for(List<QRecord> page : CollectionUtils.getPages(updateInput.getRecords(), 1000))
@@ -399,7 +398,12 @@ public class UpdateAction
                   QRecord      oldRecord = lookedUpRecords.get(value);
                   QFieldType   fieldType = table.getField(lock.getFieldName()).getType();
                   Serializable lockValue = ValueUtils.getValueAsFieldType(fieldType, oldRecord.getValue(lock.getFieldName()));
-                  ValidateRecordSecurityLockHelper.validateRecordSecurityValue(table, record, lock, lockValue, fieldType, ValidateRecordSecurityLockHelper.Action.UPDATE);
+
+                  List<QErrorMessage> errors = ValidateRecordSecurityLockHelper.validateRecordSecurityValue(table, lock, lockValue, fieldType, ValidateRecordSecurityLockHelper.Action.UPDATE, Collections.emptyMap());
+                  if(CollectionUtils.nullSafeHasContents(errors))
+                  {
+                     errors.forEach(e -> record.addError(e));
+                  }
                }
             }
          }